harness-everything 0.20.7 → 0.22.0

package/ci/fixtures/system-one-provider.cjs

@@ -0,0 +1,13 @@
+'use strict';
+// Deliberately synthetic transport fixture, never model-quality evidence.
+const mode = process.argv[2];
+if (mode === 'timeout') setInterval(() => {}, 1000);
+else if (mode === 'overflow') process.stdout.write('x'.repeat(2 * 1024 * 1024));
+else if (mode === 'exit') process.exit(3);
+else if (mode === 'invalid') console.log('not-json');
+else { let input = ''; process.stdin.on('data', c => { input += c; }); process.stdin.on('end', () => {
+  const req = JSON.parse(input);
+  console.log(JSON.stringify({ schemaVersion: 1, requestHash: req.requestHash, catalogHash: req.catalogHash,
+    model: { id: 'fixture', revision: 'fixture-v1', domain: mode === 'forms' ? 'forms-v1' : 'harness-routing-v1' },
+    scores: req.options.map((o, i) => ({ id: o.id, probability: i === 0 ? 1 : 0 })) }));
+}); }

package/ci/fixtures/system-one-resident-stub.py

@@ -0,0 +1,28 @@
+"""Stub scorer behind the real resident server code; never model-quality evidence."""
+import importlib.util
+from pathlib import Path
+import sys
+import time
+
+sys.dont_write_bytecode = True  # never leave __pycache__ in the source tree that packaging copies
+path = Path(__file__).resolve().parents[2] / 'harness-everything/scripts/system-one/cua_adapter.py'
+spec = importlib.util.spec_from_file_location('adapter', path)
+adapter = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(adapter)
+
+
+def factory(weights, manifest):
+    if manifest['modelId'] == 'stub-fail':
+        raise ValueError('load-failed')
+
+    def score(request):
+        if manifest['modelId'] == 'stub-slow':
+            time.sleep(3)
+        return [1.0] + [0.0] * (len(request['options']) - 1)
+    return score, {'context_tokens': 224, 'option_tokens': 96}
+
+
+if __name__ == '__main__':
+    if sys.argv[1:2] != ['--serve']:
+        sys.exit(2)
+    sys.exit(adapter.serve_main(sys.argv[2:], scorer_factory=factory))

package/ci/mechanism-packaging-bytecode.test.js

@@ -0,0 +1,61 @@
+'use strict';
+// Python bytecode written by local/CI test runs is an untracked build artifact; no distribution may ship it.
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const { copyDir } = require('../scripts/lib/skills');
+const { filesUnder } = require('../scripts/build-openai-submission');
+const root = path.resolve(__dirname, '..');
+const noise = files => files.filter(f => /(^|[\\/])__pycache__([\\/]|$)|\.py[co]$/.test(f));
+
+function plant(dir) {
+  fs.mkdirSync(path.join(dir, '__pycache__'), { recursive: true });
+  fs.writeFileSync(path.join(dir, '__pycache__', 'cua_adapter.cpython-313.pyc'), 'bytecode');
+  fs.writeFileSync(path.join(dir, 'stray.pyc'), 'bytecode');
+  fs.writeFileSync(path.join(dir, 'keep.py'), 'print(1)\n');
+}
+
+test('PK-01 skills installer copy and submission file walk skip Python bytecode', () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-pk-'));
+  try {
+    const src = path.join(tmp, 'src', 'scripts');
+    plant(src);
+    copyDir(path.join(tmp, 'src'), path.join(tmp, 'dest'));
+    const copied = filesUnder(path.join(tmp, 'dest'));
+    assert.deepEqual(noise(copied), []);
+    assert.ok(copied.includes('scripts/keep.py'));
+    const walked = filesUnder(path.join(tmp, 'src'));
+    assert.deepEqual(noise(walked), []);
+    assert.deepEqual(walked, ['scripts/keep.py']);
+  } finally { fs.rmSync(tmp, { recursive: true, force: true }); }
+});
+
+test('PK-02 plugin sync does not mirror bytecode from the canonical tree', () => {
+  const canonical = path.join(root, 'harness-everything', 'scripts', 'system-one');
+  const planted = [path.join(canonical, '__pycache__', 'cua_adapter.cpython-313.pyc'), path.join(canonical, 'stray.pyc')];
+  const hadCache = fs.existsSync(path.join(canonical, '__pycache__'));
+  try {
+    fs.mkdirSync(path.join(canonical, '__pycache__'), { recursive: true });
+    for (const file of planted) fs.writeFileSync(file, 'bytecode');
+    const sync = spawnSync(process.execPath, [path.join(root, 'scripts', 'sync-openai-plugin.js')], { cwd: root, encoding: 'utf8' });
+    assert.equal(sync.status, 0, sync.stderr);
+    const mirror = path.join(root, 'plugins', 'harness-everything', 'skills', 'harness-everything', 'scripts', 'system-one');
+    assert.deepEqual(noise(filesUnder(mirror)), []);
+    assert.equal(fs.existsSync(path.join(mirror, '__pycache__')), false);
+  } finally {
+    for (const file of planted) fs.rmSync(file, { force: true });
+    if (!hadCache) fs.rmSync(path.join(canonical, '__pycache__'), { recursive: true, force: true });
+  }
+});
+
+test('PK-03 Python tests do not write bytecode into the source tree', () => {
+  const canonical = path.join(root, 'harness-everything', 'scripts', 'system-one');
+  fs.rmSync(path.join(canonical, '__pycache__'), { recursive: true, force: true });
+  const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+  const run = spawnSync(python, [path.join(__dirname, 'system-one-adapter-test.py')], { encoding: 'utf8', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '' } });
+  assert.equal(run.status, 0, run.stderr);
+  assert.equal(fs.existsSync(path.join(canonical, '__pycache__')), false);
+});

package/ci/mechanism-system-one-contract.test.js

@@ -0,0 +1,48 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const { createRequest, validateRequest, decide } = require('../harness-everything/scripts/system-one/contract');
+const options = [{ id: 'tier1', text: 'Bounded edit' }, { id: 'tier3', text: 'Architecture' }];
+const request = () => createRequest('tier', '修正 typo', options);
+function response(req, p = [0.95, 0.05]) {
+  return { schemaVersion: 1, requestHash: req.requestHash, catalogHash: req.catalogHash,
+    model: { id: 'test', revision: 'test-v1', domain: 'harness-routing-v1' },
+    scores: req.options.map((o, i) => ({ id: o.id, probability: p[i] })) };
+}
+test('S1-C01 complete response, no mutation and repeatability', () => {
+  const req = request(); const res = response(req); const before = JSON.stringify({ req, res });
+  assert.deepEqual(decide(req, res), { status: 'accepted', reason: 'scored', selectedId: 'tier1', confidence: 0.95, margin: 0.8999999999999999, model: res.model });
+  assert.deepEqual(decide(req, res), decide(req, res));
+  assert.equal(JSON.stringify({ req, res }), before);
+  assert.deepEqual(req, request());
+  assert.equal(validateRequest(req), true);
+  assert.notEqual(req.requestHash, createRequest('tier', 'other', options).requestHash);
+  assert.notEqual(req.catalogHash, createRequest('tier', 'other', [...options].reverse()).catalogHash);
+});
+test('S1-C02 rejected input partitions and byte boundaries', () => {
+  for (const context of ['', ' ', null, 1, {}, 'x'.repeat(65537)]) assert.throws(() => createRequest('tier', context, options), { code: 'invalid-request' });
+  for (const task of ['', null, 'A', 'a'.repeat(81)]) assert.throws(() => createRequest(task, 'x', options));
+  for (const opts of [null, [], [options[0]], [options[0], options[0]], [{ id: 'x', text: '' }, options[1]], [{ ...options[0], extra: true }, options[1]], [{ id: 1, text: 'x' }, options[1]], [{ id: 'x', text: '中'.repeat(1366) }, options[1]], Array.from({ length: 257 }, (_, i) => ({ id: 'a' + i, text: 'x' }))]) assert.throws(() => createRequest('tier', 'x', opts));
+  const max = createRequest('a'.repeat(80), 'x'.repeat(65536), Array.from({ length: 256 }, (_, i) => ({ id: 'a' + i, text: 'x'.repeat(4096) })));
+  assert.equal(validateRequest(max), true);
+  for (const mutate of [r => { r.extra = 1; }, r => { delete r.context; }, r => { r.schemaVersion = 2; }, r => { r.options.reverse(); }, r => { r.context = 'replay'; }]) {
+    const r = request(); mutate(r); assert.throws(() => validateRequest(r));
+  }
+});
+test('S1-C03 complete output validation and no raw-data leakage', () => {
+  const req = request();
+  for (const mutate of [r => { r.schemaVersion = 2; }, r => { r.requestHash = 'stale'; }, r => { r.catalogHash = 'stale'; }, r => { r.scores.pop(); }, r => { r.scores.push(r.scores[0]); }, r => { r.scores.reverse(); }, r => { r.scores[1].id = r.scores[0].id; }, r => { r.scores[0].probability = NaN; }, r => { r.scores[0].probability = Infinity; }, r => { r.scores[0].probability = -1; }, r => { r.scores[0].probability = 1.1; }, r => { r.scores[0].probability = '0.95'; }, r => { r.scores[0].probability = 0.8; }, r => { r.scores[0].extra = 'secret'; }, r => { r.model.extra = 'secret'; }, r => { r.model.revision = ''; }, r => { delete r.model; }, r => { r.extra = 'secret'; }]) {
+    const res = response(req); mutate(res);
+    assert.deepEqual(decide(req, res), { status: 'invalid-output', reason: 'invalid-response', selectedId: null, confidence: null, margin: null, model: null });
+  }
+  for (const value of [null, [], 'invalid']) assert.equal(decide(req, value).status, 'invalid-output');
+});
+test('S1-C04 abstention and policy boundaries', () => {
+  const req = request(); const res = response(req);
+  res.model.domain = 'forms-v1'; assert.equal(decide(req, res).reason, 'domain-mismatch');
+  assert.equal(decide(req, response(req, [0.5, 0.5]), { minConfidence: 0, minMargin: 0 }).reason, 'tie');
+  assert.equal(decide(req, response(req, [0.8, 0.2])).reason, 'low-confidence');
+  assert.equal(decide(req, response(req, [0.6, 0.4]), { minConfidence: 0.5, minMargin: 0.3 }).reason, 'low-margin');
+  assert.equal(decide(req, response(req, [1, 0]), { minConfidence: 1, minMargin: 1 }).status, 'accepted');
+  for (const policy of [null, { minConfidence: NaN }, { minMargin: -1 }, { minConfidence: 2 }, { extra: 1 }, { domain: '' }]) assert.throws(() => decide(req, response(req), policy), { code: 'invalid-policy' });
+});

package/ci/mechanism-system-one-evaluation.test.js

@@ -0,0 +1,87 @@
+'use strict';
+const { test } = require('node:test');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const assert = require('node:assert/strict');
+const { createRequest } = require('../harness-everything/scripts/system-one/contract');
+const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const { validateCorpus, evaluate } = require('../harness-everything/scripts/system-one/evaluate');
+const make = () => ({ schemaVersion: 1, cases: [
+  { id: 'a', family: 'typo', split: 'holdout', language: 'en', source: 'human:fixture', reviewed: false, request: createRequest('tier', 'fix typo', TIER_OPTIONS), gold: 'tier1' },
+  { id: 'b', family: 'unclear', split: 'holdout', language: 'zh-TW', source: 'human:fixture', reviewed: false, request: createRequest('tier', '看看這個', TIER_OPTIONS), gold: null },
+] });
+const decision = id => ({ status: id ? 'accepted' : 'abstain', reason: id ? 'scored' : 'unclassified', selectedId: id, confidence: 1, margin: 1, model: { id: 'fixture', revision: 'v1', domain: 'harness-routing-v1' } });
+const oneHot = id => TIER_OPTIONS.map(o => (o.id === (id || 'unclassified') ? 1 : 0));
+function records() { return make().cases.map(c => ({ id: c.id, baseline: 'tier2', runs: [0, 1].map(() => ({ decision: decision(c.gold), scores: oneHot(c.gold), latencyMs: 20, coldStart: true })) })); }
+test('S1-E01 exact paired metrics, no promotion from fixtures or cold timing', () => {
+  const result = evaluate(make(), records());
+  assert.equal(result.model.accuracy, 1); assert.equal(result.model.coverage, 0.5);
+  assert.equal(result.model.acceptedPrecision, 1); assert.equal(result.model.acceptedErrorRate, 0);
+  assert.equal(result.model.macroF1, 0.5); assert.equal(result.baseline.accuracy, 0);
+  assert.equal(result.agreement, 1); assert.equal(result.latency.coldP95Ms, 20); assert.equal(result.latency.warmP95Ms, null);
+  assert.equal(result.rolloutReady, false); assert.equal(result.gates.reviewedHoldout, false);
+  assert.equal(result.gates.policyEvidence, false); assert.equal(result.gates.liveHostEvidence, false);
+  assert.deepEqual(evaluate(make(), records()), evaluate(make(), records()));
+});
+test('S1-E02 corpus completeness, family leakage and no duplicated prompts', () => {
+  assert.equal(validateCorpus(make()), true);
+  for (const mutate of [c => { c.schemaVersion = 2; }, c => { c.extra = true; }, c => { c.cases = []; }, c => { c.cases[1].id = 'a'; }, c => { c.cases[1].family = 'typo'; c.cases[1].split = 'train'; }, c => { c.cases[1].request = c.cases[0].request; }, c => { c.cases[0].gold = 'unknown'; }, c => { c.cases[0].language = 'unknown'; }, c => { c.cases[0].reviewed = 'yes'; }, c => { c.cases[0].source = ''; }, c => { c.cases[0].request = createRequest('tier', 'x', [{ id: 'x', text: 'x' }, { id: 'y', text: 'y' }]); }, c => { delete c.cases[0].family; }, c => { c.cases[0].extra = 1; }]) { const c = make(); mutate(c); assert.throws(() => validateCorpus(c)); }
+});
+test('S1-E03 missing/extra/invalid predictions cannot silently improve scores', () => {
+  for (const mutate of [r => r.pop(), r => r.push(r[0]), r => { r[0].id = 'unknown'; }, r => { r[0].runs.pop(); }, r => { r[0].runs[0].decision.selectedId = 'unknown'; }, r => { r[0].runs[0].latencyMs = NaN; }, r => { r[0].runs[0].latencyMs = -1; }, r => { r[0].runs[0].coldStart = 'false'; }, r => { r[0].runs[0].decision.status = 'abstain'; }, r => { r[0].baseline = 'unknown'; }, r => { r[0].runs[0].decision.model.revision = 'v2'; }]) { const r = records(); mutate(r); assert.throws(() => evaluate(make(), r)); }
+  for (const mutate of [r => { delete r[0].runs[0].scores; }, r => { r[0].runs[0].scores = [1, 0, 0]; }, r => { r[0].runs[0].scores = [0.5, 0.5, 0, 0]; }, r => { r[0].runs[0].scores = [1, 0, 0, NaN]; }, r => { r[0].runs[0].scores = null; }, r => { r[0].runs[0].decision = { status: 'unavailable', reason: 'provider-timeout' }; }]) { const r = records(); mutate(r); assert.throws(() => evaluate(make(), r)); }
+  const r = records(); r[0].runs[1].decision = decision('tier3'); r[0].runs[1].scores = oneHot('tier3');
+  assert.equal(evaluate(make(), r).agreement, 0.5);
+});
+test('S1-E06 repeatability compares the full score vector, not only top-2 summaries', () => {
+  const r = records();
+  const accepted = { status: 'accepted', reason: 'scored', selectedId: 'tier1', confidence: 0.9, margin: 0.85, model: { id: 'fixture', revision: 'v1', domain: 'harness-routing-v1' } };
+  r[0].runs[0] = { decision: { ...accepted }, scores: [0.9, 0.05, 0.03, 0.02], latencyMs: 20, coldStart: true };
+  r[0].runs[1] = { decision: { ...accepted }, scores: [0.9, 0.05, 0.02, 0.03], latencyMs: 20, coldStart: true };
+  const report = evaluate(make(), r);
+  assert.equal(report.agreement, 0.5); assert.equal(report.gates.repeatability, false);
+  r[0].runs[1].scores = [0.9, 0.05, 0.03, 0.02];
+  assert.equal(evaluate(make(), r).agreement, 1);
+  r.forEach(x => x.runs.forEach(v => { v.decision = { status: 'unavailable', reason: 'provider-config' }; v.scores = null; }));
+  assert.equal(evaluate(make(), r).agreement, 1);
+});
+test('S1-E04 zero coverage and wrong accepted predictions fail quality gates', () => {
+  const r = records(); r.forEach(x => x.runs.forEach(v => { v.decision = { status: 'unavailable', reason: 'provider-config' }; v.scores = null; }));
+  const report = evaluate(make(), r);
+  assert.equal(report.model.coverage, 0); assert.equal(report.model.acceptedPrecision, null); assert.equal(report.rolloutReady, false);
+  const wrong = records(); wrong.forEach(x => x.runs.forEach(v => { v.decision = decision('tier3'); v.scores = oneHot('tier3'); }));
+  assert.equal(evaluate(make(), wrong).model.acceptedErrorRate, 1);
+});
+test('S1-E05 real evaluation CLI repeats with only documented timing volatility', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-eval-'));
+  try {
+    const root = path.resolve(__dirname, '..');
+    const input = path.join(dir, 'corpus.json'); fs.writeFileSync(input, JSON.stringify(make()));
+    const run = () => {
+      const output = path.join(dir, 'report.json');
+      const child = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), input, path.join(dir, 'missing.json'), output], { encoding: 'utf8', cwd: root });
+      assert.equal(child.status, 0, child.stderr);
+      const report = JSON.parse(fs.readFileSync(output, 'utf8'));
+      assert.equal(report.model.coverage, 0); assert.equal(report.rolloutReady, false);
+      assert.equal(report.records[0].runs[0].decision.reason, 'provider-config');
+      assert.equal(report.records[0].runs[0].scores, null);
+      assert.deepEqual(report.evidence.source, { status: 'unavailable', reason: 'provider-config' });
+      assert.equal(report.evidence.environment.python, null);
+      assert.equal(report.gates.sourceProvenance, false);
+      report.records.forEach(r => r.runs.forEach(v => { v.latencyMs = 0; }));
+      report.latency.coldP95Ms = 0;
+      return { status: child.status, stdout: child.stdout, stderr: child.stderr, report, files: fs.readdirSync(dir).sort() };
+    };
+    assert.deepEqual(run(), run());
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+test('S1-E07 source provenance gate requires the pinned cua_s1 revision', () => {
+  const recorded = status => ({ status: 'recorded', provenance: { python: { implementation: 'CPython', version: '3.12.4' }, cuaS1: { distribution: 'cua-s1', version: '0.1.0', sourceRevision: null, pinnedRevision: 'b'.repeat(40), sourceRevisionStatus: status }, torch: null } });
+  assert.equal(evaluate(make(), records()).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), { status: 'unavailable', reason: 'provider-exit' }).gates.sourceProvenance, false);
+  for (const status of ['unavailable', 'mismatch']) assert.equal(evaluate(make(), records(), recorded(status)).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), recorded('pinned')).gates.sourceProvenance, true);
+  assert.equal(evaluate(make(), records(), recorded('pinned')).rolloutReady, false);
+});

package/ci/mechanism-system-one-install.test.js

@@ -0,0 +1,106 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const install = require('../harness-everything/scripts/system-one/install');
+const { validateManifest, PINNED_CUA_S1_REVISION } = require('../harness-everything/scripts/system-one/provider');
+const script = path.join(__dirname, '../harness-everything/scripts/system-one/install.js');
+const sha = buf => createHash('sha256').update(buf).digest('hex');
+
+test('S1-I01 pins stay consistent with the provider contract', () => {
+  const { PINS } = install;
+  assert.equal(PINS.cuaS1Revision, PINNED_CUA_S1_REVISION);
+  assert.ok(PINS.cuaS1Spec.includes(`@${PINNED_CUA_S1_REVISION}#subdirectory=libs/cua-s1/python`));
+  assert.match(PINS.modelRevision, /^[0-9a-f]{40}$/);
+  assert.equal(PINS.modelId, 'cua-ai/cua-s1-forms'); assert.equal(PINS.domain, 'forms-v1');
+  for (const file of Object.values(PINS.files)) {
+    assert.match(file.sha256, /^[0-9a-f]{64}$/); assert.ok(Number.isInteger(file.size) && file.size > 0);
+    assert.equal(install.fileUrl(file.name), `https://huggingface.co/cua-ai/cua-s1-forms/resolve/${PINS.modelRevision}/${file.name}`);
+  }
+  assert.equal(path.basename(PINS.files.config.name, '.json'), path.basename(PINS.files.weights.name, '.safetensors'));
+  const dir = path.resolve(os.tmpdir(), 'system-one');
+  const manifest = install.buildManifest(dir, path.join(dir, 'venv', 'python'));
+  assert.equal(validateManifest(manifest), true);
+  assert.equal(manifest.checkpoint, path.join(dir, 'checkpoints', PINS.modelRevision, PINS.files.weights.name));
+  assert.equal(manifest.weightsSha256, PINS.files.weights.sha256); assert.equal(manifest.configSha256, PINS.files.config.sha256);
+  assert.equal(manifest.revision, PINS.modelRevision); assert.equal(manifest.timeoutMs, 10000);
+  assert.equal(manifest.transport, 'resident'); assert.equal(manifest.idleTimeoutMs, 1800000);
+});
+
+test('S1-I02 interpreter selection accepts only the upstream-supported Python range', () => {
+  for (const v of ['3.11.0', '3.12.4', '3.13.6']) assert.equal(install.supportedPython(v), true);
+  for (const v of ['3.10.9', '3.14.0', '2.7.18', '3.13', '', 'x', null]) assert.equal(install.supportedPython(v), false);
+  const win = install.pythonCandidates('win32');
+  assert.deepEqual(win[0], ['py', '-3.13']);
+  assert.ok(win.every(c => !c.includes('-3.14')));
+  assert.deepEqual(install.pythonCandidates('linux')[0], ['python3.13']);
+  assert.equal(install.venvPython('/x', 'win32'), path.join('/x', 'venv', 'Scripts', 'python.exe'));
+  assert.equal(install.venvPython('/x', 'linux'), path.join('/x', 'venv', 'bin', 'python'));
+});
+
+test('S1-I03 argument parsing is explicit and rejects ambiguous input', () => {
+  const home = install.parseArgs([]);
+  assert.equal(home.dir, path.join(os.homedir(), '.agents', 'harness-everything', 'system-one'));
+  assert.equal(home.python, null); assert.equal(home.dryRun, false);
+  const dir = path.resolve(os.tmpdir(), 's1');
+  assert.deepEqual(install.parseArgs(['--dir', dir, '--python', 'py', '--dry-run']), { dir, python: 'py', dryRun: true });
+  for (const argv of [['--dir', 'relative'], ['--dir'], ['--python'], ['--unknown'], ['extra'], ['--dry-run', '--dry-run']]) {
+    assert.throws(() => install.parseArgs(argv), { message: 'usage' });
+  }
+});
+
+test('S1-I04 downloads are verified before an atomic write; a valid file is reused', async () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-install-'));
+  try {
+    const good = Buffer.from('pinned-bytes\r\n');
+    const spec = { name: 'x.json', sha256: sha(good), size: good.length };
+    const dest = path.join(dir, 'x.json');
+    let calls = 0;
+    const fetcher = bytes => async () => { calls++; return bytes; };
+    for (const bad of [Buffer.from('tampered-bytes'), Buffer.concat([good, Buffer.from('x')]), Buffer.alloc(0)]) {
+      await assert.rejects(install.ensureFile('https://example.invalid/x', dest, spec, fetcher(bad)), /download-verification/);
+      assert.equal(fs.existsSync(dest), false);
+      assert.deepEqual(fs.readdirSync(dir), []);
+    }
+    assert.equal(await install.ensureFile('https://example.invalid/x', dest, spec, fetcher(good)), 'downloaded');
+    assert.ok(fs.readFileSync(dest).equals(good));
+    const before = calls;
+    assert.equal(await install.ensureFile('https://example.invalid/x', dest, spec, fetcher(good)), 'verified-existing');
+    assert.equal(calls, before);
+    fs.writeFileSync(dest, 'corrupted');
+    assert.equal(await install.ensureFile('https://example.invalid/x', dest, spec, fetcher(good)), 'downloaded');
+    assert.ok(fs.readFileSync(dest).equals(good));
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-I06 child command output goes to stderr so stdout stays machine-readable JSON', () => {
+  const code = `require(${JSON.stringify(script)}).exec(process.execPath, ['-e', 'console.log("pip-noise")'])`;
+  const child = spawnSync(process.execPath, ['-e', code], { encoding: 'utf8' });
+  assert.equal(child.status, 0, child.stderr);
+  assert.equal(child.stdout, '');
+  assert.match(child.stderr, /pip-noise/);
+  const captured = spawnSync(process.execPath, ['-e', `process.stdout.write(require(${JSON.stringify(script)}).exec(process.execPath, ['-e', 'console.log("3.13.6")'], true))`], { encoding: 'utf8' });
+  assert.equal(captured.stdout, '3.13.6');
+});
+
+test('S1-I05 dry run prints the pinned plan and changes nothing, repeatably', () => {
+  const dir = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-dry-')), 'target');
+  try {
+    const run = () => spawnSync(process.execPath, [script, '--dir', dir, '--python', 'python-not-used', '--dry-run'], { encoding: 'utf8' });
+    const one = run(); const two = run();
+    assert.equal(one.status, 0, one.stderr); assert.equal(one.stdout, two.stdout);
+    const plan = JSON.parse(one.stdout);
+    assert.equal(plan.dryRun, true); assert.equal(plan.dir, dir);
+    assert.equal(plan.manifest, path.join(dir, 'manifest.json'));
+    assert.ok(plan.steps.some(s => s.includes(install.PINS.cuaS1Spec)));
+    assert.ok(plan.steps.some(s => s.includes(install.PINS.torchIndex)));
+    assert.deepEqual(plan.env, { HARNESS_SYSTEM_ONE_MODE: 'shadow', HARNESS_SYSTEM_ONE_CONFIG: path.join(dir, 'manifest.json') });
+    assert.equal(fs.existsSync(dir), false);
+    const bad = spawnSync(process.execPath, [script, '--dir', 'relative'], { encoding: 'utf8' });
+    assert.equal(bad.status, 2); assert.match(bad.stderr, /Usage/);
+  } finally { fs.rmSync(path.dirname(dir), { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-provider.test.js

@@ -0,0 +1,63 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const path = require('node:path');
+const fs = require('node:fs');
+const os = require('node:os');
+const { spawnSync } = require('node:child_process');
+const { createRequest, decide } = require('../harness-everything/scripts/system-one/contract');
+const { score, runProvider, validateManifest, provenance, validateProvenance, PINNED_CUA_S1_REVISION } = require('../harness-everything/scripts/system-one/provider');
+const fixture = path.join(__dirname, 'fixtures/system-one-provider.cjs');
+const request = createRequest('tier', '修正 typo; $(never execute)', [{ id: 'tier1', text: 'Small' }, { id: 'tier3', text: 'Large' }]);
+const manifest = () => ({ schemaVersion: 1, python: 'python3', checkpoint: path.resolve('model.safetensors'), weightsSha256: 'a'.repeat(64), configSha256: 'b'.repeat(64), modelId: 'test', revision: 'v1', domain: 'harness-routing-v1' });
+test('S1-P01 real IPC exact input/output repeated in clean child processes', () => {
+  const one = runProvider(request, process.execPath, [fixture, 'ok']);
+  const two = runProvider(request, process.execPath, [fixture, 'ok']);
+  assert.deepEqual(one, two); assert.equal(one.status, 'scored');
+  assert.equal(decide(request, one.response).selectedId, 'tier1');
+  assert.deepEqual(Object.keys(one).sort(), ['response', 'status']);
+});
+test('S1-P02 unavailable, timeout, output bounds, malformed output and repeatability', () => {
+  for (const [mode, reason] of [['timeout', 'provider-timeout'], ['overflow', 'provider-output-limit'], ['exit', 'provider-exit'], ['invalid', 'provider-json']]) {
+    // Only the timeout case uses a short budget; child startup under parallel CI load can exceed 300ms.
+    const call = () => runProvider(request, process.execPath, [fixture, mode], mode === 'timeout' ? 300 : 10000);
+    assert.deepEqual(call(), { status: 'unavailable', reason }); assert.deepEqual(call(), call());
+  }
+  assert.deepEqual(runProvider(request, path.join(os.tmpdir(), 'absent-system-one-python'), []), { status: 'unavailable', reason: 'provider-unavailable' });
+  assert.deepEqual(score(request, '/missing/system-one-manifest.json'), { status: 'unavailable', reason: 'provider-config' });
+});
+test('S1-P03 trusted manifest validation', () => {
+  assert.equal(validateManifest(manifest()), true);
+  assert.equal(validateManifest({ ...manifest(), timeoutMs: 1 }), true);
+  assert.equal(validateManifest({ ...manifest(), timeoutMs: 10000 }), true);
+  for (const change of [{ schemaVersion: 2 }, { python: '' }, { checkpoint: 'relative.safetensors' }, { checkpoint: path.resolve('model.pt') }, { weightsSha256: 'abc' }, { configSha256: null }, { modelId: '' }, { revision: null }, { domain: '' }, { timeoutMs: 0 }, { timeoutMs: 10001 }, { timeoutMs: '100' }, { unexpected: true }, { modelId: 'cua-ai/cua-s1-forms' }]) assert.throws(() => validateManifest({ ...manifest(), ...change }));
+  for (const key of Object.keys(manifest())) { const m = manifest(); delete m[key]; assert.throws(() => validateManifest(m)); }
+});
+test('S1-P04 Python artifact validation and byte-limit rejection without ML dependency', () => {
+  const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+  const result = spawnSync(python, [path.join(__dirname, 'system-one-adapter-test.py')], { encoding: 'utf8' });
+  assert.equal(result.status, 0, result.stderr || String(result.error));
+  assert.match(result.stderr, /OK/);
+});
+test('S1-P05 runtime/source provenance is recorded, and a missing or mismatched revision stays visible', () => {
+  const probe = rev => ({ schemaVersion: 1, python: { implementation: 'CPython', version: '3.12.4' }, cuaS1: { distribution: 'cua-s1', version: '0.1.0', sourceRevision: rev }, torch: '2.5.1' });
+  assert.equal(validateProvenance(probe(PINNED_CUA_S1_REVISION)).cuaS1.sourceRevisionStatus, 'pinned');
+  assert.equal(validateProvenance(probe('0'.repeat(40))).cuaS1.sourceRevisionStatus, 'mismatch');
+  assert.equal(validateProvenance(probe(null)).cuaS1.sourceRevisionStatus, 'unavailable');
+  assert.equal(validateProvenance(probe(null)).cuaS1.pinnedRevision, PINNED_CUA_S1_REVISION);
+  for (const mutate of [p => { p.schemaVersion = 2; }, p => { p.extra = 1; }, p => { delete p.torch; }, p => { p.python.version = ''; }, p => { p.python.extra = 1; }, p => { p.cuaS1.sourceRevision = 'abc'; }, p => { p.cuaS1.sourceRevision = 'A'.repeat(40); }, p => { p.cuaS1.version = 1; }, p => { p.cuaS1.extra = 1; }, p => { p.torch = 'x'.repeat(129); }, p => { p.python.version = '3.12\n'; }]) {
+    const p = probe(null); mutate(p); assert.throws(() => validateProvenance(p));
+  }
+  assert.deepEqual(provenance('/missing/system-one-manifest.json'), { status: 'unavailable', reason: 'provider-config' });
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-prov-'));
+  try {
+    const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+    const file = path.join(dir, 'manifest.json');
+    fs.writeFileSync(file, JSON.stringify({ ...manifest(), python, checkpoint: path.join(dir, 'absent.safetensors') }));
+    const one = provenance(file); const two = provenance(file);
+    assert.deepEqual(one, two);
+    assert.equal(one.status, 'recorded', JSON.stringify(one));
+    assert.match(one.provenance.python.version, /^3\.\d+\.\d+/);
+    if (one.provenance.cuaS1.distribution === null) assert.equal(one.provenance.cuaS1.sourceRevisionStatus, 'unavailable');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-resident.test.js

@@ -0,0 +1,185 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const resident = require('../harness-everything/scripts/system-one/resident');
+const provider = require('../harness-everything/scripts/system-one/provider');
+const { createRequest, decide } = require('../harness-everything/scripts/system-one/contract');
+const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+const stub = path.join(__dirname, 'fixtures/system-one-resident-stub.py');
+const request = createRequest('tier', 'fix a typo in README', TIER_OPTIONS);
+const sha = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+
+function fixture(modelId = 'stub') {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-resident-'));
+  const checkpoint = path.join(dir, 'model.safetensors');
+  fs.writeFileSync(checkpoint, 'synthetic-not-real-weights');
+  fs.writeFileSync(path.join(dir, 'model.json'), '{}');
+  const manifest = { schemaVersion: 1, python, checkpoint, weightsSha256: sha(checkpoint), configSha256: sha(path.join(dir, 'model.json')),
+    modelId, revision: 'v1', domain: 'harness-routing-v1', transport: 'resident', idleTimeoutMs: 60000 };
+  const manifestPath = path.join(dir, 'manifest.json');
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest));
+  const spawns = [];
+  const deps = { command: m => { spawns.push(m.modelId); return [python, [stub, '--serve']]; } };
+  return { dir, manifest, manifestPath, spawns, deps };
+}
+function cleanup(f) {
+  try { resident.stop(f.manifest, f.manifestPath); } catch (_) { /* best effort */ }
+  fs.rmSync(f.dir, { recursive: true, force: true });
+}
+
+test('S1-RS01 manifest transport fields and digest-bound file names', () => {
+  const f = fixture();
+  try {
+    assert.equal(provider.validateManifest(f.manifest), true);
+    for (const change of [{ transport: 'oneshot' }, { idleTimeoutMs: 60000 }, { idleTimeoutMs: 14400000 }]) assert.equal(provider.validateManifest({ ...f.manifest, ...change }), true);
+    for (const change of [{ transport: 'daemon' }, { transport: '' }, { idleTimeoutMs: 59999 }, { idleTimeoutMs: 14400001 }, { idleTimeoutMs: '60000' }]) {
+      assert.throws(() => provider.validateManifest({ ...f.manifest, ...change }));
+    }
+    const files = resident.stateFiles(f.manifest, f.manifestPath);
+    const digest = createHash('sha256').update(JSON.stringify(f.manifest)).digest('hex').slice(0, 16);
+    assert.equal(files.state, path.join(f.dir, `system-one-resident-${digest}.json`));
+    assert.equal(files.lock, path.join(f.dir, `system-one-resident-${digest}.starting`));
+    assert.notEqual(resident.stateFiles({ ...f.manifest, revision: 'v2' }, f.manifestPath).state, files.state);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS02 cold start never blocks, spawns once, then serves warm repeatable scores', () => {
+  const f = fixture();
+  try {
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.deepEqual(f.spawns, ['stub']);
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    assert.equal(f.spawns.length, 1);
+    const one = resident.score(request, f.manifest, f.manifestPath, f.deps);
+    const two = resident.score(request, f.manifest, f.manifestPath, f.deps);
+    assert.equal(one.status, 'scored'); assert.deepEqual(one, two);
+    assert.equal(decide(request, one.response).selectedId, 'tier1');
+    assert.equal(resident.status(f.manifest, f.manifestPath).running, true);
+    assert.equal(resident.stop(f.manifest, f.manifestPath), true);
+    const deadline = Date.now() + 5000;
+    while (fs.existsSync(resident.stateFiles(f.manifest, f.manifestPath).state) && Date.now() < deadline) sleep(50);
+    assert.equal(resident.status(f.manifest, f.manifestPath).running, false);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS03 stale state is removed and replaced; a wrong token is rejected without respawn', () => {
+  const f = fixture();
+  try {
+    const { state } = resident.stateFiles(f.manifest, f.manifestPath);
+    const dead = spawnSync(process.execPath, ['-e', '0']).pid;
+    fs.writeFileSync(state, JSON.stringify({ schemaVersion: 1, pid: dead, port: 9, token: 'a'.repeat(64), startedAt: 1 }));
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.equal(f.spawns.length, 1);
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const good = JSON.parse(fs.readFileSync(state, 'utf8'));
+    fs.writeFileSync(state, JSON.stringify({ ...good, token: 'b'.repeat(64) }));
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    assert.equal(f.spawns.length, 1);
+    fs.writeFileSync(state, JSON.stringify(good));
+    for (const bad of ['not-json', JSON.stringify({ ...good, extra: 1 }), JSON.stringify({ ...good, token: 'short' })]) {
+      fs.writeFileSync(state, bad);
+      assert.equal(resident.status(f.manifest, f.manifestPath).running, false);
+    }
+    fs.writeFileSync(state, JSON.stringify(good));
+  } finally { cleanup(f); }
+});
+
+test('S1-RS04 a slow server times out within the deadline and is not respawned', () => {
+  const f = fixture('stub-slow');
+  try {
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const start = Date.now();
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-timeout' });
+    assert.ok(Date.now() - start < 2500);
+    assert.equal(f.spawns.length, 1);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS05 startup failure is visible and throttled instead of respawning every prompt', () => {
+  const f = fixture('stub-fail');
+  try {
+    assert.equal(resident.score(request, f.manifest, f.manifestPath, f.deps).reason, 'provider-starting');
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), false);
+    const lock = JSON.parse(fs.readFileSync(resident.stateFiles(f.manifest, f.manifestPath).lock, 'utf8'));
+    assert.equal(lock.failed, 'load-failed');
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    assert.equal(f.spawns.length, 1);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS06 provider dispatch uses the real adapter and reports a missing ML runtime', () => {
+  const f = fixture();
+  try {
+    assert.deepEqual(provider.score(request, f.manifestPath), { status: 'unavailable', reason: 'provider-starting' });
+    const { lock } = resident.stateFiles(f.manifest, f.manifestPath);
+    const deadline = Date.now() + 20000;
+    while (Date.now() < deadline && !(fs.existsSync(lock) && JSON.parse(fs.readFileSync(lock, 'utf8')).failed)) sleep(100);
+    const probe = spawnSync(python, ['-c', 'import torch, cua_s1'], { encoding: 'utf8' });
+    if (probe.status !== 0) {
+      assert.equal(JSON.parse(fs.readFileSync(lock, 'utf8')).failed, 'load-failed');
+      assert.deepEqual(provider.score(request, f.manifestPath), { status: 'unavailable', reason: 'provider-unavailable' });
+    }
+    const off = { ...f.manifest, transport: 'oneshot' };
+    fs.writeFileSync(f.manifestPath, JSON.stringify(off));
+    assert.notEqual(provider.score(request, f.manifestPath).reason, 'provider-starting');
+  } finally { cleanup(f); }
+});
+
+test('S1-RS07 operator CLI reports status and stops a running server', () => {
+  const f = fixture();
+  try {
+    const cli = args => spawnSync(process.execPath, [path.join(__dirname, '../harness-everything/scripts/system-one/resident.js'), ...args], { encoding: 'utf8' });
+    assert.equal(JSON.parse(cli(['status', f.manifestPath]).stdout).running, false);
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const running = JSON.parse(cli(['status', f.manifestPath]).stdout);
+    assert.equal(running.running, true); assert.ok(!JSON.stringify(running).includes('token'));
+    assert.equal(cli(['stop', f.manifestPath]).status, 0);
+    const deadline = Date.now() + 5000;
+    while (JSON.parse(cli(['status', f.manifestPath]).stdout).running && Date.now() < deadline) sleep(50);
+    assert.equal(JSON.parse(cli(['status', f.manifestPath]).stdout).running, false);
+    assert.equal(cli(['bogus', f.manifestPath]).status, 2);
+    assert.equal(cli(['status', 'relative.json']).status, 2);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS08 an executable that cannot spawn is recorded as a visible, throttled failure', () => {
+  const f = fixture();
+  try {
+    const missing = path.join(f.dir, 'no-such-python.exe');
+    const deps = { command: () => { f.spawns.push('missing'); return [missing, []]; } };
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    assert.equal(JSON.parse(fs.readFileSync(resident.stateFiles(f.manifest, f.manifestPath).lock, 'utf8')).failed, 'spawn-failed');
+    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 2000, deps), false);
+    assert.deepEqual(f.spawns, ['missing']);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS09 a corrupted state file never leaves an orphan server behind', () => {
+  const f = fixture();
+  try {
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const { state } = resident.stateFiles(f.manifest, f.manifestPath);
+    const old = JSON.parse(fs.readFileSync(state, 'utf8'));
+    fs.writeFileSync(state, 'corrupt');
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const fresh = JSON.parse(fs.readFileSync(state, 'utf8'));
+    assert.notEqual(fresh.pid, old.pid);
+    // "Stopped serving" is the contract. On POSIX an exited child stays a zombie (kill(pid, 0) succeeds)
+    // until this blocked test process reaps it, so a refused/failed ping also proves the server is gone.
+    const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
+    const serving = () => alive(old.pid) && resident.callSync(old.port, { token: old.token, op: 'ping' }).reply?.ok === true;
+    const deadline = Date.now() + 5000;
+    while (serving() && Date.now() < deadline) sleep(100);
+    assert.equal(serving(), false);
+    assert.equal(resident.status(f.manifest, f.manifestPath).pid, fresh.pid);
+  } finally { cleanup(f); }
+});