harness-everything 0.22.0 → 0.23.0

package/ci/fixtures/system-one-worker-probe.js

@@ -0,0 +1,15 @@
+'use strict';
+// Preload (NODE_OPTIONS --require) that proves which System One client path ran.
+// HARNESS_TEST_WORKER_MODE=deny makes every worker thread fail to start; =count records
+// how many were started into HARNESS_TEST_WORKER_COUNT_FILE when the process exits.
+const fs = require('node:fs');
+const threads = require('node:worker_threads');
+const mode = process.env.HARNESS_TEST_WORKER_MODE;
+const Original = threads.Worker;
+let count = 0;
+if (mode === 'deny') {
+  threads.Worker = class { constructor() { throw new Error('worker-disabled-by-test'); } };
+} else if (mode === 'count') {
+  threads.Worker = class extends Original { constructor(...args) { count += 1; super(...args); } };
+  process.on('exit', () => fs.writeFileSync(process.env.HARNESS_TEST_WORKER_COUNT_FILE, String(count)));
+}

package/ci/mechanism-system-one-corpus.test.js

@@ -0,0 +1,154 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const { validateDraft, validateReviews, build, promptHash } = require('../scripts/system-one-corpus');
+const { validateCorpus } = require('../harness-everything/scripts/system-one/evaluate');
+const root = path.resolve(__dirname, '..');
+const draftPath = path.join(root, 'benchmarks/fixtures/system-one-holdout-draft.json');
+
+const draftCase = (id, language = 'en', proposedGold = 'tier1', prompt = `commit the change for case ${id}`) => ({
+  id, family: `family-${id}`, language, source: 'synthetic:authored', prompt, proposedGold, rationale: 'bounded git operation' });
+const draft = () => ({ schemaVersion: 1, cases: [draftCase('a'), draftCase('b', 'zh-TW', null, '繼續'), draftCase('c', 'en', 'tier2', 'fix the failing parser test and rerun it')] });
+const decision = (c, kind = 'accept', gold = c.proposedGold) => ({ id: c.id, promptHash: promptHash(c.prompt), decision: kind, gold, reviewer: 'human:repository-owner' });
+const reviews = decisions => ({ schemaVersion: 1, decisions });
+
+test('S1-K01 an unreviewed draft assembles into a valid corpus that cannot pass the review gate', () => {
+  const { corpus, summary } = build(draft(), null);
+  assert.equal(validateCorpus(corpus), true);
+  assert.ok(corpus.cases.every(c => c.reviewed === false && c.split === 'holdout'));
+  assert.deepEqual(corpus.cases.map(c => c.gold), ['tier1', null, 'tier2']);
+  assert.equal(corpus.cases[1].request.context, '繼續');
+  assert.equal(summary.unreviewed, 3);
+  assert.equal(summary.reviewedHoldoutGate, false);
+  assert.equal(promptHash('繼續'), createHash('sha256').update('繼續', 'utf8').digest('hex'));
+  assert.deepEqual(build(draft(), null), build(draft(), null));
+});
+
+test('S1-K02 accept, relabel and reject apply exactly; the reviewer gold wins', () => {
+  const d = draft();
+  const { corpus, summary } = build(d, reviews([decision(d.cases[0]), decision(d.cases[1], 'relabel', 'tier1'), decision(d.cases[2], 'reject', null)]));
+  assert.deepEqual(corpus.cases.map(c => [c.id, c.gold, c.reviewed]), [['a', 'tier1', true], ['b', 'tier1', true]]);
+  assert.equal(summary.rejected, 1);
+  assert.equal(summary.reviewed, 2);
+});
+
+test('S1-K03 a decision for changed prompt text is stale and never counts as review', () => {
+  const d = draft();
+  const r = reviews([decision(d.cases[0], 'relabel', 'tier2')]);
+  d.cases[0].prompt = 'commit and push the change for case a';
+  const { corpus, summary } = build(d, r);
+  assert.equal(corpus.cases[0].reviewed, false);
+  assert.equal(corpus.cases[0].gold, 'tier1');
+  assert.equal(summary.stale, 1);
+});
+
+test('S1-K04 malformed reviews are rejected instead of silently applied', () => {
+  const d = draft();
+  const bad = [
+    r => { r.decisions.push({ ...r.decisions[0] }); },
+    r => { r.decisions[0].id = 'unknown'; },
+    r => { r.decisions[0].decision = 'approve'; },
+    r => { r.decisions[0].gold = 'tier3'; },
+    r => { r.decisions[0] = decision(d.cases[0], 'relabel', 'tier1'); },
+    r => { r.decisions[0] = decision(d.cases[0], 'reject', 'tier1'); },
+    r => { r.decisions[0].reviewer = 'claude'; },
+    r => { r.decisions[0].reviewer = 'human:someone@example.com'; },
+    r => { r.decisions[0].extra = true; },
+    r => { r.decisions[0].promptHash = 'abc'; },
+    r => { r.schemaVersion = 2; },
+  ];
+  for (const mutate of bad) {
+    const r = reviews([decision(d.cases[0])]);
+    mutate(r);
+    assert.throws(() => build(d, r));
+  }
+  assert.equal(validateReviews(reviews([decision(d.cases[0])])), true);
+});
+
+test('S1-K05 draft validation: exact fields, unique ids and prompts, known sources, no private data', () => {
+  assert.equal(validateDraft(draft()), true);
+  const bad = [
+    d => { d.cases[1].id = 'a'; },
+    d => { d.cases[1].prompt = d.cases[0].prompt; },
+    d => { d.cases[0].source = 'real:history'; },
+    d => { d.cases[0].language = 'ja'; },
+    d => { d.cases[0].proposedGold = 'unclassified'; },
+    d => { d.cases[0].rationale = ''; },
+    d => { d.cases[0].extra = 1; },
+    d => { d.cases = []; },
+    d => { d.cases[0].prompt = 'mail the report to dev.lead@example.com'; },
+    d => { d.cases[0].prompt = 'open C:\\Users\\alice\\repo\\notes.md and fix it'; },
+    d => { d.cases[0].prompt = 'read /home/alice/project/config.yaml'; },
+    d => { d.cases[0].prompt = 'read /Users/alice/project/config.yaml'; },
+  ];
+  for (const mutate of bad) { const d = draft(); mutate(d); assert.throws(() => validateDraft(d)); }
+});
+
+test('S1-K06 the gate passes only with >=200 reviewed cases and >=50 per language', () => {
+  const make = (en, zh) => ({ schemaVersion: 1, cases: [
+    ...Array.from({ length: en }, (_, i) => draftCase(`en-${i}`, 'en', 'tier1', `rename variable number ${i} in the parser`)),
+    ...Array.from({ length: zh }, (_, i) => draftCase(`zh-${i}`, 'zh-TW', 'tier1', `把第 ${i} 個變數改名`)),
+  ] });
+  const all = d => reviews(d.cases.map(c => decision(c)));
+  const pass = make(150, 50);
+  assert.equal(build(pass, all(pass)).summary.reviewedHoldoutGate, true);
+  const fewZh = make(160, 49);
+  assert.equal(build(fewZh, all(fewZh)).summary.reviewedHoldoutGate, false);
+  const partial = make(150, 50);
+  assert.equal(build(partial, reviews(all(partial).decisions.slice(1))).summary.reviewedHoldoutGate, false);
+});
+
+test('S1-K07 CLI builds a corpus file and fails closed on invalid input', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-corpus-'));
+  try {
+    const cli = path.join(root, 'scripts/system-one-corpus.js');
+    const input = path.join(dir, 'draft.json'); const review = path.join(dir, 'reviews.json'); const out = path.join(dir, 'corpus.json');
+    const d = draft();
+    fs.writeFileSync(input, JSON.stringify(d));
+    fs.writeFileSync(review, JSON.stringify(reviews([decision(d.cases[0])])));
+    const ok = spawnSync(process.execPath, [cli, 'build', input, review, out], { encoding: 'utf8' });
+    assert.equal(ok.status, 0, ok.stderr);
+    assert.equal(JSON.parse(ok.stdout).reviewed, 1);
+    assert.equal(validateCorpus(JSON.parse(fs.readFileSync(out, 'utf8'))), true);
+    const none = spawnSync(process.execPath, [cli, 'build', input, '-', out], { encoding: 'utf8' });
+    assert.equal(none.status, 0, none.stderr);
+    assert.equal(JSON.parse(none.stdout).reviewed, 0);
+    fs.writeFileSync(review, '{"schemaVersion":1,"decisions":[{"id":"zzz"}]}');
+    assert.equal(spawnSync(process.execPath, [cli, 'build', input, review, out], { encoding: 'utf8' }).status, 1);
+    assert.equal(spawnSync(process.execPath, [cli, 'bogus'], { encoding: 'utf8' }).status, 1);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-K08 the committed holdout draft meets size, language, class and length requirements', () => {
+  const d = JSON.parse(fs.readFileSync(draftPath, 'utf8'));
+  assert.equal(validateDraft(d), true);
+  const { corpus } = build(d, null);
+  assert.equal(validateCorpus(corpus), true);
+  assert.ok(d.cases.length >= 220, 'headroom above 200 for rejected cases');
+  for (const lang of ['en', 'zh-TW']) assert.ok(d.cases.filter(c => c.language === lang).length >= 60, lang);
+  for (const gold of ['tier1', 'tier2', 'tier3', null]) assert.ok(d.cases.filter(c => c.proposedGold === gold).length >= 40, String(gold));
+  const bytes = d.cases.map(c => Buffer.byteLength(c.prompt, 'utf8'));
+  assert.ok(bytes.filter(b => b > 224).length / bytes.length >= 0.15, 'long prompts are represented');
+  assert.ok(bytes.filter(b => b <= 40).length / bytes.length >= 0.1, 'short prompts are represented');
+  assert.ok(d.cases.some(c => c.source === 'derived:local-history') && d.cases.some(c => c.source === 'synthetic:authored'));
+  assert.ok(new Set(d.cases.map(c => c.family)).size >= 40, 'enough distinct families');
+});
+
+test('S1-K09 the committed reviewed holdout is reproducible from owner decisions and meets the gate', () => {
+  const read = name => JSON.parse(fs.readFileSync(path.join(root, 'benchmarks/fixtures', name), 'utf8'));
+  const d = read('system-one-holdout-draft.json');
+  const r = read('system-one-holdout-reviews.json');
+  assert.equal(validateReviews(r), true);
+  assert.deepEqual(r.decisions.map(x => x.id).sort(), d.cases.map(c => c.id).sort(), 'every draft case has exactly one owner decision');
+  const { corpus, summary } = build(d, r);
+  assert.equal(summary.stale, 0);
+  assert.equal(summary.unreviewed, 0);
+  assert.equal(summary.reviewedHoldoutGate, true);
+  assert.deepEqual(read('system-one-holdout.json'), corpus, 'committed corpus equals the rebuild');
+  for (const gold of ['tier1', 'tier2', 'tier3', null]) assert.ok(corpus.cases.filter(c => c.gold === gold).length >= 15, String(gold));
+});

package/ci/mechanism-system-one-resident.test.js

@@ -33,6 +33,16 @@ function cleanup(f) {
   try { resident.stop(f.manifest, f.manifestPath); } catch (_) { /* best effort */ }
   fs.rmSync(f.dir, { recursive: true, force: true });
 }
+// A client view whose state file names the live server with a wrong token. The server polls only its own
+// state file and exits once that file stops naming it, so rewriting it would race the wrong-token request.
+function wrongTokenView(f) {
+  const good = JSON.parse(fs.readFileSync(resident.stateFiles(f.manifest, f.manifestPath).state, 'utf8'));
+  const viewDir = fs.mkdtempSync(path.join(f.dir, 'client-view-'));
+  const manifestPath = path.join(viewDir, 'manifest.json');
+  fs.writeFileSync(manifestPath, JSON.stringify(f.manifest));
+  fs.writeFileSync(resident.stateFiles(f.manifest, manifestPath).state, JSON.stringify({ ...good, token: 'b'.repeat(64) }));
+  return manifestPath;
+}
 
 test('S1-RS01 manifest transport fields and digest-bound file names', () => {
   const f = fixture();
@@ -80,10 +90,9 @@ test('S1-RS03 stale state is removed and replaced; a wrong token is rejected wit
     assert.equal(f.spawns.length, 1);
     assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
     const good = JSON.parse(fs.readFileSync(state, 'utf8'));
-    fs.writeFileSync(state, JSON.stringify({ ...good, token: 'b'.repeat(64) }));
-    assert.deepEqual(resident.score(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    assert.deepEqual(resident.score(request, f.manifest, wrongTokenView(f), f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
     assert.equal(f.spawns.length, 1);
-    fs.writeFileSync(state, JSON.stringify(good));
+    assert.equal(resident.status(f.manifest, f.manifestPath).running, true, 'a wrong token does not stop the server');
     for (const bad of ['not-json', JSON.stringify({ ...good, extra: 1 }), JSON.stringify({ ...good, token: 'short' })]) {
       fs.writeFileSync(state, bad);
       assert.equal(resident.status(f.manifest, f.manifestPath).running, false);
@@ -183,3 +192,105 @@ test('S1-RS09 a corrupted state file never leaves an orphan server behind', () =
     assert.equal(resident.status(f.manifest, f.manifestPath).pid, fresh.pid);
   } finally { cleanup(f); }
 });
+
+test('S1-RS10 scoreAsync matches the sync client and keeps every start/failure mapping', async () => {
+  const f = fixture();
+  try {
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.deepEqual(f.spawns, ['stub']);
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const sync = resident.score(request, f.manifest, f.manifestPath, f.deps);
+    assert.equal(sync.status, 'scored');
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), sync);
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), sync);
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, wrongTokenView(f), f.deps), { status: 'unavailable', reason: 'provider-unavailable' });
+    await assert.rejects(resident.scoreAsync({ ...request, extra: 1 }, f.manifest, f.manifestPath, f.deps));
+    assert.equal(f.spawns.length, 1);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS10b scoreAsync removes refused state, restarts once, and times out slow servers', async () => {
+  const f = fixture();
+  // A live PID that is not this test process: cleanup may kill whatever the state file names.
+  const sleeper = require('node:child_process').spawn(process.execPath, ['-e', 'setTimeout(() => {}, 60000)'], { stdio: 'ignore' });
+  try {
+    const net = require('node:net');
+    const closed = net.createServer();
+    await new Promise(resolve => closed.listen(0, '127.0.0.1', resolve));
+    const port = closed.address().port;
+    await new Promise(resolve => closed.close(resolve));
+    const { state } = resident.stateFiles(f.manifest, f.manifestPath);
+    fs.writeFileSync(state, JSON.stringify({ schemaVersion: 1, pid: sleeper.pid, port, token: 'a'.repeat(64), startedAt: 1 }));
+    assert.deepEqual(await resident.scoreAsync(request, f.manifest, f.manifestPath, f.deps), { status: 'unavailable', reason: 'provider-starting' });
+    assert.deepEqual(f.spawns, ['stub']);
+  } finally { sleeper.kill(); cleanup(f); }
+  const slow = fixture('stub-slow');
+  try {
+    assert.equal(resident.ensureReady(slow.manifest, slow.manifestPath, 20000, slow.deps), true);
+    const start = Date.now();
+    assert.deepEqual(await resident.scoreAsync(request, slow.manifest, slow.manifestPath, slow.deps), { status: 'unavailable', reason: 'provider-timeout' });
+    assert.ok(Date.now() - start < 2500);
+    assert.equal(slow.spawns.length, 1);
+  } finally { cleanup(slow); }
+});
+
+test('S1-RS10c provider.scoreAsync dispatches by transport with the sync result shapes', async () => {
+  const f = fixture();
+  try {
+    assert.deepEqual(await provider.scoreAsync(request, path.join(f.dir, 'missing.json')), { status: 'unavailable', reason: 'provider-config' });
+    assert.deepEqual(await provider.scoreAsync(request, 'relative.json'), { status: 'unavailable', reason: 'provider-config' });
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    assert.deepEqual(await provider.scoreAsync(request, f.manifestPath), provider.score(request, f.manifestPath));
+    const oneshot = path.join(f.dir, 'oneshot.json');
+    fs.writeFileSync(oneshot, JSON.stringify({ ...f.manifest, transport: 'oneshot' }));
+    assert.deepEqual(await provider.scoreAsync(request, oneshot), provider.score(request, oneshot));
+  } finally { cleanup(f); }
+});
+
+// The probe preload proves which client path ran inside a real child process.
+const probeEnv = (mode, extra = {}) => ({ ...process.env,
+  NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --require ./ci/fixtures/system-one-worker-probe.js`.trim(),
+  HARNESS_TEST_WORKER_MODE: mode, ...extra });
+
+test('S1-RS11 the hook entry pre-scores over async IPC: it still scores when worker threads cannot start', () => {
+  const f = fixture();
+  const root = path.resolve(__dirname, '..');
+  try {
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const env = probeEnv('deny', { HARNESS_SYSTEM_ONE_MODE: 'shadow', HARNESS_SYSTEM_ONE_CONFIG: f.manifestPath });
+    const kernel = path.join(root, 'harness-everything/scripts/kernel-router-core.js');
+    const diagnostic = out => JSON.parse(out.match(/=> SYSTEM ONE: (.*)/)[1]);
+    const argv = spawnSync(process.execPath, [kernel, 'fix a typo in README'], { cwd: root, encoding: 'utf8', env });
+    assert.equal(argv.status, 0, argv.stderr);
+    assert.equal(diagnostic(argv.stdout).status, 'accepted');
+    const hook = spawnSync(process.execPath, [kernel], { cwd: root, encoding: 'utf8', env, input: JSON.stringify({ prompt: 'fix a typo in README' }) });
+    assert.equal(hook.status, 0, hook.stderr);
+    assert.equal(diagnostic(hook.stdout).status, 'accepted');
+    const off = spawnSync(process.execPath, [kernel, 'fix a typo in README'], { cwd: root, encoding: 'utf8', env: { ...env, HARNESS_SYSTEM_ONE_MODE: 'off' } });
+    assert.equal(off.status, 0, off.stderr);
+    assert.doesNotMatch(off.stdout, /SYSTEM ONE/);
+    assert.equal(f.spawns.length, 1);
+  } finally { cleanup(f); }
+});
+
+test('S1-RS12 the evaluator times the async path: no worker thread per warm score', () => {
+  const f = fixture();
+  const root = path.resolve(__dirname, '..');
+  try {
+    assert.equal(resident.ensureReady(f.manifest, f.manifestPath, 20000, f.deps), true);
+    const countFile = path.join(f.dir, 'workers.txt');
+    const output = path.join(f.dir, 'report.json');
+    const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
+    const child = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, f.manifestPath, output],
+      { cwd: root, encoding: 'utf8', env: probeEnv('count', { HARNESS_TEST_WORKER_COUNT_FILE: countFile }) });
+    assert.equal(child.status, 0, child.stderr);
+    const report = JSON.parse(fs.readFileSync(output, 'utf8'));
+    assert.equal(report.evidence.residentReady, true);
+    // Every sample reached the warm server (the seed's one over-limit context is a declined provider-exit).
+    const runs = report.records.flatMap(r => r.runs);
+    assert.ok(runs.every(run => run.coldStart === false && ['accepted', 'provider-exit'].includes(run.decision.status === 'accepted' ? 'accepted' : run.decision.reason)));
+    assert.ok(runs.filter(run => run.decision.status === 'accepted').length >= runs.length - 2);
+    assert.ok(Number(fs.readFileSync(countFile, 'utf8')) < report.records.length * 2, 'warm scores must not start a worker thread each');
+  } finally { cleanup(f); }
+});

package/ci/mechanism-system-one-router.test.js

@@ -99,3 +99,28 @@ test('S1-R05 structural floor: a scorer can raise but never lower a deterministi
     assert.equal(plan.verification.independent, true);
   } finally { provider.score = original; console.log = log; if (mode === undefined) delete process.env.HARNESS_SYSTEM_ONE_MODE; else process.env.HARNESS_SYSTEM_ONE_MODE = mode; }
 });
+test('S1-R06 prescoreTier: off loads nothing; a matching request reuses the async result; mismatch and failure keep sync semantics', async () => {
+  const { prescoreTier } = require('../harness-everything/scripts/system-one/router');
+  const provider = require('../harness-everything/scripts/system-one/provider');
+  const forbidden = () => { throw new Error('must not invoke'); };
+  for (const env of [{}, { HARNESS_SYSTEM_ONE_MODE: 'off' }, { HARNESS_SYSTEM_ONE_MODE: 'bogus' }]) assert.equal(await prescoreTier('實作功能', env, forbidden), null);
+  assert.equal(await prescoreTier('', { HARNESS_SYSTEM_ONE_MODE: 'shadow' }, forbidden), null);
+  const env = { HARNESS_SYSTEM_ONE_MODE: 'prefer', HARNESS_SYSTEM_ONE_CONFIG: '/abs/manifest.json' };
+  const input = { prompt: '實作功能', tier: 'Unclassified' };
+  const configs = [];
+  const pre = await prescoreTier(input.prompt, env, async (req, config) => { configs.push(config); return scorer('tier2')(req); });
+  assert.deepEqual(configs, ['/abs/manifest.json']);
+  assert.deepEqual(selectTier(input, env, pre), selectTier(input, env, scorer('tier2')));
+  const original = provider.score;
+  try {
+    let used = 0;
+    provider.score = req => { used += 1; return scorer('tier3')(req); };
+    assert.equal(selectTier({ ...input, prompt: 'different prompt' }, env, pre).tier, 'Tier 3 (Macro Task)');
+    assert.equal(used, 1);
+  } finally { provider.score = original; }
+  const failing = await prescoreTier(input.prompt, env, async () => { throw new Error('secret'); });
+  const failed = selectTier(input, env, failing);
+  assert.deepEqual(failed, selectTier(input, env, () => { throw new Error('secret'); }));
+  assert.equal(failed.diagnostic.reason, 'provider-unavailable');
+  assert.ok(!JSON.stringify(failed).includes('secret'));
+});

package/docs/platform-capabilities.md

@@ -130,7 +130,12 @@ contract; this establishes no new live-host capability. Python, PyTorch and a
 compatible local checkpoint are optional external dependencies, not npm payloads;
 the explicit `npm run system-one:install` sets them up and hooks never run it.
 With `transport: "resident"`, an opted-in router hook may spawn one detached
-loopback-only Python server per manifest; host survival of that process after
-a hook exits is not live-verified.
+loopback-only Python server per manifest. One retained Windows + Claude Code
+headless session showed the hook-spawned server outliving the host; macOS/Linux
+and Codex hosts are not live-verified, and this does not change the matrix.
+Hooks read the mode from the host process environment: Claude Code settings
+`env` reaches hooks and every agent-run command, while a Codex started from an
+ordinary terminal sees it only when set at user/OS level. Keep the mode `off`
+until a Harness-trained checkpoint passes the rollout gates.
 Public Skills-only artifacts do not acquire lifecycle hooks from this feature.
 See [System One routing](system-one-routing.md) for limits, rollback and evidence.

package/docs/system-one-corpus.md

@@ -0,0 +1,103 @@
+# System One holdout corpus: labeling and review
+
+This document defines how the reviewed tier holdout for
+[System One](system-one-routing.md) is written, labeled, reviewed and
+assembled (issue [#233](https://github.com/dyphn1/Harness-everything/issues/233),
+Phase 3). The `reviewedHoldout` gate requires at least 200 holdout cases, at
+least 50 per language, and `reviewed: true` on every case.
+
+## Gold labels
+
+A gold label answers one question: **which tier should Harness route this prompt
+to, judged from the prompt alone?** The labels follow the tier model in
+[routing.md](routing.md#three-tier-routing-model) and the fixed tier catalog.
+
+| Gold | Use when the prompt asks for | Examples of shape |
+| --- | --- | --- |
+| `tier1` | Something answered or done without changing project behavior: Git/GitHub housekeeping (commit, push, open a PR, sync, tag, post given results as a comment), a status check, reading a log or searching and reporting, the smallest doc fix (a typo, a link, an ignore entry), or a reply the assistant can give directly, including non-engineering chores and discussion questions | "commit these changes", "is issue 88 still open?", "find every repo still using the old logging package", "close the other apps" |
+| `tier2` | A bounded change or investigation: any code, configuration or CI edit (a rename, a value, a revert), running tests or a build, filing an issue, explaining code that must be read first, most bug fixes, a focused review or benchmark, planning or synthesis that produces documents (phase plans, an ADR, an overview), and answers that settle open decisions for work in progress | "rename tmp to buffer", "run npm test", "the CLI fails with this error, fix it", "1. agree 2. keep the old name" |
+| `tier3` | New or changed behavior delivered with tests, and anything larger: a new flag or format, a behavior change across producers and tests, a large refactor, a dependency upgrade with breakage, a migration, a repository-wide change, a multi-phase implementation, or delegation to several agents | "add a --verbose flag with tests", "migrate to ESM", "dispatch agents to audit every repo" |
+| `null` | No actionable content: a bare continuation, "do what you said", a vague reaction, a question about what the assistant meant, or pasted status with no request | "go", "yes", "照剛剛說的改", "something is off here" |
+
+Rules:
+
+1. Label the work the prompt would start. A reply that settles decisions for
+   work in progress ("1. agree 2. use hash") is labeled by that work, not `null`.
+   Use `null` only when no work can be identified at all.
+2. Label the work requested, not the words used. Searching every repository and
+   reporting is `tier1`; changing every repository is `tier3`.
+3. Pick the smallest tier that covers the whole request.
+4. Explicit workflow words ("use fable", "no subagents") do not change the gold.
+   Precedence is a router policy, not a label.
+5. Never derive gold from the lexical router or any model output. A reviewer may
+   read the proposed gold, but the recorded gold is the reviewer's decision.
+
+These definitions are the repository owner's, recorded after the first review
+of the draft: the owner's labels moved many drafted cases one tier up, labeled
+context-dependent replies by their work, and gave non-engineering chores
+`tier1`. The draft's `proposedGold` values predate this and are only
+suggestions. Acknowledgements, reactions, host commands (`/compact`) and meeting
+notes were rejected from the holdout because they never reach routing as tasks.
+
+## Cases
+
+Each draft case has `id`, `family`, `language` (`en` or `zh-TW`), `source`,
+`prompt`, `proposedGold` and a one-line `rationale`.
+
+- **Family.** A family groups prompts that share one task scenario, such as
+  "commit and push" or "fix a CLI error from a pasted log". Paraphrases,
+  translations and variants of one scenario stay in one family, and a family
+  never spans splits. All families in this corpus are `holdout`. Later train or
+  validation data must use new families.
+- **Source.** `synthetic:authored` marks a prompt written for the corpus.
+  `derived:local-history` marks a prompt rewritten from the shape of a real
+  local prompt: same length class, task type and phrasing style, with every
+  name, path, product, customer, issue number and pasted log replaced. No real
+  prompt text, person, organization or local path is committed.
+- **Length.** The corpus follows the observed length profile of real local
+  prompts (median about 90 UTF-8 bytes, about one in five over 224 bytes, a
+  long tail with pasted logs), so that length limits are evaluated on realistic
+  input, not only short prompts.
+
+## Review
+
+A human reviewer decides every case on the review page. The possible decisions are:
+
+| Decision | Effect |
+| --- | --- |
+| `accept` | Gold is the proposed gold. |
+| `relabel` | Gold is the reviewer's chosen label. |
+| `reject` | The case is dropped: ambiguous even for a human, unrealistic, or it leaks private data. |
+
+Each decision records the case ID, the SHA-256 of the exact prompt text it
+reviewed (`promptHash`), the decision, the gold, and the reviewer role. The
+assembler `scripts/system-one-corpus.js` builds the evaluation corpus from the
+draft and the decisions:
+
+- A case with a valid decision becomes `reviewed: true` with the reviewer's gold.
+- A case without a decision, or whose prompt changed after review (hash
+  mismatch), stays `reviewed: false` with the proposed gold, so the gate cannot
+  pass on stale or missing review.
+- A rejected case is omitted.
+- The output passes `validateCorpus`, and the assembler reports per-language and
+  per-gold counts plus whether the `reviewedHoldout` gate would pass.
+
+The agent that drafts cases never records decisions. `reviewed: true` means a
+human reviewed that exact text. Review records store a role
+(`human:repository-owner`), not a name or account. When a decision contradicts
+the owner's decisions on sibling cases, the agent may ask the owner to confirm
+it. The owner's answer is recorded as the decision, and the agent never changes
+a decision on its own.
+
+## Committed files
+
+| File | Content |
+| --- | --- |
+| `benchmarks/fixtures/system-one-holdout-draft.json` | Draft cases with proposed gold |
+| `benchmarks/fixtures/system-one-holdout-reviews.json` | Owner decisions (ID, prompt hash, decision, gold, role) |
+| `benchmarks/fixtures/system-one-holdout.json` | Evaluation corpus, rebuilt with `node scripts/system-one-corpus.js build <draft> <reviews> <corpus>` |
+
+The reviewed holdout has 215 cases (11 rejected): en 93 and zh-TW 122; gold
+`tier1` 47, `tier2` 97, `tier3` 51 and `null` 20. It meets the `reviewedHoldout`
+gate. It says nothing about model quality until a `harness-routing-v1`
+checkpoint is evaluated on it.

package/docs/system-one-routing.md

@@ -148,9 +148,21 @@ format, which Phase 0 validation still checks. Any other failure returns
 `{ok: false, reason}` with a bounded code; no prompt, traceback, path or score is
 echoed in errors. The server never executes actions and is not given a driver.
 
-**Client.** The router API is synchronous, so the client performs one socket
-round trip on a worker thread and waits with `Atomics.wait` (1000 ms deadline).
-Results map to the existing reasons: timeout → `provider-timeout` (no respawn);
+**Client.** The hook entry pre-scores asynchronously. When `tier-router.js` runs
+as a script (the kernel and packaged hooks spawn it) in `shadow` or `prefer`
+mode, it builds the fixed-catalog tier request before routing and makes one
+direct async loopback round trip (`scoreAsync`), with no worker thread. Start,
+lock, stale-state, deadline (1000 ms) and reply-limit (1 MiB) rules are the same
+as the synchronous path. The router uses the pre-scored result only when its
+request hash equals the request the router builds; otherwise it scores
+synchronously. A pre-score error is reported exactly as a synchronous scorer
+error would be. In `off` mode the entry neither pre-scores nor contacts a
+provider. The synchronous `score` API remains for programmatic callers: it
+makes the same round trip on a worker thread and waits with `Atomics.wait`.
+Starting that worker costs 29–41 ms per call on the reference host, which is why
+the hook avoids it. The paired measurements are in
+[cua-s1-forms-resident-async-cpu-2026-09-23](../benchmarks/results/system-one/cua-s1-forms-resident-async-cpu-2026-09-23/README.md).
+Both paths map results to the existing reasons: timeout → `provider-timeout` (no respawn);
 refused connection or dead PID → stale state removed and a new start
 (`provider-starting`); `unauthorized` → `provider-unavailable`; any other
 server-declined request → `provider-exit`; malformed replies → `provider-json`.
@@ -158,7 +170,9 @@ server-declined request → `provider-exit`; malformed replies → `provider-jso
 **Operators.** `node harness-everything/scripts/system-one/resident.js
 start|status|stop <absolute manifest>` controls a server explicitly; `start`
 waits up to 60 s for readiness. The evaluation CLI waits for readiness before
-measuring, so resident runs are recorded with `coldStart: false`. The installer
+measuring, so resident runs are recorded with `coldStart: false`. It times
+`scoreAsync`, the path the hook uses, so `warmLatency` measures what a prompt
+actually pays. The installer
 writes `transport: "resident"`, verifies one resident score, and stops the server
 it started. Repository tests use a stub scorer behind the real server/protocol
 code; they prove the mechanism, not host survival of detached processes or model
@@ -195,6 +209,11 @@ selected candidate, model identity, confidence and margin; it omits prompt text,
 score vectors and artifact paths. The existing router contract schema is unchanged.
 Kernel/packaged entry points inherit these host environment variables. Prompts
 and hook payload fields cannot set model configuration or launch executables.
+Claude Code settings `env` is hot-reloaded and reaches hooks and every command
+the agent runs; Codex hooks inherit the `codex` process environment, so a Codex
+started from an ordinary terminal needs user/OS-level variables (which then apply
+to every program the user starts). The recommended setting is `off` until a
+`harness-routing-v1` checkpoint passes the rollout gates.
 Repository tests establish mechanism/package evidence only; actual host loading
 and semantic quality remain unverified until retained sessions demonstrate them.
 
@@ -208,7 +227,8 @@ The corpus is `{schemaVersion: 1, cases: [...]}`. Each case has exactly `id`,
 The complete fixed tier catalog must match. IDs and request hashes are unique;
 no family may span splits. Evaluation uses only holdout, never training labels.
 The committed seed corpus is a mechanism fixture, not an independently reviewed
-holdout. Expand and review it separately before making quality claims.
+holdout. The reviewed holdout is written, labeled and assembled as described in
+[system-one-corpus.md](system-one-corpus.md).
 
 The CLI runs the real lexical baseline and the configured provider twice per
 holdout case. Records contain IDs, baseline prediction, decision, model identity,

package/evals/pilot-router-nonsoftware/graders/no-tier-language.md

@@ -0,0 +1,6 @@
+---
+type: regex
+pattern: '(Tier 1|Tier 2|Tier 3|Routing Checkpoint)'
+match: not_contains
+weight: 1
+---

package/evals/pilot-router-nonsoftware/graders/skill-not-fired.md

@@ -0,0 +1,9 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?harness-everything"'
+min: 0
+max: 0
+arm: both
+weight: 1
+---

package/evals/pilot-router-nonsoftware/prompt.md

@@ -0,0 +1,8 @@
+---
+max_turns: 10
+allowed_tools: [Read, Glob, Grep, Skill]
+tags: [pilot, negative-trigger]
+description: "Casual non-software small talk must not engage the harness-everything router."
+---
+
+What's the difference between latte and cappuccino?

package/evals/pilot-router-tier3/graders/skill-fired.md

@@ -0,0 +1,6 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?harness-everything"'
+weight: 1
+---

package/evals/pilot-router-tier3/prompt.md

@@ -0,0 +1,8 @@
+---
+max_turns: 15
+allowed_tools: [Read, Glob, Grep, Skill]
+tags: [pilot, positive-trigger, tier-3]
+description: "Architectural, repo-wide request should route through the harness-everything router and land on Tier 3."
+---
+
+Route this coding request: split our monolith into two services (orders and billing) with an event bus between them; this touches deployment, data migration, and every team's code.

package/evals/pilot-tdd-bugfix/graders/skill-fired.md

@@ -0,0 +1,6 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?tdd"'
+weight: 1
+---

package/evals/pilot-tdd-bugfix/prompt.md

@@ -0,0 +1,8 @@
+---
+max_turns: 15
+allowed_tools: [Read, Glob, Grep, Skill]
+tags: [pilot, positive-trigger]
+description: "A bug-fix request that asks for a regression test should engage the tdd skill's RED/GREEN loop."
+---
+
+Fix a bug with a regression test: the discount calculation rounds wrong for prices ending in .99.

package/evals/pilot-tdd-trivia/graders/no-red-phase-claim.md

@@ -0,0 +1,7 @@
+---
+type: regex
+pattern: '(writing a failing test now|entering RED)'
+flags: i
+match: not_contains
+weight: 1
+---

package/evals/pilot-tdd-trivia/graders/skill-not-fired.md

@@ -0,0 +1,9 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?tdd"'
+min: 0
+max: 0
+arm: both
+weight: 1
+---

package/evals/pilot-tdd-trivia/prompt.md

@@ -0,0 +1,8 @@
+---
+max_turns: 10
+allowed_tools: [Read, Glob, Grep, Skill]
+tags: [pilot, negative-trigger]
+description: "A one-line trivia question about TDD terminology must not start a RED/GREEN test-writing loop."
+---
+
+In one sentence, what does the RED phase of test-driven development mean?

package/evals/pilot-verify-discussion/graders/no-verification-claim.md

@@ -0,0 +1,7 @@
+---
+type: regex
+pattern: '(running the verification report|verification passed)'
+flags: i
+match: not_contains
+weight: 1
+---

package/evals/pilot-verify-discussion/graders/skill-not-fired.md

@@ -0,0 +1,9 @@
+---
+type: tool_used
+tool: Skill
+input_match: '"skill"\s*:\s*"(?:[\w-]+:)?verification-loop"'
+min: 0
+max: 0
+arm: both
+weight: 1
+---

package/evals/pilot-verify-discussion/prompt.md

@@ -0,0 +1,8 @@
+---
+max_turns: 10
+allowed_tools: [Read, Glob, Grep, Skill]
+tags: [pilot, negative-trigger]
+description: "A design discussion with no completed work must not launch the verification-loop skill."
+---
+
+Sketch two possible approaches for caching API responses and list trade-offs. Don't write any code yet.