harness-everything 0.22.1 → 0.24.0

package/ci/fixtures/system-one-resident-stub.py

@@ -18,6 +18,9 @@ def factory(weights, manifest):
     def score(request):
         if manifest['modelId'] == 'stub-slow':
             time.sleep(3)
+        if manifest['modelId'] == 'stub-soft':
+            rest = 0.3 / (len(request['options']) - 1)
+            return [0.7] + [rest] * (len(request['options']) - 1)
         return [1.0] + [0.0] * (len(request['options']) - 1)
     return score, {'context_tokens': 224, 'option_tokens': 96}
 

package/ci/mechanism-system-one-collect.test.js

@@ -0,0 +1,129 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const collect = require('../scripts/system-one-collect');
+const root = path.resolve(__dirname, '..');
+const jsonl = rows => rows.map(r => JSON.stringify(r)).join('\n');
+
+// A fake home with one history file per source; each holds user text plus records that must be ignored.
+function fakeHome() {
+  const home = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-collect-'));
+  const appdata = path.join(home, 'AppData', 'Roaming');
+  const write = (rel, body) => { const p = path.join(home, rel); fs.mkdirSync(path.dirname(p), { recursive: true }); fs.writeFileSync(p, body); };
+  write('.claude/projects/p1/s-claude.jsonl', jsonl([
+    { type: 'user', message: { content: 'fix the typo in README' } },
+    { type: 'user', message: { content: [{ type: 'text', text: 'commit all changes' }] } },
+    { type: 'user', isMeta: true, message: { content: 'meta text must be skipped' } },
+    { type: 'user', isSidechain: true, message: { content: 'sidechain text must be skipped' } },
+    { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'x', content: 'tool output must be skipped' }] } },
+    { type: 'user', message: { content: '<system-reminder>injected</system-reminder>' } },
+    { type: 'user', message: { content: '<command-name>/compact</command-name>' } },
+    { type: 'assistant', message: { content: 'assistant text must be skipped' } },
+  ]));
+  write('.codex/sessions/2026/09/01/rollout-a.jsonl', jsonl([
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: '<environment_context>cwd</environment_context>' }] } },
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'duplicate of the event message' }] } },
+    { type: 'event_msg', payload: { type: 'user_message', message: 'add a --verbose flag with tests' } },
+  ]));
+  write('.codex/sessions/2026/09/02/rollout-b.jsonl', jsonl([
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: '<user_instructions>agents</user_instructions>' }] } },
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'older session without event messages' }] } },
+  ]));
+  write('.copilot/session-state/c1/events.jsonl', jsonl([
+    { type: 'system.message', data: { content: 'system prompt must be skipped' } },
+    { type: 'user.message', data: { content: 'run npm test and report failures' } },
+  ]));
+  write('AppData/Roaming/Code/User/workspaceStorage/w1/chatSessions/old.json', JSON.stringify({
+    requests: [{ message: { text: '把 parseCfg 改名成 parseConfig' } }, { message: { text: '' } }] }));
+  write('AppData/Roaming/Code/User/workspaceStorage/w1/chatSessions/new.jsonl', jsonl([
+    { kind: 0, v: { sessionId: 's', requests: [{ message: { text: '整個 repo 遷移到 ESM' } }] } },
+    { kind: 2, k: ['requests'], v: [{ message: { text: '繼續' } }] },
+    { kind: 2, k: ['requests', 0, 'response'], v: [{ value: 'response text must be skipped' }] },
+    { kind: 1, k: ['requests', 0, 'result'], v: { metadata: { renderedUserMessage: 'rendered prompt must be skipped' } } },
+  ]));
+  write('AppData/Roaming/Code/User/globalStorage/emptyWindowChatSessions/e.jsonl', jsonl([
+    { kind: 0, v: { requests: [{ message: { text: 'explain the backoff in retry.go' } }] } },
+  ]));
+  return { home, appdata };
+}
+const texts = rows => rows.map(r => r.text).sort();
+
+test('S1-C101 each source yields only owner-typed text', () => {
+  const { home, appdata } = fakeHome();
+  try {
+    const rows = collect.readSources({ home, appdata });
+    assert.deepEqual(texts(rows), texts([
+      { text: 'fix the typo in README' }, { text: 'commit all changes' }, { text: 'add a --verbose flag with tests' },
+      { text: 'older session without event messages' }, { text: 'run npm test and report failures' },
+      { text: '把 parseCfg 改名成 parseConfig' }, { text: '整個 repo 遷移到 ESM' }, { text: '繼續' }, { text: 'explain the backoff in retry.go' },
+    ]));
+    const bySource = Object.fromEntries(['claude', 'codex', 'copilot-cli', 'vscode'].map(s => [s, rows.filter(r => r.source === s).length]));
+    assert.deepEqual(bySource, { claude: 2, codex: 2, 'copilot-cli': 1, vscode: 4 });
+    assert.ok(rows.every(r => typeof r.family === 'string' && r.family.length > 0));
+    const claude = rows.filter(r => r.source === 'claude');
+    assert.equal(new Set(claude.map(r => r.family)).size, 1, 'one session file is one family');
+  } finally { fs.rmSync(home, { recursive: true, force: true }); }
+});
+
+test('S1-C102 filters, redaction and deduplication', () => {
+  assert.equal(collect.keep('go'), true);
+  assert.equal(collect.keep('x'.repeat(1024)), true);
+  assert.equal(collect.keep('x'.repeat(1025)), false);
+  assert.equal(collect.keep('中'.repeat(342)), false, '1026 UTF-8 bytes');
+  assert.equal(collect.keep('/compact'), false);
+  assert.equal(collect.keep('This session is being continued from a previous conversation that ran out of context.'), false);
+  assert.equal(collect.keep('   '), false);
+  assert.equal(collect.redact('mail dev.lead@example.com now'), 'mail <email> now');
+  assert.equal(collect.redact('open C:\\Users\\alice\\repo\\a.md'), 'open <home>\\repo\\a.md');
+  assert.equal(collect.redact('read /home/alice/p and /Users/bob/q'), 'read <home>/p and <home>/q');
+  const rows = collect.dedupe([{ text: 'fix  it', family: 'a' }, { text: 'fix it', family: 'b' }, { text: 'other', family: 'c' }]);
+  assert.deepEqual(rows.map(r => r.family), ['a', 'c']);
+});
+
+test('S1-C103 prompts that near-duplicate a holdout prompt are dropped', () => {
+  const holdout = ['把 parseCfg 改名成 parseConfig, 只有這個檔案有用到', 'Rebase this branch onto the latest main.'];
+  const guard = collect.leakGuard(holdout);
+  assert.equal(guard('把 parseCfg 改名成 parseConfig'), true, 'contained, longer than 12 chars');
+  assert.equal(guard('rebase this branch onto the latest main'), true, 'normalized near-duplicate');
+  assert.equal(guard('add pagination to the users endpoint'), false);
+  assert.equal(guard('main'), false, 'short containment does not count');
+  assert.ok(collect.jaccard('abcdef', 'abcdef') === 1 && collect.jaccard('abc', 'xyz') === 0);
+});
+
+test('S1-C104 splits are stable per family and near 15% validation', () => {
+  assert.equal(collect.splitFor('family-a'), collect.splitFor('family-a'));
+  const families = Array.from({ length: 2000 }, (_, i) => `session-${i}`);
+  const share = families.filter(f => collect.splitFor(f) === 'validation').length / families.length;
+  assert.ok(share > 0.12 && share < 0.18, String(share));
+  assert.ok(families.every(f => ['train', 'validation'].includes(collect.splitFor(f))));
+});
+
+test('S1-C105 CLI writes a deterministic local dataset outside the repository', () => {
+  const { home, appdata } = fakeHome();
+  const out = path.join(home, 'out');
+  const holdout = path.join(home, 'holdout.json');
+  fs.writeFileSync(holdout, JSON.stringify({ schemaVersion: 1, cases: [{ request: { context: 'Commit all changes.' } }] }));
+  try {
+    const run = () => spawnSync(process.execPath, [path.join(root, 'scripts/system-one-collect.js'), '--home', home, '--appdata', appdata, '--out', out, '--holdout', holdout],
+      { encoding: 'utf8', cwd: root });
+    const one = run();
+    assert.equal(one.status, 0, one.stderr);
+    const first = fs.readFileSync(path.join(out, 'prompts.jsonl'), 'utf8');
+    const stats = JSON.parse(one.stdout);
+    assert.equal(stats.kept, 8);
+    assert.equal(stats.dropped.leak, 1);
+    assert.deepEqual(Object.keys(stats.bySource).sort(), ['claude', 'codex', 'copilot-cli', 'vscode']);
+    assert.match(stats.sha256, /^[0-9a-f]{64}$/);
+    const rows = first.trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(r => Object.keys(r).sort().join() === 'family,id,source,split,text' && /^[0-9a-f]{16}$/.test(r.id)));
+    assert.equal(run().status, 0);
+    assert.equal(fs.readFileSync(path.join(out, 'prompts.jsonl'), 'utf8'), first, 'deterministic');
+    const inRepo = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-collect.js'), '--home', home, '--appdata', appdata, '--out', path.join(root, 'tmp-dataset'), '--holdout', holdout], { encoding: 'utf8', cwd: root });
+    assert.equal(inRepo.status, 1, 'refuses to write the dataset inside the repository');
+    assert.equal(fs.existsSync(path.join(root, 'tmp-dataset')), false);
+  } finally { fs.rmSync(home, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-corpus.test.js

@@ -0,0 +1,154 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const { validateDraft, validateReviews, build, promptHash } = require('../scripts/system-one-corpus');
+const { validateCorpus } = require('../harness-everything/scripts/system-one/evaluate');
+const root = path.resolve(__dirname, '..');
+const draftPath = path.join(root, 'benchmarks/fixtures/system-one-holdout-draft.json');
+
+const draftCase = (id, language = 'en', proposedGold = 'tier1', prompt = `commit the change for case ${id}`) => ({
+  id, family: `family-${id}`, language, source: 'synthetic:authored', prompt, proposedGold, rationale: 'bounded git operation' });
+const draft = () => ({ schemaVersion: 1, cases: [draftCase('a'), draftCase('b', 'zh-TW', null, '繼續'), draftCase('c', 'en', 'tier2', 'fix the failing parser test and rerun it')] });
+const decision = (c, kind = 'accept', gold = c.proposedGold) => ({ id: c.id, promptHash: promptHash(c.prompt), decision: kind, gold, reviewer: 'human:repository-owner' });
+const reviews = decisions => ({ schemaVersion: 1, decisions });
+
+test('S1-K01 an unreviewed draft assembles into a valid corpus that cannot pass the review gate', () => {
+  const { corpus, summary } = build(draft(), null);
+  assert.equal(validateCorpus(corpus), true);
+  assert.ok(corpus.cases.every(c => c.reviewed === false && c.split === 'holdout'));
+  assert.deepEqual(corpus.cases.map(c => c.gold), ['tier1', null, 'tier2']);
+  assert.equal(corpus.cases[1].request.context, '繼續');
+  assert.equal(summary.unreviewed, 3);
+  assert.equal(summary.reviewedHoldoutGate, false);
+  assert.equal(promptHash('繼續'), createHash('sha256').update('繼續', 'utf8').digest('hex'));
+  assert.deepEqual(build(draft(), null), build(draft(), null));
+});
+
+test('S1-K02 accept, relabel and reject apply exactly; the reviewer gold wins', () => {
+  const d = draft();
+  const { corpus, summary } = build(d, reviews([decision(d.cases[0]), decision(d.cases[1], 'relabel', 'tier1'), decision(d.cases[2], 'reject', null)]));
+  assert.deepEqual(corpus.cases.map(c => [c.id, c.gold, c.reviewed]), [['a', 'tier1', true], ['b', 'tier1', true]]);
+  assert.equal(summary.rejected, 1);
+  assert.equal(summary.reviewed, 2);
+});
+
+test('S1-K03 a decision for changed prompt text is stale and never counts as review', () => {
+  const d = draft();
+  const r = reviews([decision(d.cases[0], 'relabel', 'tier2')]);
+  d.cases[0].prompt = 'commit and push the change for case a';
+  const { corpus, summary } = build(d, r);
+  assert.equal(corpus.cases[0].reviewed, false);
+  assert.equal(corpus.cases[0].gold, 'tier1');
+  assert.equal(summary.stale, 1);
+});
+
+test('S1-K04 malformed reviews are rejected instead of silently applied', () => {
+  const d = draft();
+  const bad = [
+    r => { r.decisions.push({ ...r.decisions[0] }); },
+    r => { r.decisions[0].id = 'unknown'; },
+    r => { r.decisions[0].decision = 'approve'; },
+    r => { r.decisions[0].gold = 'tier3'; },
+    r => { r.decisions[0] = decision(d.cases[0], 'relabel', 'tier1'); },
+    r => { r.decisions[0] = decision(d.cases[0], 'reject', 'tier1'); },
+    r => { r.decisions[0].reviewer = 'claude'; },
+    r => { r.decisions[0].reviewer = 'human:someone@example.com'; },
+    r => { r.decisions[0].extra = true; },
+    r => { r.decisions[0].promptHash = 'abc'; },
+    r => { r.schemaVersion = 2; },
+  ];
+  for (const mutate of bad) {
+    const r = reviews([decision(d.cases[0])]);
+    mutate(r);
+    assert.throws(() => build(d, r));
+  }
+  assert.equal(validateReviews(reviews([decision(d.cases[0])])), true);
+});
+
+test('S1-K05 draft validation: exact fields, unique ids and prompts, known sources, no private data', () => {
+  assert.equal(validateDraft(draft()), true);
+  const bad = [
+    d => { d.cases[1].id = 'a'; },
+    d => { d.cases[1].prompt = d.cases[0].prompt; },
+    d => { d.cases[0].source = 'real:history'; },
+    d => { d.cases[0].language = 'ja'; },
+    d => { d.cases[0].proposedGold = 'unclassified'; },
+    d => { d.cases[0].rationale = ''; },
+    d => { d.cases[0].extra = 1; },
+    d => { d.cases = []; },
+    d => { d.cases[0].prompt = 'mail the report to dev.lead@example.com'; },
+    d => { d.cases[0].prompt = 'open C:\\Users\\alice\\repo\\notes.md and fix it'; },
+    d => { d.cases[0].prompt = 'read /home/alice/project/config.yaml'; },
+    d => { d.cases[0].prompt = 'read /Users/alice/project/config.yaml'; },
+  ];
+  for (const mutate of bad) { const d = draft(); mutate(d); assert.throws(() => validateDraft(d)); }
+});
+
+test('S1-K06 the gate passes only with >=200 reviewed cases and >=50 per language', () => {
+  const make = (en, zh) => ({ schemaVersion: 1, cases: [
+    ...Array.from({ length: en }, (_, i) => draftCase(`en-${i}`, 'en', 'tier1', `rename variable number ${i} in the parser`)),
+    ...Array.from({ length: zh }, (_, i) => draftCase(`zh-${i}`, 'zh-TW', 'tier1', `把第 ${i} 個變數改名`)),
+  ] });
+  const all = d => reviews(d.cases.map(c => decision(c)));
+  const pass = make(150, 50);
+  assert.equal(build(pass, all(pass)).summary.reviewedHoldoutGate, true);
+  const fewZh = make(160, 49);
+  assert.equal(build(fewZh, all(fewZh)).summary.reviewedHoldoutGate, false);
+  const partial = make(150, 50);
+  assert.equal(build(partial, reviews(all(partial).decisions.slice(1))).summary.reviewedHoldoutGate, false);
+});
+
+test('S1-K07 CLI builds a corpus file and fails closed on invalid input', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-corpus-'));
+  try {
+    const cli = path.join(root, 'scripts/system-one-corpus.js');
+    const input = path.join(dir, 'draft.json'); const review = path.join(dir, 'reviews.json'); const out = path.join(dir, 'corpus.json');
+    const d = draft();
+    fs.writeFileSync(input, JSON.stringify(d));
+    fs.writeFileSync(review, JSON.stringify(reviews([decision(d.cases[0])])));
+    const ok = spawnSync(process.execPath, [cli, 'build', input, review, out], { encoding: 'utf8' });
+    assert.equal(ok.status, 0, ok.stderr);
+    assert.equal(JSON.parse(ok.stdout).reviewed, 1);
+    assert.equal(validateCorpus(JSON.parse(fs.readFileSync(out, 'utf8'))), true);
+    const none = spawnSync(process.execPath, [cli, 'build', input, '-', out], { encoding: 'utf8' });
+    assert.equal(none.status, 0, none.stderr);
+    assert.equal(JSON.parse(none.stdout).reviewed, 0);
+    fs.writeFileSync(review, '{"schemaVersion":1,"decisions":[{"id":"zzz"}]}');
+    assert.equal(spawnSync(process.execPath, [cli, 'build', input, review, out], { encoding: 'utf8' }).status, 1);
+    assert.equal(spawnSync(process.execPath, [cli, 'bogus'], { encoding: 'utf8' }).status, 1);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-K08 the committed holdout draft meets size, language, class and length requirements', () => {
+  const d = JSON.parse(fs.readFileSync(draftPath, 'utf8'));
+  assert.equal(validateDraft(d), true);
+  const { corpus } = build(d, null);
+  assert.equal(validateCorpus(corpus), true);
+  assert.ok(d.cases.length >= 220, 'headroom above 200 for rejected cases');
+  for (const lang of ['en', 'zh-TW']) assert.ok(d.cases.filter(c => c.language === lang).length >= 60, lang);
+  for (const gold of ['tier1', 'tier2', 'tier3', null]) assert.ok(d.cases.filter(c => c.proposedGold === gold).length >= 40, String(gold));
+  const bytes = d.cases.map(c => Buffer.byteLength(c.prompt, 'utf8'));
+  assert.ok(bytes.filter(b => b > 224).length / bytes.length >= 0.15, 'long prompts are represented');
+  assert.ok(bytes.filter(b => b <= 40).length / bytes.length >= 0.1, 'short prompts are represented');
+  assert.ok(d.cases.some(c => c.source === 'derived:local-history') && d.cases.some(c => c.source === 'synthetic:authored'));
+  assert.ok(new Set(d.cases.map(c => c.family)).size >= 40, 'enough distinct families');
+});
+
+test('S1-K09 the committed reviewed holdout is reproducible from owner decisions and meets the gate', () => {
+  const read = name => JSON.parse(fs.readFileSync(path.join(root, 'benchmarks/fixtures', name), 'utf8'));
+  const d = read('system-one-holdout-draft.json');
+  const r = read('system-one-holdout-reviews.json');
+  assert.equal(validateReviews(r), true);
+  assert.deepEqual(r.decisions.map(x => x.id).sort(), d.cases.map(c => c.id).sort(), 'every draft case has exactly one owner decision');
+  const { corpus, summary } = build(d, r);
+  assert.equal(summary.stale, 0);
+  assert.equal(summary.unreviewed, 0);
+  assert.equal(summary.reviewedHoldoutGate, true);
+  assert.deepEqual(read('system-one-holdout.json'), corpus, 'committed corpus equals the rebuild');
+  for (const gold of ['tier1', 'tier2', 'tier3', null]) assert.ok(corpus.cases.filter(c => c.gold === gold).length >= 15, String(gold));
+});

package/ci/mechanism-system-one-label.test.js

@@ -0,0 +1,101 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const label = require('../scripts/system-one-label');
+const root = path.resolve(__dirname, '..');
+const items = n => Array.from({ length: n }, (_, i) => ({ id: `id${i}`, text: `prompt number ${i}` }));
+const reply = (batch, gold = () => 'tier1') => ({ labels: batch.map((_, i) => ({ i, gold: gold(i) })) });
+
+test('S1-L01 the instructions carry the owner rules from the corpus document and every item', () => {
+  const rules = label.rulesFromDoc(fs.readFileSync(path.join(root, 'docs/system-one-corpus.md'), 'utf8'));
+  assert.match(rules, /\| `tier1` \|/);
+  assert.match(rules, /Label the work the prompt would start/);
+  assert.doesNotMatch(rules, /## Cases/);
+  const prompt = label.buildPrompt(rules, items(3));
+  assert.ok(prompt.includes(rules));
+  for (let i = 0; i < 3; i++) assert.ok(prompt.includes(JSON.stringify({ i, text: `prompt number ${i}` })));
+  assert.throws(() => label.rulesFromDoc('# no gold section'));
+});
+
+test('S1-L02 a batch reply must label every index exactly once with a known label', () => {
+  const batch = items(3);
+  assert.deepEqual(label.validateReply(batch, reply(batch, i => ['tier1', 'null', 'tier3'][i])), ['tier1', null, 'tier3']);
+  const bad = [
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier1' }] },
+    { labels: [...reply(batch).labels, { i: 0, gold: 'tier2' }] },
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier1' }, { i: 7, gold: 'tier1' }] },
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier4' }, { i: 2, gold: 'tier1' }] },
+    { labels: 'nope' }, null,
+  ];
+  for (const r of bad) assert.equal(label.validateReply(batch, r), null);
+});
+
+test('S1-L03 invalid replies are retried once, persistent failures are recorded, never partially accepted, and runs resume', async () => {
+  const data = items(250);
+  const calls = [];
+  let flaky = true;
+  const run = async (prompt, batch) => {
+    calls.push(batch.length);
+    if (batch[0].id === 'id100' && flaky) { flaky = false; return { labels: [] }; }
+    if (batch[0].id === 'id200') return { labels: [] };
+    return reply(batch, i => (i % 2 ? 'tier2' : 'null'));
+  };
+  const reasons = [];
+  const first = await label.labelAll(data, { run, rules: 'R', batchSize: 100, concurrency: 2, done: new Set(), onFailure: (batch, reason) => reasons.push([batch[0].id, reason]) });
+  assert.deepEqual(reasons, [['id100', 'invalid-reply'], ['id200', 'invalid-reply'], ['id200', 'invalid-reply']], 'every failed attempt is reported with its reason');
+  const thrown = [];
+  await label.labelAll(items(1), { run: async () => { throw new Error('rate limited'); }, rules: 'R', retryDelayMs: 0, onFailure: (b, r) => thrown.push(r) });
+  assert.deepEqual(thrown, ['rate limited', 'rate limited']);
+  assert.equal(first.labeled.length, 200);
+  assert.deepEqual(first.failed.sort(), data.slice(200).map(d => d.id).sort());
+  assert.equal(calls.filter(n => n === 100).length, 3, 'the flaky batch ran twice');
+  assert.equal(first.labeled.find(r => r.id === 'id101').gold, 'tier2');
+  assert.equal(first.labeled.find(r => r.id === 'id100').gold, null);
+  calls.length = 0;
+  const again = await label.labelAll(data, { run, rules: 'R', batchSize: 100, concurrency: 2, done: new Set(first.labeled.map(r => r.id)) });
+  assert.deepEqual(calls, [50, 50], 'only the unlabeled remainder is sent, retried once');
+  assert.equal(again.labeled.length, 0);
+});
+
+test('S1-L04 holdout agreement reports overall, per-class recall and confusion', () => {
+  const gold = [['a', 'tier1'], ['b', 'tier1'], ['c', 'tier2'], ['d', null]];
+  const pred = { a: 'tier1', b: 'tier2', c: 'tier2', d: null };
+  const r = label.agreement(gold.map(([id, g]) => ({ id, gold: g })), pred);
+  assert.equal(r.cases, 4); assert.equal(r.agreement, 0.75);
+  assert.deepEqual(r.recall, { tier1: 0.5, tier2: 1, tier3: null, unclassified: 1 });
+  assert.equal(r.confusion['tier1->tier2'], 1);
+  assert.equal(r.missing, 0);
+  assert.equal(label.agreement([{ id: 'x', gold: 'tier1' }], {}).missing, 1);
+});
+
+test('S1-L05 CLI labels through the configured command, resumes, checks the holdout and refuses repository output', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-label-'));
+  try {
+    const fake = path.join(dir, 'fake-claude.js');
+    fs.writeFileSync(fake, `let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const m=[...s.matchAll(/\\{"i":(\\d+),"text"/g)];
+      process.stdout.write(JSON.stringify({is_error:false,structured_output:{labels:m.map(x=>({i:Number(x[1]),gold:'tier2'}))}}));});`);
+    const input = path.join(dir, 'prompts.jsonl');
+    fs.writeFileSync(input, items(5).map(r => JSON.stringify({ ...r, family: 'f', source: 'claude', split: 'train' })).join('\n') + '\n');
+    const env = { ...process.env, HARNESS_S1_LABEL_COMMAND: JSON.stringify([process.execPath, fake]) };
+    const cli = args => spawnSync(process.execPath, [path.join(root, 'scripts/system-one-label.js'), ...args], { encoding: 'utf8', cwd: root, env });
+    const out = path.join(dir, 'labels.jsonl');
+    const one = cli(['label', '--in', input, '--out', out]);
+    assert.equal(one.status, 0, one.stderr);
+    assert.deepEqual(JSON.parse(one.stdout), { labeled: 5, failed: 0, total: 5 });
+    const rows = fs.readFileSync(out, 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(r => r.gold === 'tier2' && r.labeler.model === 'sonnet' && /^[0-9a-f]{16}$/.test(r.labeler.rulesSha256)));
+    assert.deepEqual(JSON.parse(cli(['label', '--in', input, '--out', out]).stdout), { labeled: 0, failed: 0, total: 5 });
+    const report = path.join(dir, 'check.json');
+    const check = cli(['check-holdout', '--out', report]);
+    assert.equal(check.status, 0, check.stderr);
+    const r = JSON.parse(fs.readFileSync(report, 'utf8'));
+    assert.equal(r.cases, 215);
+    assert.equal(typeof r.agreement, 'number');
+    assert.equal(cli(['label', '--in', input, '--out', path.join(root, 'tmp-labels.jsonl')]).status, 1);
+    assert.equal(fs.existsSync(path.join(root, 'tmp-labels.jsonl')), false);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-resident.test.js

@@ -16,13 +16,13 @@ const request = createRequest('tier', 'fix a typo in README', TIER_OPTIONS);
 const sha = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
 const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 
-function fixture(modelId = 'stub') {
+function fixture(modelId = 'stub', extra = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-resident-'));
   const checkpoint = path.join(dir, 'model.safetensors');
   fs.writeFileSync(checkpoint, 'synthetic-not-real-weights');
   fs.writeFileSync(path.join(dir, 'model.json'), '{}');
   const manifest = { schemaVersion: 1, python, checkpoint, weightsSha256: sha(checkpoint), configSha256: sha(path.join(dir, 'model.json')),
-    modelId, revision: 'v1', domain: 'harness-routing-v1', transport: 'resident', idleTimeoutMs: 60000 };
+    modelId, revision: 'v1', domain: 'harness-routing-v1', transport: 'resident', idleTimeoutMs: 60000, ...extra };
   const manifestPath = path.join(dir, 'manifest.json');
   fs.writeFileSync(manifestPath, JSON.stringify(manifest));
   const spawns = [];
@@ -294,3 +294,37 @@ test('S1-RS12 the evaluator times the async path: no worker thread per warm scor
     assert.ok(Number(fs.readFileSync(countFile, 'utf8')) < report.records.length * 2, 'warm scores must not start a worker thread each');
   } finally { cleanup(f); }
 });
+
+test('S1-RS13 manifest acceptance thresholds are validated, carried by scored results and applied by the evaluator', async () => {
+  const acceptance = { minConfidence: 0.6, minMargin: 0.3 };
+  const base = fixture();
+  try {
+    for (const ok of [acceptance, { minConfidence: 0.5, minMargin: 0 }, { minConfidence: 0.99, minMargin: 0.9 }]) assert.equal(provider.validateManifest({ ...base.manifest, acceptance: ok }), true);
+    for (const bad of [{ minConfidence: 0.49, minMargin: 0.2 }, { minConfidence: 1, minMargin: 0.2 }, { minConfidence: 0.9, minMargin: 0.91 }, { minConfidence: 0.9 }, { minConfidence: 0.9, minMargin: 0.2, extra: 1 }, { minConfidence: '0.9', minMargin: 0.2 }, null, []]) {
+      assert.throws(() => provider.validateManifest({ ...base.manifest, acceptance: bad }), JSON.stringify(bad));
+    }
+  } finally { cleanup(base); }
+  const root = path.resolve(__dirname, '..');
+  const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
+  const evaluateWith = f => {
+    const output = path.join(f.dir, 'report.json');
+    const child = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, f.manifestPath, output], { cwd: root, encoding: 'utf8' });
+    assert.equal(child.status, 0, child.stderr);
+    return JSON.parse(fs.readFileSync(output, 'utf8')).records.flatMap(r => r.runs).filter(r => r.decision.reason !== 'provider-exit');
+  };
+  const soft = fixture('stub-soft', { acceptance });
+  try {
+    assert.equal(resident.ensureReady(soft.manifest, soft.manifestPath, 20000, soft.deps), true);
+    const result = provider.score(request, soft.manifestPath);
+    assert.equal(result.status, 'scored');
+    assert.deepEqual(result.acceptance, acceptance);
+    assert.deepEqual((await provider.scoreAsync(request, soft.manifestPath)).acceptance, acceptance);
+    assert.ok(evaluateWith(soft).every(r => r.decision.status === 'accepted' && r.decision.selectedId === 'tier1'));
+  } finally { cleanup(soft); }
+  const strict = fixture('stub-soft');
+  try {
+    assert.equal(resident.ensureReady(strict.manifest, strict.manifestPath, 20000, strict.deps), true);
+    assert.equal(provider.score(request, strict.manifestPath).acceptance, undefined);
+    assert.ok(evaluateWith(strict).every(r => r.decision.status === 'abstain' && r.decision.reason === 'low-confidence'));
+  } finally { cleanup(strict); }
+});

package/ci/mechanism-system-one-router.test.js

@@ -99,6 +99,19 @@ test('S1-R05 structural floor: a scorer can raise but never lower a deterministi
     assert.equal(plan.verification.independent, true);
   } finally { provider.score = original; console.log = log; if (mode === undefined) delete process.env.HARNESS_SYSTEM_ONE_MODE; else process.env.HARNESS_SYSTEM_ONE_MODE = mode; }
 });
+test('S1-R07 a scored result with calibrated acceptance thresholds decides with them', () => {
+  const env = { HARNESS_SYSTEM_ONE_MODE: 'prefer' };
+  const input = { prompt: 'rename tmp to buffer', tier: 'Unclassified' };
+  const soft = (acceptance) => req => ({ status: 'scored', ...(acceptance ? { acceptance } : {}), response: { schemaVersion: 1, requestHash: req.requestHash, catalogHash: req.catalogHash,
+    model: { id: 'fixture', revision: 'v1', domain: 'harness-routing-v1' }, scores: req.options.map(o => ({ id: o.id, probability: o.id === 'tier2' ? 0.7 : 0.1 })) } });
+  const without = selectTier(input, env, soft(null));
+  assert.equal(without.diagnostic.reason, 'low-confidence');
+  assert.equal(without.tier, input.tier);
+  const withThresholds = selectTier(input, env, soft({ minConfidence: 0.6, minMargin: 0.3 }));
+  assert.equal(withThresholds.diagnostic.status, 'accepted');
+  assert.equal(withThresholds.tier, 'Tier 2 (Standard Task)');
+  assert.equal(selectTier(input, env, soft({ minConfidence: 0.6, minMargin: 0.7 })).diagnostic.reason, 'low-margin');
+});
 test('S1-R06 prescoreTier: off loads nothing; a matching request reuses the async result; mismatch and failure keep sync semantics', async () => {
   const { prescoreTier } = require('../harness-everything/scripts/system-one/router');
   const provider = require('../harness-everything/scripts/system-one/provider');

package/ci/mechanism-system-one-train.test.js

@@ -0,0 +1,82 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const { calibrate, decideRow } = require('../scripts/system-one-calibrate');
+const root = path.resolve(__dirname, '..');
+const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+
+test('S1-T01 trainer data handling is exercised without torch', () => {
+  const r = spawnSync(python, [path.join(__dirname, 'system-one-train-test.py')], { encoding: 'utf8', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } });
+  assert.equal(r.status, 0, r.stdout + r.stderr);
+  assert.match(r.stdout, /trainer data tests passed/);
+});
+
+const row = (gold, probs) => ({ gold, probs });
+test('S1-T02 a validation row is accepted only above both thresholds and never as unclassified', () => {
+  assert.deepEqual(decideRow([0.95, 0.03, 0.01, 0.01], { minConfidence: 0.9, minMargin: 0.2 }), 'tier1');
+  assert.equal(decideRow([0.85, 0.1, 0.03, 0.02], { minConfidence: 0.9, minMargin: 0.2 }), null);
+  assert.equal(decideRow([0.55, 0.45, 0, 0], { minConfidence: 0.5, minMargin: 0.2 }), null);
+  assert.equal(decideRow([0.01, 0.01, 0.01, 0.97], { minConfidence: 0.5, minMargin: 0 }), null);
+});
+
+test('S1-T03 calibration maximizes coverage subject to accepted precision on validation only', () => {
+  const rows = [
+    ...Array.from({ length: 90 }, () => row('tier1', [0.97, 0.01, 0.01, 0.01])),
+    ...Array.from({ length: 5 }, () => row('tier2', [0.8, 0.15, 0.03, 0.02])),
+    ...Array.from({ length: 5 }, () => row('tier2', [0.1, 0.85, 0.03, 0.02])),
+  ];
+  const c = calibrate(rows, { minPrecision: 0.98 });
+  assert.ok(c.precision >= 0.98, JSON.stringify(c));
+  assert.equal(c.coverage, 0.95, JSON.stringify(c));
+  // Either threshold may separate the rows; the chosen pair must reject the wrong ones and keep the right ones.
+  assert.equal(decideRow([0.8, 0.15, 0.03, 0.02], c), null, JSON.stringify(c));
+  assert.equal(decideRow([0.1, 0.85, 0.03, 0.02], c), 'tier2', JSON.stringify(c));
+  const none = calibrate([row('tier1', [0.3, 0.7, 0, 0])], { minPrecision: 0.98 });
+  assert.equal(none.feasible, false);
+  assert.equal(none.coverage, 0);
+  assert.deepEqual(calibrate(rows, { minPrecision: 0.98 }), c, 'deterministic');
+});
+
+test('S1-T04 calibration CLI reads validation scores and writes thresholds', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-cal-'));
+  try {
+    const input = path.join(dir, 'val-scores.jsonl');
+    fs.writeFileSync(input, Array.from({ length: 50 }, (_, i) => JSON.stringify({ id: `v${i}`, gold: 'tier2', probs: [0.02, 0.95, 0.02, 0.01] })).join('\n') + '\n');
+    const out = path.join(dir, 'calibration.json');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-calibrate.js'), input, out], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stderr);
+    const c = JSON.parse(fs.readFileSync(out, 'utf8'));
+    assert.equal(c.feasible, true); assert.equal(c.coverage, 1); assert.equal(c.rows, 50);
+    assert.equal(spawnSync(process.execPath, [path.join(root, 'scripts/system-one-calibrate.js')], { encoding: 'utf8' }).status, 1);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-T05 a real tiny training run produces a checkpoint the adapter loads (skipped without torch)', t => {
+  const probe = spawnSync(python, ['-c', 'import torch, cua_s1'], { encoding: 'utf8' });
+  if (probe.status !== 0) { t.skip('torch/cua_s1 not installed; mechanism tests above still run'); return; }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-train-'));
+  try {
+    const data = path.join(dir, 'data'); fs.mkdirSync(data);
+    const texts = ['commit all changes', 'push and open a pr', 'add a flag with tests', 'migrate every service', 'go', 'yes'];
+    const golds = ['tier1', 'tier1', 'tier3', 'tier3', null, null];
+    const prompts = []; const labels = [];
+    for (let k = 0; k < 8; k++) texts.forEach((t, i) => { const id = `p${k}-${i}`; prompts.push({ id, family: `f${k}`, source: 'claude', split: k < 6 ? 'train' : 'validation', text: `${t} ${k}` }); labels.push({ id, gold: golds[i] }); });
+    fs.writeFileSync(path.join(data, 'prompts.jsonl'), prompts.map(p => JSON.stringify(p)).join('\n') + '\n');
+    fs.writeFileSync(path.join(data, 'labels.jsonl'), labels.map(p => JSON.stringify(p)).join('\n') + '\n');
+    const out = path.join(dir, 'model');
+    const r = spawnSync(python, [path.join(root, 'scripts/system-one-train.py'), '--data-dir', data, '--out', out, '--epochs', '2', '--context-tokens', '64'], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stdout + r.stderr);
+    const report = JSON.parse(fs.readFileSync(path.join(out, 'train-report.json'), 'utf8'));
+    assert.equal(report.counts.train, 36); assert.equal(report.counts.validation, 12);
+    const scores = fs.readFileSync(path.join(out, 'val-scores.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.equal(scores.length, 12);
+    assert.ok(scores.every(s => s.probs.length === 4 && Math.abs(s.probs.reduce((a, b) => a + b, 0) - 1) < 1e-5));
+    const load = spawnSync(python, ['-c', `from cua_s1.model import load_checkpoint; m,c,cfg=load_checkpoint(r'${path.join(out, 'harness-routing-v1.safetensors')}','cpu'); print(cfg['context_tokens'])`], { encoding: 'utf8' });
+    assert.equal(load.status, 0, load.stderr);
+    assert.equal(load.stdout.trim(), '64');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/system-one-train-test.py

@@ -0,0 +1,55 @@
+"""Stdlib-only tests for the System One trainer's data handling (no torch needed)."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import tempfile
+
+sys.dont_write_bytecode = True
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('trainer', ROOT / 'scripts' / 'system-one-train.py')
+trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(trainer)
+assert 'torch' not in sys.modules, 'importing the trainer must not import torch'
+
+catalog = trainer.catalog_from_holdout(ROOT / 'benchmarks' / 'fixtures' / 'system-one-holdout.json')
+assert [o['id'] for o in catalog] == ['tier1', 'tier2', 'tier3', 'unclassified'], catalog
+assert all(o['text'] for o in catalog)
+assert trainer.label_index('tier2', catalog) == 1
+assert trainer.label_index(None, catalog) == 3
+try:
+    trainer.label_index('tier9', catalog)
+    raise SystemExit('unknown gold must be rejected')
+except ValueError:
+    pass
+
+prompts = [
+    {'id': 'a', 'family': 'f1', 'source': 'claude', 'split': 'train', 'text': 'commit'},
+    {'id': 'b', 'family': 'f2', 'source': 'vscode', 'split': 'validation', 'text': 'fix the parser'},
+    {'id': 'c', 'family': 'f3', 'source': 'codex', 'split': 'train', 'text': 'unlabeled prompt'},
+    {'id': 'd', 'family': 'f4', 'source': 'codex', 'split': 'train', 'text': 'owner override'},
+]
+labels = [{'id': 'a', 'gold': 'tier1'}, {'id': 'b', 'gold': None}, {'id': 'd', 'gold': 'tier3'}]
+overrides = [{'id': 'd', 'gold': 'tier2'}]
+examples = trainer.join(prompts, labels, overrides)
+assert [(e['id'], e['split'], e['gold']) for e in examples] == [('a', 'train', 'tier1'), ('b', 'validation', None), ('d', 'train', 'tier2')], examples
+try:
+    trainer.join(prompts, labels + [{'id': 'a', 'gold': 'tier2'}], [])
+    raise SystemExit('duplicate labels must be rejected')
+except ValueError:
+    pass
+
+lengths = [10, 1024, 300, 5, 700, 1024, 50] * 20
+batches = trainer.make_batches(lengths, token_budget=4096)
+flat = sorted(i for b in batches for i in b)
+assert flat == list(range(len(lengths))), 'every example exactly once'
+assert all(len(b) * max(lengths[i] for i in b) <= 4096 or len(b) == 1 for b in batches), 'budget respected'
+assert batches == trainer.make_batches(lengths, token_budget=4096), 'deterministic'
+
+with tempfile.TemporaryDirectory() as tmp:
+    p = Path(tmp) / 'rows.jsonl'
+    p.write_text('\n'.join(json.dumps(r) for r in prompts) + '\n', encoding='utf-8')
+    assert trainer.load_jsonl(p) == prompts
+    assert len(trainer.sha256_file(p)) == 64
+
+print('system-one trainer data tests passed')