harness-everything 0.23.0 → 0.24.0

package/ci/fixtures/system-one-resident-stub.py

@@ -18,6 +18,9 @@ def factory(weights, manifest):
     def score(request):
         if manifest['modelId'] == 'stub-slow':
             time.sleep(3)
+        if manifest['modelId'] == 'stub-soft':
+            rest = 0.3 / (len(request['options']) - 1)
+            return [0.7] + [rest] * (len(request['options']) - 1)
         return [1.0] + [0.0] * (len(request['options']) - 1)
     return score, {'context_tokens': 224, 'option_tokens': 96}
 

package/ci/mechanism-system-one-collect.test.js

@@ -0,0 +1,129 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const collect = require('../scripts/system-one-collect');
+const root = path.resolve(__dirname, '..');
+const jsonl = rows => rows.map(r => JSON.stringify(r)).join('\n');
+
+// A fake home with one history file per source; each holds user text plus records that must be ignored.
+function fakeHome() {
+  const home = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-collect-'));
+  const appdata = path.join(home, 'AppData', 'Roaming');
+  const write = (rel, body) => { const p = path.join(home, rel); fs.mkdirSync(path.dirname(p), { recursive: true }); fs.writeFileSync(p, body); };
+  write('.claude/projects/p1/s-claude.jsonl', jsonl([
+    { type: 'user', message: { content: 'fix the typo in README' } },
+    { type: 'user', message: { content: [{ type: 'text', text: 'commit all changes' }] } },
+    { type: 'user', isMeta: true, message: { content: 'meta text must be skipped' } },
+    { type: 'user', isSidechain: true, message: { content: 'sidechain text must be skipped' } },
+    { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'x', content: 'tool output must be skipped' }] } },
+    { type: 'user', message: { content: '<system-reminder>injected</system-reminder>' } },
+    { type: 'user', message: { content: '<command-name>/compact</command-name>' } },
+    { type: 'assistant', message: { content: 'assistant text must be skipped' } },
+  ]));
+  write('.codex/sessions/2026/09/01/rollout-a.jsonl', jsonl([
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: '<environment_context>cwd</environment_context>' }] } },
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'duplicate of the event message' }] } },
+    { type: 'event_msg', payload: { type: 'user_message', message: 'add a --verbose flag with tests' } },
+  ]));
+  write('.codex/sessions/2026/09/02/rollout-b.jsonl', jsonl([
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: '<user_instructions>agents</user_instructions>' }] } },
+    { type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'older session without event messages' }] } },
+  ]));
+  write('.copilot/session-state/c1/events.jsonl', jsonl([
+    { type: 'system.message', data: { content: 'system prompt must be skipped' } },
+    { type: 'user.message', data: { content: 'run npm test and report failures' } },
+  ]));
+  write('AppData/Roaming/Code/User/workspaceStorage/w1/chatSessions/old.json', JSON.stringify({
+    requests: [{ message: { text: '把 parseCfg 改名成 parseConfig' } }, { message: { text: '' } }] }));
+  write('AppData/Roaming/Code/User/workspaceStorage/w1/chatSessions/new.jsonl', jsonl([
+    { kind: 0, v: { sessionId: 's', requests: [{ message: { text: '整個 repo 遷移到 ESM' } }] } },
+    { kind: 2, k: ['requests'], v: [{ message: { text: '繼續' } }] },
+    { kind: 2, k: ['requests', 0, 'response'], v: [{ value: 'response text must be skipped' }] },
+    { kind: 1, k: ['requests', 0, 'result'], v: { metadata: { renderedUserMessage: 'rendered prompt must be skipped' } } },
+  ]));
+  write('AppData/Roaming/Code/User/globalStorage/emptyWindowChatSessions/e.jsonl', jsonl([
+    { kind: 0, v: { requests: [{ message: { text: 'explain the backoff in retry.go' } }] } },
+  ]));
+  return { home, appdata };
+}
+const texts = rows => rows.map(r => r.text).sort();
+
+test('S1-C101 each source yields only owner-typed text', () => {
+  const { home, appdata } = fakeHome();
+  try {
+    const rows = collect.readSources({ home, appdata });
+    assert.deepEqual(texts(rows), texts([
+      { text: 'fix the typo in README' }, { text: 'commit all changes' }, { text: 'add a --verbose flag with tests' },
+      { text: 'older session without event messages' }, { text: 'run npm test and report failures' },
+      { text: '把 parseCfg 改名成 parseConfig' }, { text: '整個 repo 遷移到 ESM' }, { text: '繼續' }, { text: 'explain the backoff in retry.go' },
+    ]));
+    const bySource = Object.fromEntries(['claude', 'codex', 'copilot-cli', 'vscode'].map(s => [s, rows.filter(r => r.source === s).length]));
+    assert.deepEqual(bySource, { claude: 2, codex: 2, 'copilot-cli': 1, vscode: 4 });
+    assert.ok(rows.every(r => typeof r.family === 'string' && r.family.length > 0));
+    const claude = rows.filter(r => r.source === 'claude');
+    assert.equal(new Set(claude.map(r => r.family)).size, 1, 'one session file is one family');
+  } finally { fs.rmSync(home, { recursive: true, force: true }); }
+});
+
+test('S1-C102 filters, redaction and deduplication', () => {
+  assert.equal(collect.keep('go'), true);
+  assert.equal(collect.keep('x'.repeat(1024)), true);
+  assert.equal(collect.keep('x'.repeat(1025)), false);
+  assert.equal(collect.keep('中'.repeat(342)), false, '1026 UTF-8 bytes');
+  assert.equal(collect.keep('/compact'), false);
+  assert.equal(collect.keep('This session is being continued from a previous conversation that ran out of context.'), false);
+  assert.equal(collect.keep('   '), false);
+  assert.equal(collect.redact('mail dev.lead@example.com now'), 'mail <email> now');
+  assert.equal(collect.redact('open C:\\Users\\alice\\repo\\a.md'), 'open <home>\\repo\\a.md');
+  assert.equal(collect.redact('read /home/alice/p and /Users/bob/q'), 'read <home>/p and <home>/q');
+  const rows = collect.dedupe([{ text: 'fix  it', family: 'a' }, { text: 'fix it', family: 'b' }, { text: 'other', family: 'c' }]);
+  assert.deepEqual(rows.map(r => r.family), ['a', 'c']);
+});
+
+test('S1-C103 prompts that near-duplicate a holdout prompt are dropped', () => {
+  const holdout = ['把 parseCfg 改名成 parseConfig, 只有這個檔案有用到', 'Rebase this branch onto the latest main.'];
+  const guard = collect.leakGuard(holdout);
+  assert.equal(guard('把 parseCfg 改名成 parseConfig'), true, 'contained, longer than 12 chars');
+  assert.equal(guard('rebase this branch onto the latest main'), true, 'normalized near-duplicate');
+  assert.equal(guard('add pagination to the users endpoint'), false);
+  assert.equal(guard('main'), false, 'short containment does not count');
+  assert.ok(collect.jaccard('abcdef', 'abcdef') === 1 && collect.jaccard('abc', 'xyz') === 0);
+});
+
+test('S1-C104 splits are stable per family and near 15% validation', () => {
+  assert.equal(collect.splitFor('family-a'), collect.splitFor('family-a'));
+  const families = Array.from({ length: 2000 }, (_, i) => `session-${i}`);
+  const share = families.filter(f => collect.splitFor(f) === 'validation').length / families.length;
+  assert.ok(share > 0.12 && share < 0.18, String(share));
+  assert.ok(families.every(f => ['train', 'validation'].includes(collect.splitFor(f))));
+});
+
+test('S1-C105 CLI writes a deterministic local dataset outside the repository', () => {
+  const { home, appdata } = fakeHome();
+  const out = path.join(home, 'out');
+  const holdout = path.join(home, 'holdout.json');
+  fs.writeFileSync(holdout, JSON.stringify({ schemaVersion: 1, cases: [{ request: { context: 'Commit all changes.' } }] }));
+  try {
+    const run = () => spawnSync(process.execPath, [path.join(root, 'scripts/system-one-collect.js'), '--home', home, '--appdata', appdata, '--out', out, '--holdout', holdout],
+      { encoding: 'utf8', cwd: root });
+    const one = run();
+    assert.equal(one.status, 0, one.stderr);
+    const first = fs.readFileSync(path.join(out, 'prompts.jsonl'), 'utf8');
+    const stats = JSON.parse(one.stdout);
+    assert.equal(stats.kept, 8);
+    assert.equal(stats.dropped.leak, 1);
+    assert.deepEqual(Object.keys(stats.bySource).sort(), ['claude', 'codex', 'copilot-cli', 'vscode']);
+    assert.match(stats.sha256, /^[0-9a-f]{64}$/);
+    const rows = first.trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(r => Object.keys(r).sort().join() === 'family,id,source,split,text' && /^[0-9a-f]{16}$/.test(r.id)));
+    assert.equal(run().status, 0);
+    assert.equal(fs.readFileSync(path.join(out, 'prompts.jsonl'), 'utf8'), first, 'deterministic');
+    const inRepo = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-collect.js'), '--home', home, '--appdata', appdata, '--out', path.join(root, 'tmp-dataset'), '--holdout', holdout], { encoding: 'utf8', cwd: root });
+    assert.equal(inRepo.status, 1, 'refuses to write the dataset inside the repository');
+    assert.equal(fs.existsSync(path.join(root, 'tmp-dataset')), false);
+  } finally { fs.rmSync(home, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-label.test.js

@@ -0,0 +1,101 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const label = require('../scripts/system-one-label');
+const root = path.resolve(__dirname, '..');
+const items = n => Array.from({ length: n }, (_, i) => ({ id: `id${i}`, text: `prompt number ${i}` }));
+const reply = (batch, gold = () => 'tier1') => ({ labels: batch.map((_, i) => ({ i, gold: gold(i) })) });
+
+test('S1-L01 the instructions carry the owner rules from the corpus document and every item', () => {
+  const rules = label.rulesFromDoc(fs.readFileSync(path.join(root, 'docs/system-one-corpus.md'), 'utf8'));
+  assert.match(rules, /\| `tier1` \|/);
+  assert.match(rules, /Label the work the prompt would start/);
+  assert.doesNotMatch(rules, /## Cases/);
+  const prompt = label.buildPrompt(rules, items(3));
+  assert.ok(prompt.includes(rules));
+  for (let i = 0; i < 3; i++) assert.ok(prompt.includes(JSON.stringify({ i, text: `prompt number ${i}` })));
+  assert.throws(() => label.rulesFromDoc('# no gold section'));
+});
+
+test('S1-L02 a batch reply must label every index exactly once with a known label', () => {
+  const batch = items(3);
+  assert.deepEqual(label.validateReply(batch, reply(batch, i => ['tier1', 'null', 'tier3'][i])), ['tier1', null, 'tier3']);
+  const bad = [
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier1' }] },
+    { labels: [...reply(batch).labels, { i: 0, gold: 'tier2' }] },
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier1' }, { i: 7, gold: 'tier1' }] },
+    { labels: [{ i: 0, gold: 'tier1' }, { i: 1, gold: 'tier4' }, { i: 2, gold: 'tier1' }] },
+    { labels: 'nope' }, null,
+  ];
+  for (const r of bad) assert.equal(label.validateReply(batch, r), null);
+});
+
+test('S1-L03 invalid replies are retried once, persistent failures are recorded, never partially accepted, and runs resume', async () => {
+  const data = items(250);
+  const calls = [];
+  let flaky = true;
+  const run = async (prompt, batch) => {
+    calls.push(batch.length);
+    if (batch[0].id === 'id100' && flaky) { flaky = false; return { labels: [] }; }
+    if (batch[0].id === 'id200') return { labels: [] };
+    return reply(batch, i => (i % 2 ? 'tier2' : 'null'));
+  };
+  const reasons = [];
+  const first = await label.labelAll(data, { run, rules: 'R', batchSize: 100, concurrency: 2, done: new Set(), onFailure: (batch, reason) => reasons.push([batch[0].id, reason]) });
+  assert.deepEqual(reasons, [['id100', 'invalid-reply'], ['id200', 'invalid-reply'], ['id200', 'invalid-reply']], 'every failed attempt is reported with its reason');
+  const thrown = [];
+  await label.labelAll(items(1), { run: async () => { throw new Error('rate limited'); }, rules: 'R', retryDelayMs: 0, onFailure: (b, r) => thrown.push(r) });
+  assert.deepEqual(thrown, ['rate limited', 'rate limited']);
+  assert.equal(first.labeled.length, 200);
+  assert.deepEqual(first.failed.sort(), data.slice(200).map(d => d.id).sort());
+  assert.equal(calls.filter(n => n === 100).length, 3, 'the flaky batch ran twice');
+  assert.equal(first.labeled.find(r => r.id === 'id101').gold, 'tier2');
+  assert.equal(first.labeled.find(r => r.id === 'id100').gold, null);
+  calls.length = 0;
+  const again = await label.labelAll(data, { run, rules: 'R', batchSize: 100, concurrency: 2, done: new Set(first.labeled.map(r => r.id)) });
+  assert.deepEqual(calls, [50, 50], 'only the unlabeled remainder is sent, retried once');
+  assert.equal(again.labeled.length, 0);
+});
+
+test('S1-L04 holdout agreement reports overall, per-class recall and confusion', () => {
+  const gold = [['a', 'tier1'], ['b', 'tier1'], ['c', 'tier2'], ['d', null]];
+  const pred = { a: 'tier1', b: 'tier2', c: 'tier2', d: null };
+  const r = label.agreement(gold.map(([id, g]) => ({ id, gold: g })), pred);
+  assert.equal(r.cases, 4); assert.equal(r.agreement, 0.75);
+  assert.deepEqual(r.recall, { tier1: 0.5, tier2: 1, tier3: null, unclassified: 1 });
+  assert.equal(r.confusion['tier1->tier2'], 1);
+  assert.equal(r.missing, 0);
+  assert.equal(label.agreement([{ id: 'x', gold: 'tier1' }], {}).missing, 1);
+});
+
+test('S1-L05 CLI labels through the configured command, resumes, checks the holdout and refuses repository output', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-label-'));
+  try {
+    const fake = path.join(dir, 'fake-claude.js');
+    fs.writeFileSync(fake, `let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const m=[...s.matchAll(/\\{"i":(\\d+),"text"/g)];
+      process.stdout.write(JSON.stringify({is_error:false,structured_output:{labels:m.map(x=>({i:Number(x[1]),gold:'tier2'}))}}));});`);
+    const input = path.join(dir, 'prompts.jsonl');
+    fs.writeFileSync(input, items(5).map(r => JSON.stringify({ ...r, family: 'f', source: 'claude', split: 'train' })).join('\n') + '\n');
+    const env = { ...process.env, HARNESS_S1_LABEL_COMMAND: JSON.stringify([process.execPath, fake]) };
+    const cli = args => spawnSync(process.execPath, [path.join(root, 'scripts/system-one-label.js'), ...args], { encoding: 'utf8', cwd: root, env });
+    const out = path.join(dir, 'labels.jsonl');
+    const one = cli(['label', '--in', input, '--out', out]);
+    assert.equal(one.status, 0, one.stderr);
+    assert.deepEqual(JSON.parse(one.stdout), { labeled: 5, failed: 0, total: 5 });
+    const rows = fs.readFileSync(out, 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(r => r.gold === 'tier2' && r.labeler.model === 'sonnet' && /^[0-9a-f]{16}$/.test(r.labeler.rulesSha256)));
+    assert.deepEqual(JSON.parse(cli(['label', '--in', input, '--out', out]).stdout), { labeled: 0, failed: 0, total: 5 });
+    const report = path.join(dir, 'check.json');
+    const check = cli(['check-holdout', '--out', report]);
+    assert.equal(check.status, 0, check.stderr);
+    const r = JSON.parse(fs.readFileSync(report, 'utf8'));
+    assert.equal(r.cases, 215);
+    assert.equal(typeof r.agreement, 'number');
+    assert.equal(cli(['label', '--in', input, '--out', path.join(root, 'tmp-labels.jsonl')]).status, 1);
+    assert.equal(fs.existsSync(path.join(root, 'tmp-labels.jsonl')), false);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-resident.test.js

@@ -16,13 +16,13 @@ const request = createRequest('tier', 'fix a typo in README', TIER_OPTIONS);
 const sha = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
 const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 
-function fixture(modelId = 'stub') {
+function fixture(modelId = 'stub', extra = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-resident-'));
   const checkpoint = path.join(dir, 'model.safetensors');
   fs.writeFileSync(checkpoint, 'synthetic-not-real-weights');
   fs.writeFileSync(path.join(dir, 'model.json'), '{}');
   const manifest = { schemaVersion: 1, python, checkpoint, weightsSha256: sha(checkpoint), configSha256: sha(path.join(dir, 'model.json')),
-    modelId, revision: 'v1', domain: 'harness-routing-v1', transport: 'resident', idleTimeoutMs: 60000 };
+    modelId, revision: 'v1', domain: 'harness-routing-v1', transport: 'resident', idleTimeoutMs: 60000, ...extra };
   const manifestPath = path.join(dir, 'manifest.json');
   fs.writeFileSync(manifestPath, JSON.stringify(manifest));
   const spawns = [];
@@ -294,3 +294,37 @@ test('S1-RS12 the evaluator times the async path: no worker thread per warm scor
     assert.ok(Number(fs.readFileSync(countFile, 'utf8')) < report.records.length * 2, 'warm scores must not start a worker thread each');
   } finally { cleanup(f); }
 });
+
+test('S1-RS13 manifest acceptance thresholds are validated, carried by scored results and applied by the evaluator', async () => {
+  const acceptance = { minConfidence: 0.6, minMargin: 0.3 };
+  const base = fixture();
+  try {
+    for (const ok of [acceptance, { minConfidence: 0.5, minMargin: 0 }, { minConfidence: 0.99, minMargin: 0.9 }]) assert.equal(provider.validateManifest({ ...base.manifest, acceptance: ok }), true);
+    for (const bad of [{ minConfidence: 0.49, minMargin: 0.2 }, { minConfidence: 1, minMargin: 0.2 }, { minConfidence: 0.9, minMargin: 0.91 }, { minConfidence: 0.9 }, { minConfidence: 0.9, minMargin: 0.2, extra: 1 }, { minConfidence: '0.9', minMargin: 0.2 }, null, []]) {
+      assert.throws(() => provider.validateManifest({ ...base.manifest, acceptance: bad }), JSON.stringify(bad));
+    }
+  } finally { cleanup(base); }
+  const root = path.resolve(__dirname, '..');
+  const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
+  const evaluateWith = f => {
+    const output = path.join(f.dir, 'report.json');
+    const child = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, f.manifestPath, output], { cwd: root, encoding: 'utf8' });
+    assert.equal(child.status, 0, child.stderr);
+    return JSON.parse(fs.readFileSync(output, 'utf8')).records.flatMap(r => r.runs).filter(r => r.decision.reason !== 'provider-exit');
+  };
+  const soft = fixture('stub-soft', { acceptance });
+  try {
+    assert.equal(resident.ensureReady(soft.manifest, soft.manifestPath, 20000, soft.deps), true);
+    const result = provider.score(request, soft.manifestPath);
+    assert.equal(result.status, 'scored');
+    assert.deepEqual(result.acceptance, acceptance);
+    assert.deepEqual((await provider.scoreAsync(request, soft.manifestPath)).acceptance, acceptance);
+    assert.ok(evaluateWith(soft).every(r => r.decision.status === 'accepted' && r.decision.selectedId === 'tier1'));
+  } finally { cleanup(soft); }
+  const strict = fixture('stub-soft');
+  try {
+    assert.equal(resident.ensureReady(strict.manifest, strict.manifestPath, 20000, strict.deps), true);
+    assert.equal(provider.score(request, strict.manifestPath).acceptance, undefined);
+    assert.ok(evaluateWith(strict).every(r => r.decision.status === 'abstain' && r.decision.reason === 'low-confidence'));
+  } finally { cleanup(strict); }
+});

package/ci/mechanism-system-one-router.test.js

@@ -99,6 +99,19 @@ test('S1-R05 structural floor: a scorer can raise but never lower a deterministi
     assert.equal(plan.verification.independent, true);
   } finally { provider.score = original; console.log = log; if (mode === undefined) delete process.env.HARNESS_SYSTEM_ONE_MODE; else process.env.HARNESS_SYSTEM_ONE_MODE = mode; }
 });
+test('S1-R07 a scored result with calibrated acceptance thresholds decides with them', () => {
+  const env = { HARNESS_SYSTEM_ONE_MODE: 'prefer' };
+  const input = { prompt: 'rename tmp to buffer', tier: 'Unclassified' };
+  const soft = (acceptance) => req => ({ status: 'scored', ...(acceptance ? { acceptance } : {}), response: { schemaVersion: 1, requestHash: req.requestHash, catalogHash: req.catalogHash,
+    model: { id: 'fixture', revision: 'v1', domain: 'harness-routing-v1' }, scores: req.options.map(o => ({ id: o.id, probability: o.id === 'tier2' ? 0.7 : 0.1 })) } });
+  const without = selectTier(input, env, soft(null));
+  assert.equal(without.diagnostic.reason, 'low-confidence');
+  assert.equal(without.tier, input.tier);
+  const withThresholds = selectTier(input, env, soft({ minConfidence: 0.6, minMargin: 0.3 }));
+  assert.equal(withThresholds.diagnostic.status, 'accepted');
+  assert.equal(withThresholds.tier, 'Tier 2 (Standard Task)');
+  assert.equal(selectTier(input, env, soft({ minConfidence: 0.6, minMargin: 0.7 })).diagnostic.reason, 'low-margin');
+});
 test('S1-R06 prescoreTier: off loads nothing; a matching request reuses the async result; mismatch and failure keep sync semantics', async () => {
   const { prescoreTier } = require('../harness-everything/scripts/system-one/router');
   const provider = require('../harness-everything/scripts/system-one/provider');

package/ci/mechanism-system-one-train.test.js

@@ -0,0 +1,82 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const { calibrate, decideRow } = require('../scripts/system-one-calibrate');
+const root = path.resolve(__dirname, '..');
+const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+
+test('S1-T01 trainer data handling is exercised without torch', () => {
+  const r = spawnSync(python, [path.join(__dirname, 'system-one-train-test.py')], { encoding: 'utf8', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1' } });
+  assert.equal(r.status, 0, r.stdout + r.stderr);
+  assert.match(r.stdout, /trainer data tests passed/);
+});
+
+const row = (gold, probs) => ({ gold, probs });
+test('S1-T02 a validation row is accepted only above both thresholds and never as unclassified', () => {
+  assert.deepEqual(decideRow([0.95, 0.03, 0.01, 0.01], { minConfidence: 0.9, minMargin: 0.2 }), 'tier1');
+  assert.equal(decideRow([0.85, 0.1, 0.03, 0.02], { minConfidence: 0.9, minMargin: 0.2 }), null);
+  assert.equal(decideRow([0.55, 0.45, 0, 0], { minConfidence: 0.5, minMargin: 0.2 }), null);
+  assert.equal(decideRow([0.01, 0.01, 0.01, 0.97], { minConfidence: 0.5, minMargin: 0 }), null);
+});
+
+test('S1-T03 calibration maximizes coverage subject to accepted precision on validation only', () => {
+  const rows = [
+    ...Array.from({ length: 90 }, () => row('tier1', [0.97, 0.01, 0.01, 0.01])),
+    ...Array.from({ length: 5 }, () => row('tier2', [0.8, 0.15, 0.03, 0.02])),
+    ...Array.from({ length: 5 }, () => row('tier2', [0.1, 0.85, 0.03, 0.02])),
+  ];
+  const c = calibrate(rows, { minPrecision: 0.98 });
+  assert.ok(c.precision >= 0.98, JSON.stringify(c));
+  assert.equal(c.coverage, 0.95, JSON.stringify(c));
+  // Either threshold may separate the rows; the chosen pair must reject the wrong ones and keep the right ones.
+  assert.equal(decideRow([0.8, 0.15, 0.03, 0.02], c), null, JSON.stringify(c));
+  assert.equal(decideRow([0.1, 0.85, 0.03, 0.02], c), 'tier2', JSON.stringify(c));
+  const none = calibrate([row('tier1', [0.3, 0.7, 0, 0])], { minPrecision: 0.98 });
+  assert.equal(none.feasible, false);
+  assert.equal(none.coverage, 0);
+  assert.deepEqual(calibrate(rows, { minPrecision: 0.98 }), c, 'deterministic');
+});
+
+test('S1-T04 calibration CLI reads validation scores and writes thresholds', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-cal-'));
+  try {
+    const input = path.join(dir, 'val-scores.jsonl');
+    fs.writeFileSync(input, Array.from({ length: 50 }, (_, i) => JSON.stringify({ id: `v${i}`, gold: 'tier2', probs: [0.02, 0.95, 0.02, 0.01] })).join('\n') + '\n');
+    const out = path.join(dir, 'calibration.json');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-calibrate.js'), input, out], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stderr);
+    const c = JSON.parse(fs.readFileSync(out, 'utf8'));
+    assert.equal(c.feasible, true); assert.equal(c.coverage, 1); assert.equal(c.rows, 50);
+    assert.equal(spawnSync(process.execPath, [path.join(root, 'scripts/system-one-calibrate.js')], { encoding: 'utf8' }).status, 1);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-T05 a real tiny training run produces a checkpoint the adapter loads (skipped without torch)', t => {
+  const probe = spawnSync(python, ['-c', 'import torch, cua_s1'], { encoding: 'utf8' });
+  if (probe.status !== 0) { t.skip('torch/cua_s1 not installed; mechanism tests above still run'); return; }
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-train-'));
+  try {
+    const data = path.join(dir, 'data'); fs.mkdirSync(data);
+    const texts = ['commit all changes', 'push and open a pr', 'add a flag with tests', 'migrate every service', 'go', 'yes'];
+    const golds = ['tier1', 'tier1', 'tier3', 'tier3', null, null];
+    const prompts = []; const labels = [];
+    for (let k = 0; k < 8; k++) texts.forEach((t, i) => { const id = `p${k}-${i}`; prompts.push({ id, family: `f${k}`, source: 'claude', split: k < 6 ? 'train' : 'validation', text: `${t} ${k}` }); labels.push({ id, gold: golds[i] }); });
+    fs.writeFileSync(path.join(data, 'prompts.jsonl'), prompts.map(p => JSON.stringify(p)).join('\n') + '\n');
+    fs.writeFileSync(path.join(data, 'labels.jsonl'), labels.map(p => JSON.stringify(p)).join('\n') + '\n');
+    const out = path.join(dir, 'model');
+    const r = spawnSync(python, [path.join(root, 'scripts/system-one-train.py'), '--data-dir', data, '--out', out, '--epochs', '2', '--context-tokens', '64'], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stdout + r.stderr);
+    const report = JSON.parse(fs.readFileSync(path.join(out, 'train-report.json'), 'utf8'));
+    assert.equal(report.counts.train, 36); assert.equal(report.counts.validation, 12);
+    const scores = fs.readFileSync(path.join(out, 'val-scores.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.equal(scores.length, 12);
+    assert.ok(scores.every(s => s.probs.length === 4 && Math.abs(s.probs.reduce((a, b) => a + b, 0) - 1) < 1e-5));
+    const load = spawnSync(python, ['-c', `from cua_s1.model import load_checkpoint; m,c,cfg=load_checkpoint(r'${path.join(out, 'harness-routing-v1.safetensors')}','cpu'); print(cfg['context_tokens'])`], { encoding: 'utf8' });
+    assert.equal(load.status, 0, load.stderr);
+    assert.equal(load.stdout.trim(), '64');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/system-one-train-test.py

@@ -0,0 +1,55 @@
+"""Stdlib-only tests for the System One trainer's data handling (no torch needed)."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+import tempfile
+
+sys.dont_write_bytecode = True
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('trainer', ROOT / 'scripts' / 'system-one-train.py')
+trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(trainer)
+assert 'torch' not in sys.modules, 'importing the trainer must not import torch'
+
+catalog = trainer.catalog_from_holdout(ROOT / 'benchmarks' / 'fixtures' / 'system-one-holdout.json')
+assert [o['id'] for o in catalog] == ['tier1', 'tier2', 'tier3', 'unclassified'], catalog
+assert all(o['text'] for o in catalog)
+assert trainer.label_index('tier2', catalog) == 1
+assert trainer.label_index(None, catalog) == 3
+try:
+    trainer.label_index('tier9', catalog)
+    raise SystemExit('unknown gold must be rejected')
+except ValueError:
+    pass
+
+prompts = [
+    {'id': 'a', 'family': 'f1', 'source': 'claude', 'split': 'train', 'text': 'commit'},
+    {'id': 'b', 'family': 'f2', 'source': 'vscode', 'split': 'validation', 'text': 'fix the parser'},
+    {'id': 'c', 'family': 'f3', 'source': 'codex', 'split': 'train', 'text': 'unlabeled prompt'},
+    {'id': 'd', 'family': 'f4', 'source': 'codex', 'split': 'train', 'text': 'owner override'},
+]
+labels = [{'id': 'a', 'gold': 'tier1'}, {'id': 'b', 'gold': None}, {'id': 'd', 'gold': 'tier3'}]
+overrides = [{'id': 'd', 'gold': 'tier2'}]
+examples = trainer.join(prompts, labels, overrides)
+assert [(e['id'], e['split'], e['gold']) for e in examples] == [('a', 'train', 'tier1'), ('b', 'validation', None), ('d', 'train', 'tier2')], examples
+try:
+    trainer.join(prompts, labels + [{'id': 'a', 'gold': 'tier2'}], [])
+    raise SystemExit('duplicate labels must be rejected')
+except ValueError:
+    pass
+
+lengths = [10, 1024, 300, 5, 700, 1024, 50] * 20
+batches = trainer.make_batches(lengths, token_budget=4096)
+flat = sorted(i for b in batches for i in b)
+assert flat == list(range(len(lengths))), 'every example exactly once'
+assert all(len(b) * max(lengths[i] for i in b) <= 4096 or len(b) == 1 for b in batches), 'budget respected'
+assert batches == trainer.make_batches(lengths, token_budget=4096), 'deterministic'
+
+with tempfile.TemporaryDirectory() as tmp:
+    p = Path(tmp) / 'rows.jsonl'
+    p.write_text('\n'.join(json.dumps(r) for r in prompts) + '\n', encoding='utf-8')
+    assert trainer.load_jsonl(p) == prompts
+    assert len(trainer.sha256_file(p)) == 64
+
+print('system-one trainer data tests passed')

package/docs/system-one-routing.md

@@ -41,7 +41,11 @@ never a prompt-supplied executable. No shell or automatic download is used.
 The manifest is version 1 with `python` (executable), `checkpoint` (absolute
 `.safetensors` path), `weightsSha256`, `configSha256` (matching `.json` sidecar),
 `modelId`, `revision`, `domain`, and optional `timeoutMs` (1–10000, default 2000).
-All fields are required except the timeout; unknown fields are rejected. Hashes
+All fields are required except the timeout; unknown fields are rejected. An
+optional `acceptance` object holds exactly `minConfidence` (0.5–0.99) and
+`minMargin` (0–0.9). These are the thresholds calibrated on validation for this
+checkpoint. A scored result carries them, and the router and the evaluator pass
+them to `decide`. Without the field the Phase 0 defaults apply. Hashes
 must be lowercase SHA-256. Revision and domain are artifact-owner declarations,
 not independently verified quality claims. The known `cua-ai/cua-s1-forms` ID
 must use domain `forms-v1`. No pickle format or remote URL is accepted.
@@ -279,6 +283,9 @@ unversioned download during a prompt hook.
 | 3 | Corpus and paired evaluator | Leakage checks, coverage/error metrics, repeatability |
 | 4 | Harness training and rollout | Real checkpoint, reviewed holdout, measured host results |
 
+Phase 4 training data, labeling, calibration and release are defined in
+[system-one-training.md](system-one-training.md).
+
 ## Rollout gates
 
 Phase 4 requires a separately reviewed, family-disjoint holdout of at least 200

package/docs/system-one-training.md

@@ -0,0 +1,121 @@
+# System One training data and weights
+
+This document defines how the `harness-routing-v1` tier checkpoint for
+[System One](system-one-routing.md) is trained, calibrated and released
+(issue [#233](https://github.com/dyphn1/Harness-everything/issues/233), Phase 4).
+Gold labels follow [system-one-corpus.md](system-one-corpus.md). The owner-reviewed
+holdout in that document is used only for evaluation. It is never used for training,
+threshold selection or labeler tuning.
+
+## Pipeline
+
+1. **Collect**: copy the owner's own prompts from local assistant histories into a local
+   dataset (`scripts/system-one-collect.js`).
+2. **Label**: an LLM labels each prompt against the owner's rules. The labeler is checked
+   once against the holdout, and the owner spot-checks a sample
+   (`scripts/system-one-label.js`).
+3. **Train**: train a fresh `cua_s1` `tinyx` scorer on the train split and select the epoch
+   on the validation split.
+4. **Calibrate**: choose the confidence and margin thresholds on validation.
+5. **Evaluate**: produce one paired report on the holdout.
+6. **Release**: publish the weights as a GitHub release asset. The installer pins them by
+   SHA-256.
+
+## Sources
+
+Only text the owner typed is collected. Injected context, tool output and assistant text are
+never collected.
+
+| Source | Location | User text |
+| --- | --- | --- |
+| Claude Code | `~/.claude/projects/*/*.jsonl` | `type: "user"` records that are not meta or sidechain, string content or text parts |
+| Codex | `~/.codex/sessions/**/*.jsonl` | `event_msg` `user_message.message`. Older sessions without it fall back to `response_item` user text |
+| Copilot CLI | `~/.copilot/session-state/*/events.jsonl` | `user.message` `data.content` |
+| VS Code Copilot Chat | `%APPDATA%/Code/User/workspaceStorage/*/chatSessions/*.json(l)` and `globalStorage/emptyWindowChatSessions` | `requests[].message.text` in snapshots (`kind: 0`) and request pushes (`kind: 2`, key `["requests"]`) |
+
+The following are dropped:
+
+- Text that starts with an injected wrapper tag, such as `<environment_context>`,
+  `<system-reminder>`, `<command-…>`, `<task-notification>` or `<ide_…>`.
+- Tool results.
+- Slash commands.
+- Session-continuation summaries ("This session is being continued…").
+- Empty text.
+
+Codex sessions include prompts that another agent dispatched. They are kept and tagged with
+their source, because Harness routes them in Codex too.
+
+## Filters, families and splits
+
+- **Length**: at most 1024 UTF-8 bytes, which matches the checkpoint's `context_tokens`. A
+  longer prompt is still refused at inference and falls back to lexical routing. It is never
+  truncated.
+- **Deduplication**: exact duplicates after whitespace normalization are dropped. The first
+  occurrence is kept.
+- **Holdout leakage**: a prompt is dropped if its character-trigram Jaccard similarity to any
+  holdout prompt is at least 0.5. The same applies if either prompt contains the other after
+  normalization and the shorter one is more than 12 characters long. This matters because
+  the holdout includes rewrites of local prompt shapes.
+- **Family**: the source session. Prompts from one conversation never span splits.
+- **Split**: 15% of families go to `validation`, chosen by a stable SHA-256 of the family ID.
+  The rest go to `train`.
+
+## Privacy
+
+- Collected prompts and labels stay on the local machine under
+  `~/.agents/harness-everything/system-one/training/` and are **never committed**. The
+  repository holds only the scripts, aggregate statistics and the SHA-256 of each dataset
+  file.
+- Email addresses and user home paths are replaced with placeholders before storage. Names
+  and product terms are not reliably detectable and remain.
+- The released weights come from a 4-way byte-level classifier with about 0.7M parameters. It
+  has no text-generation head, but a trained model can still encode fragments of its data.
+  The owner accepted this for their own prompts.
+- Labeling sends the prompts to the Claude API. The owner authorized this.
+
+## Labeling
+
+- **Labeler**: `claude -p` with Sonnet.
+  - Flags: `--setting-sources ""` (no hooks or plugins), `--no-session-persistence` (labeling
+    sessions never become collectable history), `--tools ""` and a JSON schema.
+  - The instructions contain the gold table and rules from
+    [system-one-corpus.md](system-one-corpus.md).
+  - Prompts go in batches of 100. Every returned index must be labeled exactly once with
+    `tier1`, `tier2`, `tier3` or `null`. A malformed batch is retried once and then
+    recorded as failed. It is never partially accepted.
+- **Labeler check**: the labeler labels the 215 holdout prompts once, before any training
+  data is labeled. The report gives agreement with the owner's gold and the per-class recall.
+  - Labeling proceeds only when agreement is at least 80%.
+  - The instructions are never tuned on the holdout. If the check fails, the owner reviews
+    the disagreements and decides how to proceed.
+- **Owner spot-check**: a random 5% sample of the labeled training data is reviewed on the
+  review page, and the disagreement rate is reported. The owner's decisions replace the LLM
+  labels for those cases.
+
+## Training and calibration
+
+- **Model**: a new `cua_s1` `tinyx` scorer built with `make_system`.
+  - Config: `width` 128, `rank` 128, `layers` 2, `heads` 4, `context_tokens` 1024 and
+    `option_tokens` 96.
+  - Weights are initialized fresh, not from the forms checkpoint, whose domain and
+    positional range differ.
+- **Target**: each example is the fixed tier catalog with the index of the gold option.
+  `null` maps to `unclassified`.
+- **Optimization**:
+  - Cross-entropy, AdamW, a fixed seed and CPU.
+  - The epoch with the lowest validation negative log-likelihood is kept.
+  - The checkpoint metadata records the dataset hashes, the config and the validation metrics.
+- **Calibration**: `minConfidence` and `minMargin` are chosen on validation to maximize
+  coverage, subject to an accepted precision of at least 98%. The router currently uses the
+  Phase 0 defaults (0.9 and 0.2). Carrying calibrated thresholds into the manifest is a
+  separate contract change.
+
+## Evaluation and release
+
+- **Evaluation**: `scripts/evaluate-system-one.js` runs once on
+  `benchmarks/fixtures/system-one-holdout.json` with the new manifest. The rollout gates and
+  the lexical baseline (macro-F1 0.254) decide the result. The report goes under
+  `benchmarks/results/system-one/`.
+- **Release**: the `.safetensors` and `.json` files are attached to a GitHub release. The
+  installer's pins gain the new model ID, revision, `domain: "harness-routing-v1"` and both
+  SHA-256 values. Until every rollout gate passes, the router default stays `off`.

package/harness-everything/SKILL.md

@@ -4,7 +4,7 @@ description: "Route software/project work through Harness: classify tier/topolog
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.22.1
+  version: 0.24.0
 ---
 
 # Harness Everything