harness-everything 0.28.0 → 0.29.0

package/ci/mechanism-system-one-evaluation.test.js

@@ -85,3 +85,20 @@ test('S1-E07 source provenance gate requires the pinned cua_s1 revision', () =>
   assert.equal(evaluate(make(), records(), recorded('pinned')).gates.sourceProvenance, true);
   assert.equal(evaluate(make(), records(), recorded('pinned')).rolloutReady, false);
 });
+test('S1-E09 an ngram source passes provenance only with a verified artifact', () => {
+  const ngramSource = verified => ({ status: 'recorded', transport: 'ngram', artifactVerified: verified });
+  assert.equal(evaluate(make(), records(), ngramSource(true)).gates.sourceProvenance, true);
+  assert.equal(evaluate(make(), records(), ngramSource(false)).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), { status: 'recorded', transport: 'ngram' }).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), { status: 'unavailable', reason: 'provider-config', transport: 'ngram', artifactVerified: true }).gates.sourceProvenance, false);
+});
+test('S1-E08 the accepted-precision gate is the owner advisory 85%', () => {
+  // 7 accepted tier1 answers: 6 correct (85.7%) passes, 5 correct (71.4%) fails.
+  const cases = Array.from({ length: 7 }, (_, i) => ({ id: `p${i}`, family: `f${i}`, split: 'holdout', language: 'en', source: 'human:fixture', reviewed: false,
+    request: createRequest('tier', `bounded prompt ${i}`, TIER_OPTIONS), gold: 'tier1' }));
+  const corpus = { schemaVersion: 1, cases };
+  const recs = wrong => cases.map((c, i) => { const id = i < wrong ? 'tier2' : 'tier1';
+    return { id: c.id, baseline: 'tier2', runs: [0, 1].map(() => ({ decision: decision(id), scores: oneHot(id), latencyMs: 20, coldStart: true })) }; });
+  assert.equal(evaluate(corpus, recs(1)).gates.acceptedPrecision, true);
+  assert.equal(evaluate(corpus, recs(2)).gates.acceptedPrecision, false);
+});

package/ci/mechanism-system-one-label.test.js

@@ -61,6 +61,33 @@ test('S1-L03 invalid replies are retried once, persistent failures are recorded,
   assert.equal(again.labeled.length, 0);
 });
 
+test('S1-L06 the codex engine runs one stateless exec per batch and reads the schema-checked last message', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-codex-'));
+  try {
+    // Fake codex: records its argv, reads the prompt from stdin, writes the reply to the -o file.
+    const fake = path.join(dir, 'fake-codex.js'); const argvLog = path.join(dir, 'argv.json');
+    // Without an -o target the fake exits instead of writing (a[-1 + 1] would land in the working directory).
+    fs.writeFileSync(fake, `const fs=require('fs');const a=process.argv.slice(2);fs.writeFileSync(${JSON.stringify(argvLog)},JSON.stringify(a));
+      const o=a.indexOf('-o');if(o<0)process.exit(3);
+      let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const m=[...s.matchAll(/\\{"i":(\\d+),"text"/g)];
+      fs.writeFileSync(a[o+1],JSON.stringify({labels:m.map(x=>({i:Number(x[1]),gold:'tier1'}))}));});`);
+    const input = path.join(dir, 'prompts.jsonl');
+    fs.writeFileSync(input, items(3).map(r => JSON.stringify({ ...r, family: 'f', source: 'claude', split: 'train' })).join('\n') + '\n');
+    const out = path.join(dir, 'labels.jsonl');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-label.js'), 'label', '--engine', 'codex', '--in', input, '--out', out],
+      { encoding: 'utf8', cwd: root, env: { ...process.env, HARNESS_S1_LABEL_COMMAND: JSON.stringify([process.execPath, fake]) } });
+    assert.equal(r.status, 0, r.stderr);
+    assert.deepEqual(JSON.parse(r.stdout), { labeled: 3, failed: 0, total: 3 });
+    const rows = fs.readFileSync(out, 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(x => x.gold === 'tier1' && x.labeler.model === 'gpt-5.6-luna' && x.labeler.engine === 'codex'));
+    const argv = JSON.parse(fs.readFileSync(argvLog, 'utf8'));
+    for (const flag of ['exec', '--ephemeral', '--ignore-user-config', '--output-schema', '-o']) assert.ok(argv.includes(flag), flag);
+    assert.equal(argv[argv.indexOf('--sandbox') + 1], 'read-only');
+    assert.equal(argv[argv.indexOf('--model') + 1], 'gpt-5.6-luna');
+    assert.equal(argv[argv.length - 1], '-', 'the prompt comes from stdin');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
 test('S1-L04 holdout agreement reports overall, per-class recall and confusion', () => {
   const gold = [['a', 'tier1'], ['b', 'tier1'], ['c', 'tier2'], ['d', null]];
   const pred = { a: 'tier1', b: 'tier2', c: 'tier2', d: null };

package/ci/mechanism-system-one-ngram.test.js

@@ -0,0 +1,161 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const ngram = require('../harness-everything/scripts/system-one/ngram');
+const provider = require('../harness-everything/scripts/system-one/provider');
+const { createRequest, decide } = require('../harness-everything/scripts/system-one/contract');
+const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const root = path.resolve(__dirname, '..');
+const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+const sha = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+const IDS = TIER_OPTIONS.map(o => o.id);
+
+// A tiny model whose weights push any text containing the "commit" trigram toward tier1.
+function artifact(dir, { dim = 1024, nmax = 3, catalog = IDS, sidecar = {}, bytes } = {}) {
+  const W = new Float32Array(dim * catalog.length);
+  const hot = ngram.fnv1a32('3:com') % dim;
+  W[hot * catalog.length + 0] = 40;
+  const bin = path.join(dir, 'tier.bin'); const json = path.join(dir, 'tier.json');
+  fs.writeFileSync(bin, bytes || Buffer.from(W.buffer));
+  fs.writeFileSync(json, JSON.stringify({ format: 'harness-ngram', formatVersion: 1, dim, nmax, hash: 'fnv1a32', catalog, bias: catalog.map(() => 0), metadata: { fixture: true }, ...sidecar }));
+  return { bin, json };
+}
+function manifest(dir, files, extra = {}) {
+  const m = { schemaVersion: 1, transport: 'ngram', checkpoint: files.bin, weightsSha256: sha(files.bin), configSha256: sha(files.json),
+    modelId: 'fixture/ngram', revision: 'v1', domain: 'harness-routing-v1', ...extra };
+  const p = path.join(dir, 'manifest.json'); fs.writeFileSync(p, JSON.stringify(m)); return { m, p };
+}
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-ngram-'));
+
+test('S1-N01 fnv1a32 matches the published test vectors', () => {
+  assert.equal(ngram.fnv1a32(''), 0x811c9dc5);
+  assert.equal(ngram.fnv1a32('a'), 0xe40c292c);
+  assert.equal(ngram.fnv1a32('foobar'), 0xbf9cf968);
+});
+
+test('S1-N02 featurization follows the pinned rules', () => {
+  const f = ngram.featurize('  Commit   ALL\tchanges ', 3, 1 << 16);
+  assert.deepEqual(f, ngram.featurize('commit all changes', 3, 1 << 16), 'lowercase and whitespace normalization');
+  const norm = Math.sqrt([...f.values()].reduce((s, v) => s + v * v, 0));
+  assert.ok(Math.abs(norm - 1) < 1e-12);
+  assert.ok(f.has(ngram.fnv1a32('len:4') % (1 << 16)), 'length bucket for 18 UTF-8 bytes');
+  const cjk = ngram.featurize('修正😀', 2, 1 << 16);
+  assert.ok(cjk.has(ngram.fnv1a32('2:正😀') % (1 << 16)), 'n-grams are over code points, not UTF-16 units');
+  assert.ok(!ngram.featurize('a b', 3, 1 << 16).has(ngram.fnv1a32('1: ') % (1 << 16)), 'space-only grams are skipped');
+});
+
+test('S1-N03 the Python trainer and the Node provider produce identical features', () => {
+  const texts = ['commit all changes', '幫我修正 parser 的錯誤並補測試', '  MIXED Case\ttext 😀 ', 'x', '重構 plugin 介面，跨兩個 repo'];
+  const hashes = ['', 'a', 'foobar', '3:修正測'];
+  const r = spawnSync(python, [path.join(__dirname, 'system-one-ngram-parity.py')], { input: JSON.stringify({ texts, hashes, nmax: 3, dim: 1 << 16 }), encoding: 'utf8', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', PYTHONIOENCODING: 'utf-8' } });
+  assert.equal(r.status, 0, r.stderr);
+  const py = JSON.parse(r.stdout);
+  assert.deepEqual(py.hash, hashes.map(h => ngram.fnv1a32(h)));
+  texts.forEach((t, i) => {
+    const node = ngram.featurize(t, 3, 1 << 16);
+    assert.deepEqual(Object.keys(py.features[i]).map(Number).sort((a, b) => a - b), [...node.keys()].sort((a, b) => a - b), t);
+    for (const [k, v] of Object.entries(py.features[i])) assert.ok(Math.abs(v - node.get(Number(k))) < 1e-12, t);
+  });
+});
+
+test('S1-N04 the ngram transport scores in process with the Phase 0 response shape', async () => {
+  const dir = tmp();
+  try {
+    const { m, p } = manifest(dir, artifact(dir), { acceptance: { minConfidence: 0.5, minMargin: 0 } });
+    assert.equal(provider.validateManifest(m), true);
+    const request = createRequest('tier', 'please commit these changes', TIER_OPTIONS);
+    const result = provider.score(request, p);
+    assert.equal(result.status, 'scored');
+    assert.deepEqual(result.acceptance, { minConfidence: 0.5, minMargin: 0 });
+    assert.deepEqual(result.response.model, { id: 'fixture/ngram', revision: 'v1', domain: 'harness-routing-v1' });
+    assert.ok(Math.abs(result.response.scores.reduce((s, x) => s + x.probability, 0) - 1) < 1e-9);
+    const d = decide(request, result.response, result.acceptance);
+    assert.equal(d.status, 'accepted'); assert.equal(d.selectedId, 'tier1');
+    assert.deepEqual(await provider.scoreAsync(request, p), result);
+    const other = provider.score(createRequest('tier', 'unrelated words only', TIER_OPTIONS), p);
+    assert.ok(other.response.scores.every(s => Math.abs(s.probability - 0.25) < 1e-9), 'no active weights gives a uniform vector');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N05 any artifact mismatch is provider-config and never scores', () => {
+  const request = createRequest('tier', 'commit it', TIER_OPTIONS);
+  const cases = [
+    dir => { const f = artifact(dir); const { m, p } = manifest(dir, f); fs.appendFileSync(f.bin, Buffer.alloc(4)); return p; },
+    dir => { const f = artifact(dir, { bytes: Buffer.alloc(8) }); return manifest(dir, f).p; },
+    dir => manifest(dir, artifact(dir, { catalog: ['tier1', 'tier2', 'tier3', 'other'] })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { format: 'other' } })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { extra: 1 } })).p,
+    dir => manifest(dir, artifact(dir, { dim: 1000 })).p,
+    dir => manifest(dir, artifact(dir, { nmax: 9 })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { bias: [0, 0] } })).p,
+  ];
+  for (const make of cases) {
+    const dir = tmp();
+    try { assert.deepEqual(provider.score(request, make(dir)), { status: 'unavailable', reason: 'provider-config' }); }
+    finally { fs.rmSync(dir, { recursive: true, force: true }); }
+  }
+});
+
+test('S1-N06 manifest rules: ngram needs a .bin and no python; other transports still need python', () => {
+  const dir = tmp();
+  try {
+    const { m } = manifest(dir, artifact(dir));
+    assert.equal(provider.validateManifest(m), true);
+    assert.throws(() => provider.validateManifest({ ...m, checkpoint: m.checkpoint.replace(/\.bin$/, '.safetensors') }));
+    assert.throws(() => provider.validateManifest({ ...m, transport: 'resident' }), 'resident requires python and a .safetensors checkpoint');
+    assert.throws(() => provider.validateManifest({ ...m, transport: 'bogus' }));
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N08 the evaluator records ngram samples warm and verifies the artifact as provenance', () => {
+  const dir = tmp();
+  try {
+    const { p } = manifest(dir, artifact(dir));
+    const output = path.join(dir, 'report.json');
+    const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, p, output], { encoding: 'utf8', cwd: root });
+    assert.equal(r.status, 0, r.stderr);
+    const report = JSON.parse(fs.readFileSync(output, 'utf8'));
+    assert.ok(report.records.every(x => x.runs.every(run => run.coldStart === false)));
+    assert.equal(report.latency.coldSamples, 0);
+    assert.equal(report.gates.warmLatency, true);
+    assert.deepEqual(report.evidence.source, { status: 'recorded', transport: 'ngram', artifactVerified: true });
+    assert.equal(report.gates.sourceProvenance, true);
+    assert.equal(report.evidence.kind, 'offline-ngram');
+    fs.appendFileSync(path.join(dir, 'tier.bin'), Buffer.alloc(4));
+    const tampered = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, p, output], { encoding: 'utf8', cwd: root });
+    assert.equal(tampered.status, 0, tampered.stderr);
+    const bad = JSON.parse(fs.readFileSync(output, 'utf8'));
+    assert.equal(bad.evidence.source.artifactVerified, false);
+    assert.equal(bad.gates.sourceProvenance, false);
+    assert.equal(bad.model.coverage, 0);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N07 a real ngram training run produces an artifact the provider scores (skipped without torch)', t => {
+  if (spawnSync(python, ['-c', 'import torch'], { encoding: 'utf8' }).status !== 0) { t.skip('torch not installed'); return; }
+  const dir = tmp();
+  try {
+    const data = path.join(dir, 'data'); fs.mkdirSync(data);
+    const texts = [['commit all changes', 'tier1'], ['push and open a pr', 'tier1'], ['add a new flag with tests', 'tier3'], ['migrate every service', 'tier3'], ['fix the failing parser test', 'tier2'], ['go', null]];
+    const prompts = []; const labels = [];
+    for (let k = 0; k < 10; k++) texts.forEach(([x, g], i) => { const id = `p${k}-${i}`; prompts.push({ id, family: `f${k}`, source: 'claude', split: k < 8 ? 'train' : 'validation', text: `${x} ${k}` }); labels.push({ id, gold: g }); });
+    fs.writeFileSync(path.join(data, 'prompts.jsonl'), prompts.map(x => JSON.stringify(x)).join('\n') + '\n');
+    fs.writeFileSync(path.join(data, 'labels.jsonl'), labels.map(x => JSON.stringify(x)).join('\n') + '\n');
+    const out = path.join(dir, 'model');
+    const r = spawnSync(python, [path.join(root, 'scripts/system-one-train-ngram.py'), '--data-dir', data, '--out', out, '--dim', '4096', '--epochs', '40'], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stdout + r.stderr);
+    const files = { bin: path.join(out, 'harness-routing-v1.bin'), json: path.join(out, 'harness-routing-v1.json') };
+    assert.equal(fs.statSync(files.bin).size, 4096 * 4 * 4);
+    assert.equal(fs.readFileSync(path.join(out, 'val-scores.jsonl'), 'utf8').trim().split('\n').length, 12);
+    const { p } = manifest(dir, files);
+    const scored = provider.score(createRequest('tier', 'commit all changes now', TIER_OPTIONS), p);
+    assert.equal(scored.status, 'scored');
+    assert.equal(decide(createRequest('tier', 'commit all changes now', TIER_OPTIONS), scored.response, { minConfidence: 0.5, minMargin: 0 }).selectedId, 'tier1');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-train.test.js

@@ -41,6 +41,18 @@ test('S1-T03 calibration maximizes coverage subject to accepted precision on val
   assert.deepEqual(calibrate(rows, { minPrecision: 0.98 }), c, 'deterministic');
 });
 
+test('S1-T06 the default calibration target is the owner advisory 85%', () => {
+  const rows = [
+    ...Array.from({ length: 86 }, () => row('tier1', [0.9, 0.05, 0.03, 0.02])),
+    ...Array.from({ length: 14 }, () => row('tier2', [0.9, 0.05, 0.03, 0.02])),
+  ];
+  const c = calibrate(rows);
+  assert.equal(c.minPrecision, 0.85);
+  assert.equal(c.feasible, true);
+  assert.equal(c.coverage, 1);
+  assert.equal(calibrate(rows, { minPrecision: 0.9 }).feasible, false);
+});
+
 test('S1-T04 calibration CLI reads validation scores and writes thresholds', () => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-cal-'));
   try {

package/ci/system-one-ngram-parity.py

@@ -0,0 +1,19 @@
+"""Print the trainer's n-gram features for texts on stdin (JSON list) so the Node provider can be compared exactly."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+
+sys.dont_write_bytecode = True
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('ngram_trainer', ROOT / 'scripts' / 'system-one-train-ngram.py')
+trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(trainer)
+assert 'torch' not in sys.modules, 'importing the ngram trainer must not import torch'
+
+request = json.loads(sys.stdin.read())
+out = []
+for text in request['texts']:
+    feats = trainer.featurize(text, request['nmax'], request['dim'])
+    out.append({str(k): v for k, v in sorted(feats.items())})
+print(json.dumps({'hash': [trainer.fnv1a32(s) for s in request['hashes']], 'features': out}))

package/ci/system-one-train-test.py

@@ -46,6 +46,12 @@ assert flat == list(range(len(lengths))), 'every example exactly once'
 assert all(len(b) * max(lengths[i] for i in b) <= 4096 or len(b) == 1 for b in batches), 'budget respected'
 assert batches == trainer.make_batches(lengths, token_budget=4096), 'deterministic'
 
+weights = trainer.class_weights([0, 0, 0, 1, 2, 2], 4)
+assert len(weights) == 4
+assert weights[3] == 0.0, 'an absent class gets no weight'
+assert weights[1] > weights[2] > weights[0], weights
+assert abs(sum(w * n for w, n in zip(weights, [3, 1, 2, 0])) - 6) < 1e-9, 'weights average to 1 over the examples'
+
 with tempfile.TemporaryDirectory() as tmp:
     p = Path(tmp) / 'rows.jsonl'
     p.write_text('\n'.join(json.dumps(r) for r in prompts) + '\n', encoding='utf-8')

package/docs/system-one-corpus.md

@@ -16,7 +16,7 @@ to, judged from the prompt alone?** The labels follow the tier model in
 | --- | --- | --- |
 | `tier1` | Something answered or done without changing project behavior: Git/GitHub housekeeping (commit, push, open a PR, sync, tag, post given results as a comment), a status check, reading a log or searching and reporting, the smallest doc fix (a typo, a link, an ignore entry), or a reply the assistant can give directly, including non-engineering chores and discussion questions | "commit these changes", "is issue 88 still open?", "find every repo still using the old logging package", "close the other apps" |
 | `tier2` | A bounded change or investigation: any code, configuration or CI edit (a rename, a value, a revert), running tests or a build, filing an issue, explaining code that must be read first, most bug fixes, a focused review or benchmark, planning or synthesis that produces documents (phase plans, an ADR, an overview), and answers that settle open decisions for work in progress | "rename tmp to buffer", "run npm test", "the CLI fails with this error, fix it", "1. agree 2. keep the old name" |
-| `tier3` | New or changed behavior delivered with tests, and anything larger: a new flag or format, a behavior change across producers and tests, a large refactor, a dependency upgrade with breakage, a migration, a repository-wide change, a multi-phase implementation, or delegation to several agents | "add a --verbose flag with tests", "migrate to ESM", "dispatch agents to audit every repo" |
+| `tier3` | Work that needs several agents (sub-agents included); a new module or subsystem, a refactor, or a change to a framework or interface definition; or code or design work that spans several repositories or several workflows | "add a plugin system to the CLI", "refactor the parser into modules", "change the plugin interface", "dispatch agents to audit every repo" |
 | `null` | No actionable content: a bare continuation, "do what you said", a vague reaction, a question about what the assistant meant, or pasted status with no request | "go", "yes", "照剛剛說的改", "something is off here" |
 
 Rules:
@@ -31,6 +31,30 @@ Rules:
    reformat or extend the assistant's previous answer without changing files is
    `tier1`. A requested change to code or configuration, however small, is at
    least `tier2`. Only the smallest doc fixes listed in the table stay `tier1`.
+   The line between `tier2` and `tier3` is the owner's: `tier3` when the work
+   needs several agents or sub-agents, adds a feature, refactors, changes a
+   framework or interface definition, or spans several repositories or
+   workflows. Fixing, adjusting, investigating or reviewing existing behavior
+   within one workflow is `tier2`.
+   The owner settled four more lines from the third spot-check:
+   - **Git is `tier1` at any width.** Committing sub-repositories and then the
+     main repository is still housekeeping. "Several repositories" applies to
+     code or design work only.
+   - **Scope, not the word "add".** Adding an option, a function or a parser
+     inside one existing tool or component is `tier2`. A new module or
+     subsystem, or an addition that spans several components, is `tier3`.
+     A bug fix, even with a regression test, is `tier2`.
+   - **Repository-wide or cross-component work is `tier3`.** Replacing or
+     unifying a pattern across the repository (every `console.log`, all error
+     handling, every path in a report), upgrading a dependency and fixing what
+     breaks, a data migration, a CI matrix across platforms, or restructuring a
+     documentation tree is `tier3`, even when each edit is small.
+   - **A short reply follows the work in progress.** A reply that asks to
+     change an output (code, a document, an example) is `tier2`. Only a pure
+     question or a bare acknowledgement stays `tier1`.
+   - **Explicit agents or architecture make `tier3`.** A request for several
+     agents, a team discussion or a verifier pass, or one that redefines an
+     architecture, is `tier3` even when it is phrased as discussion.
 4. Explicit workflow words ("use fable", "no subagents") do not change the gold.
    Precedence is a router policy, not a label.
 5. Never derive gold from the lexical router or any model output. A reviewer may
@@ -102,6 +126,10 @@ a decision on its own.
 | `benchmarks/fixtures/system-one-holdout.json` | Evaluation corpus, rebuilt with `node scripts/system-one-corpus.js build <draft> <reviews> <corpus>` |
 
 The reviewed holdout has 215 cases (11 rejected): en 93 and zh-TW 122; gold
-`tier1` 47, `tier2` 97, `tier3` 51 and `null` 20. It meets the `reviewedHoldout`
-gate. It says nothing about model quality until a `harness-routing-v1`
+`tier1` 48, `tier2` 104, `tier3` 43 and `null` 20. It meets the `reviewedHoldout`
+gate. After the third training spot-check the owner moved nine cases to the new
+lines: `h-long-git-ops-1` to `tier1`, and the `h-add-cli-flag-*`,
+`h-hash-algorithm-support-1/2`, `h-fix-cli-error-log-3/4` and
+`h-fix-failing-test-3` cases to `tier2`. Results recorded before that change
+used 47 `tier1`, 97 `tier2` and 51 `tier3`. It says nothing about model quality until a `harness-routing-v1`
 checkpoint is evaluated on it.

package/docs/system-one-routing.md

@@ -103,6 +103,52 @@ responses are checked by Phase 0. Transport fixtures prove IPC and failure
 handling only. Real checkpoint inference and measured CPU latency remain a
 separate gate, with explicit unavailable evidence if dependencies are absent.
 
+## N-gram provider (Phase 4)
+
+`transport: "ngram"` scores in the Node process itself. It needs no Python, no
+torch, no resident server and no subprocess. It exists because a byte-level
+transformer trained from scratch learned little beyond class priors on the
+owner's data. A linear model over character n-grams learned more from the same
+labels at about 1 MB. See
+[harness-routing-v1-cpu-2026-09-24](../benchmarks/results/system-one/harness-routing-v1-cpu-2026-09-24/README.md).
+
+**Artifact.** The manifest `checkpoint` is an absolute path to `<name>.bin`, and
+`weightsSha256` is that file's hash. A `<name>.json` sidecar beside it has the
+hash `configSha256`. `python` is not required for this transport. The sidecar
+holds exactly:
+
+| Field | Value |
+| --- | --- |
+| `format` | `harness-ngram` |
+| `formatVersion` | `1` |
+| `dim` | Feature buckets, a power of two between 2^10 and 2^20 |
+| `nmax` | Longest n-gram, 1–4 |
+| `hash` | `fnv1a32` |
+| `catalog` | The option IDs in catalog order |
+| `bias` | One number per option |
+| `metadata` | An object with training provenance |
+
+`.bin` holds `dim × catalog.length` little-endian float32 weights in row-major
+order (feature, then option), so its size must equal `dim × options × 4` bytes.
+Both files are hash-checked before use. Any mismatch in hash, shape or field
+returns `provider-config` and never scores.
+
+**Features.** Featurization is identical in the trainer and the provider, and a
+cross-language test proves it.
+
+1. Lowercase the text and join its whitespace-separated parts with one space.
+2. For each n from 1 to `nmax`, take every n-gram of Unicode code points that is
+   not only spaces. Add it to bucket `fnv1a32("<n>:<gram>" as UTF-8) mod dim`.
+3. Add one length feature, bucket `fnv1a32("len:<k>") mod dim`, where
+   `k = min(8, floor(log2(utf8Bytes + 1)))`.
+4. Weight each bucket as `log(1 + count)` and L2-normalize the vector.
+
+**Scoring.** The probabilities are `softmax(x·W + bias)` in catalog order. The
+request's option IDs must equal `catalog`, or the result is `provider-config`.
+The response has the Phase 0 shape, and the manifest `acceptance` thresholds
+apply as for any transport. The Phase 0 context limit (64 KiB) is the only
+length limit.
+
 ## Resident provider (Phase 4 prerequisite)
 
 The one-shot bridge pays Python start, `import torch` and checkpoint load on every
@@ -262,6 +308,15 @@ The latter evidence is not supplied by this offline runner, so it always reports
 those gates pending. With the one-shot adapter, all latency is cold and the warm
 gate also remains pending. Exporting a report never changes router defaults.
 
+For the `ngram` transport, samples are recorded with `coldStart: false`, because
+scoring happens in process. The evaluator loads and verifies the artifact once
+before any timed sample, so that load is not part of the warm latency. The
+source check does not run the Python `cua_s1` probe, which this transport does
+not use. It records `{status: "recorded", transport: "ngram", artifactVerified}`,
+where `artifactVerified` is true only when both artifact files load and match
+their SHA-256 values. For this transport, the `sourceProvenance` gate passes
+when `artifactVerified` is true.
+
 ## Model suitability
 
 The [CUA-S1-FORMS model card](https://huggingface.co/cua-ai/cua-s1-forms)
@@ -286,11 +341,38 @@ unversioned download during a prompt hook.
 Phase 4 training data, labeling, calibration and release are defined in
 [system-one-training.md](system-one-training.md).
 
+## Staged classification
+
+System One answers three narrow questions in sequence instead of one broad
+question. Each stage is its own fixed catalog, scorer request and holdout, so a
+small model only has to separate a few options at a time.
+
+1. **Tier**: `tier1`, `tier2`, `tier3` or `unclassified`, following
+   [system-one-corpus.md](system-one-corpus.md).
+2. **Intent**: the kind of work the prompt asks for, drawn from a small fixed
+   catalog (for example: ask or explain, discuss or decide, Git/GitHub
+   operation, fix a bug, build a feature, refactor, review or audit, test or
+   verify, write docs, plan or spec, investigate). The intent catalog is the
+   owner's and is recorded in the corpus document before any intent labels.
+3. **Skills**: which canonical skills fit the prompt, scored against skill
+   descriptions. Only skills above the stage's calibrated threshold are
+   suggested, and they are suggestions, never required reads.
+
+Each stage has its own gates, and a stage ships only when it passes them. Tier
+comes first because the reviewed holdout exists for it. Later stages reuse the
+same collector, labeler, trainer and evaluator with a different catalog.
+Suggestions are advisory: explicit workflow requests, action gates, memory
+ownership, the Rule of 3 and deterministic policy always take precedence.
+
 ## Rollout gates
 
 Phase 4 requires a separately reviewed, family-disjoint holdout of at least 200
-cases, at least 50 each in English and Traditional Chinese. Accepted precision
-must be ≥98%, coverage ≥80%, and macro-F1 at least the lexical baseline.
+cases, at least 50 each in English and Traditional Chinese. System One output is
+advisory: it informs the agent and suggests skills. It never grants or removes
+policy. The owner therefore set the target at accepted precision ≥85% (the
+80–90% band), coverage ≥80%, and macro-F1 at least the lexical baseline. A
+probabilistic scorer small enough to ship inside a skills plugin (a few MB, not
+hundreds) is not expected to reach 98%.
 Policy invariants must remain intact in every case; repeat decisions twice with
 100% agreement. Record checkpoint/source hashes, CPU, OS, runtime, thread count,
 cold latency and warm p95 (target ≤100ms). Synthetic fixtures and mock providers

package/docs/system-one-training.md

@@ -104,7 +104,19 @@ prompts written by an agent, not by the owner. VS Code terminal notifications
 
 ## Training and calibration
 
-- **Model**: a new `cua_s1` `tinyx` scorer built with `make_system`.
+- **Tier engine**: `scripts/system-one-train-ngram.py` trains the `ngram` transport, a
+  multinomial logistic regression over hashed character n-gram features. Featurization
+  follows [system-one-routing.md](system-one-routing.md#n-gram-provider-phase-4).
+  - The trainer uses torch for the optimizer only. The features are stdlib Python and match
+    the Node provider exactly.
+  - The configuration (`nmax`, class weighting, weight decay) is chosen on validation
+    macro-F1. The first exploration picked `nmax` 3, square-root inverse-frequency class
+    weights and a weight decay of 1e-4.
+  - The output is the `.bin` weights, the `.json` sidecar and validation scores for
+    calibration.
+- The `tinyx` path below is kept for comparison. Its first checkpoint learned little beyond
+  class priors.
+- **Model** (`tinyx`): a new `cua_s1` `tinyx` scorer built with `make_system`.
   - Config: `width` 128, `rank` 128, `layers` 2, `heads` 4, `context_tokens` 1024 and
     `option_tokens` 96.
   - Weights are initialized fresh, not from the forms checkpoint, whose domain and
@@ -116,7 +128,7 @@ prompts written by an agent, not by the owner. VS Code terminal notifications
   - The epoch with the lowest validation negative log-likelihood is kept.
   - The checkpoint metadata records the dataset hashes, the config and the validation metrics.
 - **Calibration**: `minConfidence` and `minMargin` are chosen on validation to maximize
-  coverage, subject to an accepted precision of at least 98%. The router currently uses the
+  coverage, subject to an accepted precision of at least 85% (the owner's advisory target). The router currently uses the
   Phase 0 defaults (0.9 and 0.2). Carrying calibrated thresholds into the manifest is a
   separate contract change.
 

package/harness-everything/SKILL.md

@@ -4,7 +4,7 @@ description: "Route software/project work through Harness: classify tier/topolog
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.26.0
+  version: 0.29.0
 ---
 
 # Harness Everything

package/harness-everything/scripts/system-one/evaluate.js

@@ -84,12 +84,13 @@ function evaluate(corpus, records, sourceEvidence = null) {
   const gates = {
     reviewedHoldout: cases.length >= 200 && cases.every(c => c.reviewed)
       && ['en', 'zh-TW'].every(lang => cases.filter(c => c.language === lang).length >= 50),
-    acceptedPrecision: model.acceptedPrecision !== null && model.acceptedPrecision >= 0.98,
+    acceptedPrecision: model.acceptedPrecision !== null && model.acceptedPrecision >= 0.85,
     coverage: model.coverage >= 0.8,
     macroF1: model.macroF1 >= baseline.macroF1,
     repeatability: agreement === 1,
     warmLatency: warm.length >= cases.length && latency.warmP95Ms <= 100,
-    sourceProvenance: sourceEvidence?.status === 'recorded' && sourceEvidence.provenance?.cuaS1?.sourceRevisionStatus === 'pinned',
+    sourceProvenance: sourceEvidence?.status === 'recorded' && (sourceEvidence.provenance?.cuaS1?.sourceRevisionStatus === 'pinned'
+      || (sourceEvidence.transport === 'ngram' && sourceEvidence.artifactVerified === true)),
     policyEvidence: false,
     liveHostEvidence: false,
   };

package/harness-everything/scripts/system-one/ngram.js

@@ -0,0 +1,84 @@
+'use strict';
+// In-process System One scorer: hashed character n-grams + a linear layer (docs/system-one-routing.md, "N-gram provider").
+// Featurization must match scripts/system-one-train-ngram.py exactly; ci/mechanism-system-one-ngram.test.js proves it.
+const fs = require('node:fs');
+const { createHash } = require('node:crypto');
+
+const SIDE_KEYS = ['format', 'formatVersion', 'dim', 'nmax', 'hash', 'catalog', 'bias', 'metadata'];
+const cache = new Map();
+const configError = () => Object.assign(new Error('provider-config'), { code: 'provider-config' });
+const sha = buf => createHash('sha256').update(buf).digest('hex');
+
+function fnv1a32(text) {
+  let h = 0x811c9dc5;
+  for (const byte of Buffer.from(text, 'utf8')) { h ^= byte; h = Math.imul(h, 0x01000193) >>> 0; }
+  return h >>> 0;
+}
+// Map of bucket -> weight: log(1 + count) per code-point n-gram, one length bucket, L2-normalized.
+function featurize(text, nmax, dim) {
+  const t = text.toLowerCase().split(/\s+/).filter(Boolean).join(' ');
+  const cps = Array.from(t);
+  const counts = new Map();
+  for (let n = 1; n <= nmax; n++) {
+    for (let i = 0; i + n <= cps.length; i++) {
+      const gram = cps.slice(i, i + n).join('');
+      if (!gram.trim()) continue;
+      const b = fnv1a32(`${n}:${gram}`) % dim;
+      counts.set(b, (counts.get(b) || 0) + 1);
+    }
+  }
+  const k = Math.min(8, Math.floor(Math.log2(Buffer.byteLength(t, 'utf8') + 1)));
+  counts.set(fnv1a32(`len:${k}`) % dim, 1);
+  const feats = new Map();
+  let norm = 0;
+  for (const [b, c] of counts) { const v = Math.log1p(c); feats.set(b, v); norm += v * v; }
+  norm = Math.sqrt(norm);
+  for (const [b, v] of feats) feats.set(b, v / norm);
+  return feats;
+}
+function validSidecar(s) {
+  const isPow2 = n => Number.isInteger(n) && n >= 1024 && n <= 1 << 20 && (n & (n - 1)) === 0;
+  return s && typeof s === 'object' && !Array.isArray(s) && Object.keys(s).length === SIDE_KEYS.length && SIDE_KEYS.every(k => Object.hasOwn(s, k))
+    && s.format === 'harness-ngram' && s.formatVersion === 1 && isPow2(s.dim) && Number.isInteger(s.nmax) && s.nmax >= 1 && s.nmax <= 4
+    && s.hash === 'fnv1a32' && Array.isArray(s.catalog) && s.catalog.length >= 2 && new Set(s.catalog).size === s.catalog.length
+    && s.catalog.every(id => typeof id === 'string' && /^[a-z0-9._-]{1,80}$/.test(id))
+    && Array.isArray(s.bias) && s.bias.length === s.catalog.length && s.bias.every(Number.isFinite)
+    && s.metadata && typeof s.metadata === 'object' && !Array.isArray(s.metadata);
+}
+// Loads and verifies the artifact once per process; any mismatch throws provider-config.
+function loadModel(manifest) {
+  const key = `${manifest.checkpoint}|${manifest.weightsSha256}|${manifest.configSha256}`;
+  if (cache.has(key)) return cache.get(key);
+  const sidecarPath = manifest.checkpoint.replace(/\.bin$/, '.json');
+  let side; let bin;
+  try {
+    const raw = fs.readFileSync(sidecarPath);
+    if (raw.length > 65536 || sha(raw) !== manifest.configSha256) throw configError();
+    side = JSON.parse(raw.toString('utf8'));
+    bin = fs.readFileSync(manifest.checkpoint);
+  } catch (_) { throw configError(); }
+  if (sha(bin) !== manifest.weightsSha256 || !validSidecar(side) || bin.length !== side.dim * side.catalog.length * 4) throw configError();
+  const W = new Float32Array(side.dim * side.catalog.length);
+  for (let i = 0; i < W.length; i++) W[i] = bin.readFloatLE(i * 4);
+  const model = { dim: side.dim, nmax: side.nmax, catalog: side.catalog, bias: side.bias, W };
+  cache.set(key, model);
+  return model;
+}
+function score(request, manifest) {
+  const model = loadModel(manifest);
+  const C = model.catalog.length;
+  if (request.options.length !== C || request.options.some((o, i) => o.id !== model.catalog[i])) throw configError();
+  const logits = model.bias.slice();
+  for (const [b, v] of featurize(request.context, model.nmax, model.dim)) {
+    for (let c = 0; c < C; c++) logits[c] += v * model.W[b * C + c];
+  }
+  const max = Math.max(...logits);
+  const exps = logits.map(z => Math.exp(z - max));
+  const sum = exps.reduce((s, x) => s + x, 0);
+  return {
+    schemaVersion: 1, requestHash: request.requestHash, catalogHash: request.catalogHash,
+    model: { id: manifest.modelId, revision: manifest.revision, domain: manifest.domain },
+    scores: request.options.map((o, i) => ({ id: o.id, probability: exps[i] / sum })),
+  };
+}
+module.exports = { fnv1a32, featurize, loadModel, score };

package/harness-everything/scripts/system-one/provider.js

@@ -17,15 +17,18 @@ const validAcceptance = a => exact(a, ['minConfidence', 'minMargin']) && inRange
 const withAcceptance = (result, manifest) => (result && result.status === 'scored' && manifest.acceptance
   ? { ...result, acceptance: { ...manifest.acceptance } } : result);
 function validateManifest(m) {
-  const required = ['schemaVersion', 'python', 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
+  // The in-process ngram transport needs no Python and loads a .bin; the others run the CUA-S1 adapter.
+  const ngram = !!m && m.transport === 'ngram';
+  const required = ['schemaVersion', ...(ngram ? [] : ['python']), 'checkpoint', 'weightsSha256', 'configSha256', 'modelId', 'revision', 'domain'];
+  const strings = ['modelId', 'revision', 'domain', ...(ngram && !Object.hasOwn(m, 'python') ? [] : ['python'])];
   if (!m || typeof m !== 'object' || Array.isArray(m) || required.some(k => !Object.hasOwn(m, k))
-    || Object.keys(m).some(k => ![...required, 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance'].includes(k)) || m.schemaVersion !== 1
+    || Object.keys(m).some(k => ![...required, 'python', 'timeoutMs', 'transport', 'idleTimeoutMs', 'acceptance'].includes(k)) || m.schemaVersion !== 1
     || (m.acceptance !== undefined && !validAcceptance(m.acceptance))
-    || ['python', 'modelId', 'revision', 'domain'].some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
-    || typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== '.safetensors'
+    || strings.some(k => typeof m[k] !== 'string' || !m[k].trim() || m[k].length > 4096 || m[k].includes('\0'))
+    || typeof m.checkpoint !== 'string' || !path.isAbsolute(m.checkpoint) || path.extname(m.checkpoint) !== (ngram ? '.bin' : '.safetensors')
     || ['weightsSha256', 'configSha256'].some(k => typeof m[k] !== 'string' || !/^[a-f0-9]{64}$/.test(m[k]))
     || (m.timeoutMs !== undefined && (!Number.isInteger(m.timeoutMs) || m.timeoutMs < 1 || m.timeoutMs > 10000))
-    || (m.transport !== undefined && !['oneshot', 'resident'].includes(m.transport))
+    || (m.transport !== undefined && !['oneshot', 'resident', 'ngram'].includes(m.transport))
     || (m.idleTimeoutMs !== undefined && (!Number.isInteger(m.idleTimeoutMs) || m.idleTimeoutMs < 60000 || m.idleTimeoutMs > 14400000))
     || (m.modelId === 'cua-ai/cua-s1-forms' && m.domain !== 'forms-v1')) throw new TypeError('invalid-manifest');
   return true;
@@ -57,15 +60,22 @@ function score(request, manifestPath) {
   validateRequest(request);
   let manifest;
   try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
+  if (manifest.transport === 'ngram') return withAcceptance(inProcess(request, manifest), manifest);
   if (manifest.transport === 'resident') return withAcceptance(require('./resident').score({ ...request }, manifest, manifestPath), manifest);
   return withAcceptance(oneShot(request, manifest), manifest);
 }
+// The ngram transport scores in this process; an artifact mismatch is a configuration error, never a score.
+function inProcess(request, manifest) {
+  try { return { status: 'scored', response: require('./ngram').score({ ...request }, manifest) }; }
+  catch (_) { return unavailable('provider-config'); }
+}
 // Async callers (the hook entry) reach a resident server without the sync worker bridge.
 // One-shot has no server to overlap with, so it runs the same child synchronously.
 async function scoreAsync(request, manifestPath) {
   validateRequest(request);
   let manifest;
   try { manifest = readManifest(manifestPath); } catch (_) { return unavailable('provider-config'); }
+  if (manifest.transport === 'ngram') return withAcceptance(inProcess(request, manifest), manifest);
   if (manifest.transport === 'resident') return withAcceptance(await require('./resident').scoreAsync({ ...request }, manifest, manifestPath), manifest);
   return withAcceptance(oneShot(request, manifest), manifest);
 }