harness-everything 0.27.0 → 0.29.0

package/ci/mechanism-2j-behavioral-runner.test.js

@@ -2,7 +2,8 @@ const helper = require('./test-helper');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
-const { buildEngineInvocation, buildWorkspace } = require('../behavioral-evals/run');
+const { spawnSync } = require('child_process');
+const { buildEngineInvocation, buildWorkspace, runFixtureSetup, prepareWorkspace, summarizePairResults } = require('../behavioral-evals/run');
 
 console.log('\n[2j] Behavioral runner argv integrity...');
 const prompt = "Add punctuation stripping to slug.js, then run npm test. Do not truncate this request.";
@@ -59,4 +60,123 @@ helper.check(
   traversalError && traversalError.stack ? traversalError.stack : traversalTarget
 );
 
+let setupWorkspace = null;
+try {
+  setupWorkspace = buildWorkspace({
+    id: 'fixture-setup-valid',
+    fixture: {
+      files: [{ path: 'seed.txt', content: 'seed' }],
+      setup: 'behavioral-evals/fixtures/setup/noop.js',
+    },
+  });
+  const setupResult = runFixtureSetup({
+    id: 'fixture-setup-valid',
+    fixture: { setup: 'behavioral-evals/fixtures/setup/noop.js' },
+  }, setupWorkspace);
+  helper.check(
+    '2j. fixture setup runs via Node in the generated workspace',
+    setupResult.status === 'pass' &&
+      fs.readFileSync(path.join(setupWorkspace, '.fixture-setup-ok'), 'utf8') === 'ok\n',
+    JSON.stringify(setupResult)
+  );
+} catch (error) {
+  helper.check('2j. fixture setup runs via Node in the generated workspace', false, error && error.stack ? error.stack : String(error));
+} finally {
+  if (setupWorkspace) fs.rmSync(setupWorkspace, { recursive: true, force: true });
+}
+
+let failedPreparation = null;
+try {
+  failedPreparation = prepareWorkspace({
+    id: 'fixture-setup-fails',
+    fixture: {
+      files: [{ path: 'seed.txt', content: 'seed' }],
+      setup: 'behavioral-evals/fixtures/setup/fail.js',
+    },
+  });
+} catch (error) {
+  failedPreparation = { threw: error && error.stack ? error.stack : String(error) };
+}
+helper.check(
+  '2j. setup failure is classified as fixture-error before any model session',
+  Boolean(
+    failedPreparation &&
+    failedPreparation.ok === false &&
+    failedPreparation.outcome === 'fixture-error' &&
+    failedPreparation.setup &&
+    failedPreparation.setup.status === 'fail' &&
+    failedPreparation.setup.exit_code === 17
+  ),
+  JSON.stringify(failedPreparation)
+);
+if (failedPreparation && failedPreparation.workspace) {
+  fs.rmSync(failedPreparation.workspace, { recursive: true, force: true });
+}
+
+let topologyPreparation = null;
+try {
+  topologyPreparation = prepareWorkspace({
+    id: 'worktree-submodule-setup-smoke',
+    fixture: {
+      files: [{ path: 'README.md', content: 'topology smoke' }],
+      setup: 'behavioral-evals/fixtures/setup/worktree-submodule.js',
+    },
+  });
+  const linkedSub = topologyPreparation.ok
+    ? path.join(topologyPreparation.workspace, 'super-wt', 'libs', 'sub')
+    : null;
+  const branchResult = linkedSub
+    ? spawnSync('git', ['-C', linkedSub, 'branch', '--show-current'], { encoding: 'utf8', windowsHide: true })
+    : null;
+  const helperResult = topologyPreparation.ok
+    ? spawnSync(process.execPath, [
+        path.join(topologyPreparation.workspace, '.fixture-tools', 'submodule-reachability.js'),
+        '--json', '--root', 'super-wt',
+      ], { cwd: topologyPreparation.workspace, encoding: 'utf8', windowsHide: true })
+    : null;
+  helper.check(
+    '2j. worktree/submodule setup creates the real #243 topology before model execution',
+    Boolean(
+      topologyPreparation.ok &&
+      fs.existsSync(path.join(topologyPreparation.workspace, 'super-wt', '.git')) &&
+      fs.existsSync(path.join(linkedSub, '.git')) &&
+      branchResult && branchResult.status === 0 && branchResult.stdout.trim() === '' &&
+      helperResult && helperResult.status === 0
+    ),
+    JSON.stringify({
+      preparation: topologyPreparation,
+      branchStatus: branchResult && branchResult.status,
+      branch: branchResult && branchResult.stdout,
+      helperStatus: helperResult && helperResult.status,
+      helperStderr: helperResult && helperResult.stderr,
+    })
+  );
+} catch (error) {
+  helper.check(
+    '2j. worktree/submodule setup creates the real #243 topology before model execution',
+    false,
+    error && error.stack ? error.stack : String(error)
+  );
+} finally {
+  if (topologyPreparation && topologyPreparation.workspace) {
+    fs.rmSync(topologyPreparation.workspace, { recursive: true, force: true });
+  }
+}
+
+const fixtureFailureSummary = summarizePairResults([{
+  id: 'fixture-error-pair',
+  pressure_category: null,
+  arms: {
+    baseline: { outcome: 'fixture-error', cost: null, tool_call_count: null },
+    treatment: { outcome: 'pass', cost: null, tool_call_count: 1 },
+  },
+}]);
+helper.check(
+  '2j. fixture errors remain distinct aggregate infrastructure failures',
+  fixtureFailureSummary.fixture_failures === 1 &&
+    fixtureFailureSummary.session_failures === 0 &&
+    fixtureFailureSummary.completed_pairs === 0,
+  JSON.stringify(fixtureFailureSummary)
+);
+
 helper.finish();

package/ci/mechanism-2j-fixture-entrypoints.test.js

@@ -0,0 +1,153 @@
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { parseSimpleYaml, prepareWorkspace } = require('../behavioral-evals/run');
+const paired = require('../behavioral-evals/paired-benchmark');
+const ROOT = path.resolve(__dirname, '..');
+const topology = parseSimpleYaml(fs.readFileSync(path.join(ROOT, 'behavioral-evals/cases/baseline-worktree-submodule-reachability.yaml'), 'utf8'));
+const failure = { ...topology, fixture: { ...topology.fixture, setup: 'behavioral-evals/fixtures/setup/fail.js' } };
+const paths = [];
+const topologyFingerprints = [];
+const setupHashes = [];
+function track(ws) { if (ws) paths.push(ws); return ws; }
+function gitText(cwd, args) {
+  const result = spawnSync('git', args, { cwd, encoding: 'utf8', windowsHide: true });
+  assert.strictEqual(result.status, 0, result.stderr || result.stdout);
+  return result.stdout.trim();
+}
+function topologyFingerprint(ws) {
+  return {
+    child_head: gitText(path.join(ws, 'child-origin'), ['rev-parse', 'HEAD']),
+    super_head: gitText(path.join(ws, 'super'), ['rev-parse', 'HEAD']),
+    linked_head: gitText(path.join(ws, 'super-wt'), ['rev-parse', 'HEAD']),
+    gitlink: gitText(path.join(ws, 'super-wt'), ['rev-parse', 'HEAD:libs/sub']),
+  };
+}
+function assertTopology(ws) {
+  assert.ok(fs.existsSync(path.join(ws, 'super-wt', '.git')), 'setup must construct the linked worktree');
+  assert.ok(fs.existsSync(path.join(ws, '.fixture-tools', 'verify-worktree-submodule.js')), 'setup must install the grader');
+}
+try {
+  for (const arm of ['baseline', 'treatment']) {
+    for (const effect of ['skill-text', 'plugin-enforcement']) {
+      const prepared = paired.prepareArmWorkspace(topology, effect, arm, 'opencode');
+      track(prepared.ws);
+      assertTopology(prepared.ws);
+      assert.strictEqual(prepared.fixture_setup.status, 'pass');
+      assert.match(prepared.fixture_setup.script_sha256, /^[0-9a-f]{64}$/);
+      topologyFingerprints.push(topologyFingerprint(prepared.ws));
+      setupHashes.push(prepared.fixture_setup.script_sha256);
+      const failed = paired.prepareArmWorkspace(failure, effect, arm, 'opencode');
+      track(failed.ws);
+      assert.strictEqual(failed.ok, false);
+      assert.strictEqual(failed.outcome, 'fixture-error');
+      assert.strictEqual(failed.fixture_setup.exit_code, 17);
+      assert.ok(!fs.existsSync(path.join(failed.ws, '.claude')), 'setup failure must precede skill installation');
+      assert.ok(!fs.existsSync(path.join(failed.ws, '.opencode')), 'setup failure must precede plugin installation');
+    }
+  }
+  assert.ok(topologyFingerprints.length >= 4);
+  for (const fingerprint of topologyFingerprints.slice(1)) {
+    assert.deepStrictEqual(fingerprint, topologyFingerprints[0], 'fixture setup must produce identical Git object identities across independent arms/runs');
+  }
+  assert.strictEqual(new Set(setupHashes).size, 1, 'all arms must retain the same checked-in setup script hash');
+
+  const pairDir = track(fs.mkdtempSync(path.join(os.tmpdir(), 'fixture-entrypoints-')));
+  let launches = 0;
+  const context = { effect_type: 'skill-text', engine: 'claude', model: 'test' };
+  const arms = {};
+  for (const arm of ['baseline', 'treatment']) {
+    arms[arm] = paired.runArm(failure, context, arm, pairDir, () => { launches++; throw new Error('model must not start'); });
+    assert.strictEqual(arms[arm].outcome, 'fixture-error');
+    assert.strictEqual(arms[arm].fixture_setup.exit_code, 17);
+    assert.strictEqual(arms[arm].tool_call_count, null);
+  }
+  assert.strictEqual(launches, 0);
+  const summary = paired.summarizePairs([{ id: topology.id, arms }], 10);
+  assert.strictEqual(summary.fixture_failures, 1);
+  assert.strictEqual(summary.included_pairs, 0);
+  assert.deepStrictEqual(summary.arm_outcomes, { baseline: { pass: 0, fail: 0 }, treatment: { pass: 0, fail: 0 } });
+
+  const plugin = require('../behavioral-evals/run-with-plugin');
+  const failed = plugin.runCase(failure, () => { launches++; throw new Error('model must not start'); });
+  track(failed.workspace);
+  assert.strictEqual(failed.outcome, 'fixture-error');
+  assert.strictEqual(failed.fixture_setup.exit_code, 17);
+  assert.strictEqual(launches, 0);
+  assert.ok(!fs.existsSync(path.join(failed.workspace, '.harness-src')), 'failed setup must precede Harness installation');
+
+  // Exercise the same public case path as the plugin CLI, without calling a model.
+  const result = plugin.runCase(topology, (_prompt, ws) => {
+    launches++;
+    assertTopology(ws);
+    const transcript = path.join(pairDir, 'plugin-transcript.jsonl');
+    fs.writeFileSync(transcript, '');
+    return transcript;
+  });
+  track(result.workspace);
+  assert.strictEqual(launches, 1);
+  assert.strictEqual(result.fixture_setup.status, 'pass');
+  assert.strictEqual(result.outcome, 'fail', 'an untouched topology must fail the actual final-state graders');
+
+  for (const fixture of [
+    { files: [{ path: '../escape', content: 'bad' }] },
+    { files: [{ path: 'seed', content: 'ok' }], setup: '../escape.js' },
+  ]) {
+    const result = prepareWorkspace({ id: 'invalid-fixture', fixture });
+    track(result.workspace);
+    assert.strictEqual(result.outcome, 'fixture-error');
+  }
+  // The public CLIs must retain fixture-error evidence and exit nonzero.
+  const caseId = `fixture-entrypoints-failure-${process.pid}`;
+  const casePath = path.join(ROOT, 'behavioral-evals/cases', `${caseId}.yaml`);
+  fs.writeFileSync(casePath, `id: ${caseId}
+name: Fixture failure CLI control
+loaded_skills:
+  - git-commit
+fixture:
+  files:
+    - path: seed.txt
+      content: seed
+  setup: behavioral-evals/fixtures/setup/fail.js
+prompt: This model session must never start.
+max_turns: 1
+expectations:
+  - type: file_contains
+    path: seed.txt
+    value: seed
+`);
+  paths.push(casePath);
+  const pairedCli = spawnSync(process.execPath, [
+    path.join(ROOT, 'behavioral-evals/paired-benchmark.js'), 'run',
+    '--effect', 'skill-text', '--engine', 'claude', '--model', 'fixture-test',
+    '--min-effect-pp', '10', '--case', caseId,
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 30000 });
+  const summaryMatch = pairedCli.stdout.match(/Summary: (.+)/);
+  assert.ok(summaryMatch, pairedCli.stderr || pairedCli.stdout);
+  const summaryPath = summaryMatch[1].trim();
+  track(path.dirname(summaryPath));
+  assert.strictEqual(pairedCli.status, 1, pairedCli.stderr);
+  const cliSummary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
+  assert.strictEqual(cliSummary.fixture_failures, 1);
+  assert.strictEqual(cliSummary.included_pairs, 0);
+  const pluginCli = spawnSync(process.execPath, [
+    path.join(ROOT, 'behavioral-evals/run-with-plugin.js'), 'run', '--case', caseId,
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 30000 });
+  assert.strictEqual(pluginCli.status, 1, pluginCli.stderr);
+  const resultMatch = pluginCli.stdout.match(/fixture-error -> (.+)/);
+  assert.ok(resultMatch, pluginCli.stdout);
+  const resultPath = resultMatch[1].trim();
+  paths.push(resultPath);
+  const pluginFailure = JSON.parse(fs.readFileSync(resultPath, 'utf8'));
+  track(pluginFailure.workspace);
+  assert.strictEqual(pluginFailure.fixture_setup.exit_code, 17);
+  assert.ok(!fs.existsSync(path.join(pluginFailure.workspace, '.harness-src')));
+
+  console.log('Fixture entrypoints: paired baseline/treatment and plugin execute setup; failures skip installation/model and are excluded from scores.');
+} finally {
+  for (const dir of paths) fs.rmSync(dir, { recursive: true, force: true });
+}

package/ci/mechanism-system-one-evaluation.test.js

@@ -85,3 +85,20 @@ test('S1-E07 source provenance gate requires the pinned cua_s1 revision', () =>
   assert.equal(evaluate(make(), records(), recorded('pinned')).gates.sourceProvenance, true);
   assert.equal(evaluate(make(), records(), recorded('pinned')).rolloutReady, false);
 });
+test('S1-E09 an ngram source passes provenance only with a verified artifact', () => {
+  const ngramSource = verified => ({ status: 'recorded', transport: 'ngram', artifactVerified: verified });
+  assert.equal(evaluate(make(), records(), ngramSource(true)).gates.sourceProvenance, true);
+  assert.equal(evaluate(make(), records(), ngramSource(false)).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), { status: 'recorded', transport: 'ngram' }).gates.sourceProvenance, false);
+  assert.equal(evaluate(make(), records(), { status: 'unavailable', reason: 'provider-config', transport: 'ngram', artifactVerified: true }).gates.sourceProvenance, false);
+});
+test('S1-E08 the accepted-precision gate is the owner advisory 85%', () => {
+  // 7 accepted tier1 answers: 6 correct (85.7%) passes, 5 correct (71.4%) fails.
+  const cases = Array.from({ length: 7 }, (_, i) => ({ id: `p${i}`, family: `f${i}`, split: 'holdout', language: 'en', source: 'human:fixture', reviewed: false,
+    request: createRequest('tier', `bounded prompt ${i}`, TIER_OPTIONS), gold: 'tier1' }));
+  const corpus = { schemaVersion: 1, cases };
+  const recs = wrong => cases.map((c, i) => { const id = i < wrong ? 'tier2' : 'tier1';
+    return { id: c.id, baseline: 'tier2', runs: [0, 1].map(() => ({ decision: decision(id), scores: oneHot(id), latencyMs: 20, coldStart: true })) }; });
+  assert.equal(evaluate(corpus, recs(1)).gates.acceptedPrecision, true);
+  assert.equal(evaluate(corpus, recs(2)).gates.acceptedPrecision, false);
+});

package/ci/mechanism-system-one-label.test.js

@@ -61,6 +61,33 @@ test('S1-L03 invalid replies are retried once, persistent failures are recorded,
   assert.equal(again.labeled.length, 0);
 });
 
+test('S1-L06 the codex engine runs one stateless exec per batch and reads the schema-checked last message', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-codex-'));
+  try {
+    // Fake codex: records its argv, reads the prompt from stdin, writes the reply to the -o file.
+    const fake = path.join(dir, 'fake-codex.js'); const argvLog = path.join(dir, 'argv.json');
+    // Without an -o target the fake exits instead of writing (a[-1 + 1] would land in the working directory).
+    fs.writeFileSync(fake, `const fs=require('fs');const a=process.argv.slice(2);fs.writeFileSync(${JSON.stringify(argvLog)},JSON.stringify(a));
+      const o=a.indexOf('-o');if(o<0)process.exit(3);
+      let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{const m=[...s.matchAll(/\\{"i":(\\d+),"text"/g)];
+      fs.writeFileSync(a[o+1],JSON.stringify({labels:m.map(x=>({i:Number(x[1]),gold:'tier1'}))}));});`);
+    const input = path.join(dir, 'prompts.jsonl');
+    fs.writeFileSync(input, items(3).map(r => JSON.stringify({ ...r, family: 'f', source: 'claude', split: 'train' })).join('\n') + '\n');
+    const out = path.join(dir, 'labels.jsonl');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/system-one-label.js'), 'label', '--engine', 'codex', '--in', input, '--out', out],
+      { encoding: 'utf8', cwd: root, env: { ...process.env, HARNESS_S1_LABEL_COMMAND: JSON.stringify([process.execPath, fake]) } });
+    assert.equal(r.status, 0, r.stderr);
+    assert.deepEqual(JSON.parse(r.stdout), { labeled: 3, failed: 0, total: 3 });
+    const rows = fs.readFileSync(out, 'utf8').trim().split('\n').map(l => JSON.parse(l));
+    assert.ok(rows.every(x => x.gold === 'tier1' && x.labeler.model === 'gpt-5.6-luna' && x.labeler.engine === 'codex'));
+    const argv = JSON.parse(fs.readFileSync(argvLog, 'utf8'));
+    for (const flag of ['exec', '--ephemeral', '--ignore-user-config', '--output-schema', '-o']) assert.ok(argv.includes(flag), flag);
+    assert.equal(argv[argv.indexOf('--sandbox') + 1], 'read-only');
+    assert.equal(argv[argv.indexOf('--model') + 1], 'gpt-5.6-luna');
+    assert.equal(argv[argv.length - 1], '-', 'the prompt comes from stdin');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
 test('S1-L04 holdout agreement reports overall, per-class recall and confusion', () => {
   const gold = [['a', 'tier1'], ['b', 'tier1'], ['c', 'tier2'], ['d', null]];
   const pred = { a: 'tier1', b: 'tier2', c: 'tier2', d: null };

package/ci/mechanism-system-one-ngram.test.js

@@ -0,0 +1,161 @@
+'use strict';
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+const { spawnSync } = require('node:child_process');
+const ngram = require('../harness-everything/scripts/system-one/ngram');
+const provider = require('../harness-everything/scripts/system-one/provider');
+const { createRequest, decide } = require('../harness-everything/scripts/system-one/contract');
+const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const root = path.resolve(__dirname, '..');
+const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3');
+const sha = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+const IDS = TIER_OPTIONS.map(o => o.id);
+
+// A tiny model whose weights push any text containing the "commit" trigram toward tier1.
+function artifact(dir, { dim = 1024, nmax = 3, catalog = IDS, sidecar = {}, bytes } = {}) {
+  const W = new Float32Array(dim * catalog.length);
+  const hot = ngram.fnv1a32('3:com') % dim;
+  W[hot * catalog.length + 0] = 40;
+  const bin = path.join(dir, 'tier.bin'); const json = path.join(dir, 'tier.json');
+  fs.writeFileSync(bin, bytes || Buffer.from(W.buffer));
+  fs.writeFileSync(json, JSON.stringify({ format: 'harness-ngram', formatVersion: 1, dim, nmax, hash: 'fnv1a32', catalog, bias: catalog.map(() => 0), metadata: { fixture: true }, ...sidecar }));
+  return { bin, json };
+}
+function manifest(dir, files, extra = {}) {
+  const m = { schemaVersion: 1, transport: 'ngram', checkpoint: files.bin, weightsSha256: sha(files.bin), configSha256: sha(files.json),
+    modelId: 'fixture/ngram', revision: 'v1', domain: 'harness-routing-v1', ...extra };
+  const p = path.join(dir, 'manifest.json'); fs.writeFileSync(p, JSON.stringify(m)); return { m, p };
+}
+const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-ngram-'));
+
+test('S1-N01 fnv1a32 matches the published test vectors', () => {
+  assert.equal(ngram.fnv1a32(''), 0x811c9dc5);
+  assert.equal(ngram.fnv1a32('a'), 0xe40c292c);
+  assert.equal(ngram.fnv1a32('foobar'), 0xbf9cf968);
+});
+
+test('S1-N02 featurization follows the pinned rules', () => {
+  const f = ngram.featurize('  Commit   ALL\tchanges ', 3, 1 << 16);
+  assert.deepEqual(f, ngram.featurize('commit all changes', 3, 1 << 16), 'lowercase and whitespace normalization');
+  const norm = Math.sqrt([...f.values()].reduce((s, v) => s + v * v, 0));
+  assert.ok(Math.abs(norm - 1) < 1e-12);
+  assert.ok(f.has(ngram.fnv1a32('len:4') % (1 << 16)), 'length bucket for 18 UTF-8 bytes');
+  const cjk = ngram.featurize('修正😀', 2, 1 << 16);
+  assert.ok(cjk.has(ngram.fnv1a32('2:正😀') % (1 << 16)), 'n-grams are over code points, not UTF-16 units');
+  assert.ok(!ngram.featurize('a b', 3, 1 << 16).has(ngram.fnv1a32('1: ') % (1 << 16)), 'space-only grams are skipped');
+});
+
+test('S1-N03 the Python trainer and the Node provider produce identical features', () => {
+  const texts = ['commit all changes', '幫我修正 parser 的錯誤並補測試', '  MIXED Case\ttext 😀 ', 'x', '重構 plugin 介面，跨兩個 repo'];
+  const hashes = ['', 'a', 'foobar', '3:修正測'];
+  const r = spawnSync(python, [path.join(__dirname, 'system-one-ngram-parity.py')], { input: JSON.stringify({ texts, hashes, nmax: 3, dim: 1 << 16 }), encoding: 'utf8', env: { ...process.env, PYTHONDONTWRITEBYTECODE: '1', PYTHONIOENCODING: 'utf-8' } });
+  assert.equal(r.status, 0, r.stderr);
+  const py = JSON.parse(r.stdout);
+  assert.deepEqual(py.hash, hashes.map(h => ngram.fnv1a32(h)));
+  texts.forEach((t, i) => {
+    const node = ngram.featurize(t, 3, 1 << 16);
+    assert.deepEqual(Object.keys(py.features[i]).map(Number).sort((a, b) => a - b), [...node.keys()].sort((a, b) => a - b), t);
+    for (const [k, v] of Object.entries(py.features[i])) assert.ok(Math.abs(v - node.get(Number(k))) < 1e-12, t);
+  });
+});
+
+test('S1-N04 the ngram transport scores in process with the Phase 0 response shape', async () => {
+  const dir = tmp();
+  try {
+    const { m, p } = manifest(dir, artifact(dir), { acceptance: { minConfidence: 0.5, minMargin: 0 } });
+    assert.equal(provider.validateManifest(m), true);
+    const request = createRequest('tier', 'please commit these changes', TIER_OPTIONS);
+    const result = provider.score(request, p);
+    assert.equal(result.status, 'scored');
+    assert.deepEqual(result.acceptance, { minConfidence: 0.5, minMargin: 0 });
+    assert.deepEqual(result.response.model, { id: 'fixture/ngram', revision: 'v1', domain: 'harness-routing-v1' });
+    assert.ok(Math.abs(result.response.scores.reduce((s, x) => s + x.probability, 0) - 1) < 1e-9);
+    const d = decide(request, result.response, result.acceptance);
+    assert.equal(d.status, 'accepted'); assert.equal(d.selectedId, 'tier1');
+    assert.deepEqual(await provider.scoreAsync(request, p), result);
+    const other = provider.score(createRequest('tier', 'unrelated words only', TIER_OPTIONS), p);
+    assert.ok(other.response.scores.every(s => Math.abs(s.probability - 0.25) < 1e-9), 'no active weights gives a uniform vector');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N05 any artifact mismatch is provider-config and never scores', () => {
+  const request = createRequest('tier', 'commit it', TIER_OPTIONS);
+  const cases = [
+    dir => { const f = artifact(dir); const { m, p } = manifest(dir, f); fs.appendFileSync(f.bin, Buffer.alloc(4)); return p; },
+    dir => { const f = artifact(dir, { bytes: Buffer.alloc(8) }); return manifest(dir, f).p; },
+    dir => manifest(dir, artifact(dir, { catalog: ['tier1', 'tier2', 'tier3', 'other'] })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { format: 'other' } })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { extra: 1 } })).p,
+    dir => manifest(dir, artifact(dir, { dim: 1000 })).p,
+    dir => manifest(dir, artifact(dir, { nmax: 9 })).p,
+    dir => manifest(dir, artifact(dir, { sidecar: { bias: [0, 0] } })).p,
+  ];
+  for (const make of cases) {
+    const dir = tmp();
+    try { assert.deepEqual(provider.score(request, make(dir)), { status: 'unavailable', reason: 'provider-config' }); }
+    finally { fs.rmSync(dir, { recursive: true, force: true }); }
+  }
+});
+
+test('S1-N06 manifest rules: ngram needs a .bin and no python; other transports still need python', () => {
+  const dir = tmp();
+  try {
+    const { m } = manifest(dir, artifact(dir));
+    assert.equal(provider.validateManifest(m), true);
+    assert.throws(() => provider.validateManifest({ ...m, checkpoint: m.checkpoint.replace(/\.bin$/, '.safetensors') }));
+    assert.throws(() => provider.validateManifest({ ...m, transport: 'resident' }), 'resident requires python and a .safetensors checkpoint');
+    assert.throws(() => provider.validateManifest({ ...m, transport: 'bogus' }));
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N08 the evaluator records ngram samples warm and verifies the artifact as provenance', () => {
+  const dir = tmp();
+  try {
+    const { p } = manifest(dir, artifact(dir));
+    const output = path.join(dir, 'report.json');
+    const corpus = path.join(root, 'benchmarks/fixtures/system-one-routing.json');
+    const r = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, p, output], { encoding: 'utf8', cwd: root });
+    assert.equal(r.status, 0, r.stderr);
+    const report = JSON.parse(fs.readFileSync(output, 'utf8'));
+    assert.ok(report.records.every(x => x.runs.every(run => run.coldStart === false)));
+    assert.equal(report.latency.coldSamples, 0);
+    assert.equal(report.gates.warmLatency, true);
+    assert.deepEqual(report.evidence.source, { status: 'recorded', transport: 'ngram', artifactVerified: true });
+    assert.equal(report.gates.sourceProvenance, true);
+    assert.equal(report.evidence.kind, 'offline-ngram');
+    fs.appendFileSync(path.join(dir, 'tier.bin'), Buffer.alloc(4));
+    const tampered = spawnSync(process.execPath, [path.join(root, 'scripts/evaluate-system-one.js'), corpus, p, output], { encoding: 'utf8', cwd: root });
+    assert.equal(tampered.status, 0, tampered.stderr);
+    const bad = JSON.parse(fs.readFileSync(output, 'utf8'));
+    assert.equal(bad.evidence.source.artifactVerified, false);
+    assert.equal(bad.gates.sourceProvenance, false);
+    assert.equal(bad.model.coverage, 0);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('S1-N07 a real ngram training run produces an artifact the provider scores (skipped without torch)', t => {
+  if (spawnSync(python, ['-c', 'import torch'], { encoding: 'utf8' }).status !== 0) { t.skip('torch not installed'); return; }
+  const dir = tmp();
+  try {
+    const data = path.join(dir, 'data'); fs.mkdirSync(data);
+    const texts = [['commit all changes', 'tier1'], ['push and open a pr', 'tier1'], ['add a new flag with tests', 'tier3'], ['migrate every service', 'tier3'], ['fix the failing parser test', 'tier2'], ['go', null]];
+    const prompts = []; const labels = [];
+    for (let k = 0; k < 10; k++) texts.forEach(([x, g], i) => { const id = `p${k}-${i}`; prompts.push({ id, family: `f${k}`, source: 'claude', split: k < 8 ? 'train' : 'validation', text: `${x} ${k}` }); labels.push({ id, gold: g }); });
+    fs.writeFileSync(path.join(data, 'prompts.jsonl'), prompts.map(x => JSON.stringify(x)).join('\n') + '\n');
+    fs.writeFileSync(path.join(data, 'labels.jsonl'), labels.map(x => JSON.stringify(x)).join('\n') + '\n');
+    const out = path.join(dir, 'model');
+    const r = spawnSync(python, [path.join(root, 'scripts/system-one-train-ngram.py'), '--data-dir', data, '--out', out, '--dim', '4096', '--epochs', '40'], { encoding: 'utf8' });
+    assert.equal(r.status, 0, r.stdout + r.stderr);
+    const files = { bin: path.join(out, 'harness-routing-v1.bin'), json: path.join(out, 'harness-routing-v1.json') };
+    assert.equal(fs.statSync(files.bin).size, 4096 * 4 * 4);
+    assert.equal(fs.readFileSync(path.join(out, 'val-scores.jsonl'), 'utf8').trim().split('\n').length, 12);
+    const { p } = manifest(dir, files);
+    const scored = provider.score(createRequest('tier', 'commit all changes now', TIER_OPTIONS), p);
+    assert.equal(scored.status, 'scored');
+    assert.equal(decide(createRequest('tier', 'commit all changes now', TIER_OPTIONS), scored.response, { minConfidence: 0.5, minMargin: 0 }).selectedId, 'tier1');
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});

package/ci/mechanism-system-one-train.test.js

@@ -41,6 +41,18 @@ test('S1-T03 calibration maximizes coverage subject to accepted precision on val
   assert.deepEqual(calibrate(rows, { minPrecision: 0.98 }), c, 'deterministic');
 });
 
+test('S1-T06 the default calibration target is the owner advisory 85%', () => {
+  const rows = [
+    ...Array.from({ length: 86 }, () => row('tier1', [0.9, 0.05, 0.03, 0.02])),
+    ...Array.from({ length: 14 }, () => row('tier2', [0.9, 0.05, 0.03, 0.02])),
+  ];
+  const c = calibrate(rows);
+  assert.equal(c.minPrecision, 0.85);
+  assert.equal(c.feasible, true);
+  assert.equal(c.coverage, 1);
+  assert.equal(calibrate(rows, { minPrecision: 0.9 }).feasible, false);
+});
+
 test('S1-T04 calibration CLI reads validation scores and writes thresholds', () => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harness-s1-cal-'));
   try {

package/ci/system-one-ngram-parity.py

@@ -0,0 +1,19 @@
+"""Print the trainer's n-gram features for texts on stdin (JSON list) so the Node provider can be compared exactly."""
+import importlib.util
+import json
+from pathlib import Path
+import sys
+
+sys.dont_write_bytecode = True
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('ngram_trainer', ROOT / 'scripts' / 'system-one-train-ngram.py')
+trainer = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(trainer)
+assert 'torch' not in sys.modules, 'importing the ngram trainer must not import torch'
+
+request = json.loads(sys.stdin.read())
+out = []
+for text in request['texts']:
+    feats = trainer.featurize(text, request['nmax'], request['dim'])
+    out.append({str(k): v for k, v in sorted(feats.items())})
+print(json.dumps({'hash': [trainer.fnv1a32(s) for s in request['hashes']], 'features': out}))

package/ci/system-one-train-test.py

@@ -46,6 +46,12 @@ assert flat == list(range(len(lengths))), 'every example exactly once'
 assert all(len(b) * max(lengths[i] for i in b) <= 4096 or len(b) == 1 for b in batches), 'budget respected'
 assert batches == trainer.make_batches(lengths, token_budget=4096), 'deterministic'
 
+weights = trainer.class_weights([0, 0, 0, 1, 2, 2], 4)
+assert len(weights) == 4
+assert weights[3] == 0.0, 'an absent class gets no weight'
+assert weights[1] > weights[2] > weights[0], weights
+assert abs(sum(w * n for w, n in zip(weights, [3, 1, 2, 0])) - 6) < 1e-9, 'weights average to 1 over the examples'
+
 with tempfile.TemporaryDirectory() as tmp:
     p = Path(tmp) / 'rows.jsonl'
     p.write_text('\n'.join(json.dumps(r) for r in prompts) + '\n', encoding='utf-8')

package/ci/test-behavioral-case-validation.js

@@ -21,6 +21,32 @@ assert.deepStrictEqual(validateCase({
   expectations: [{ type: 'tool_call', value: 'Bash' }]
 }), [], 'tool_call must remain part of the runner/validator schema');
 
+const setupCase = setup => ({
+  id: 'fixture-setup-schema',
+  prompt: 'inspect the prepared fixture',
+  max_turns: 1,
+  fixture: {
+    files: [{ path: 'README.md', content: 'fixture' }],
+    setup,
+  },
+  expectations: [{ type: 'trace_contains', value: 'fixture' }],
+});
+assert.deepStrictEqual(
+  validateCase(setupCase('behavioral-evals/fixtures/setup/noop.js')),
+  [],
+  'checked-in repository-relative .js setup should be valid'
+);
+for (const [setup, expected] of [
+  ['../escape.js', /fixture\.setup/],
+  ['/tmp/escape.js', /fixture\.setup/],
+  ['behavioral-evals/fixtures/setup/noop.txt', /\.js/],
+  ['behavioral-evals/fixtures/setup/missing.js', /missing|checked-in/],
+  [{ command: 'echo unsafe' }, /fixture\.setup/],
+]) {
+  const setupErrors = validateCase(setupCase(setup));
+  assert.ok(setupErrors.some(error => expected.test(error)), `setup ${JSON.stringify(setup)} should be rejected: ${setupErrors.join('; ')}`);
+}
+
 const transcriptDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-case-validation-'));
 try {
   const transcriptPath = path.join(transcriptDir, 'trace.jsonl');

package/docs/system-one-corpus.md

@@ -16,7 +16,7 @@ to, judged from the prompt alone?** The labels follow the tier model in
 | --- | --- | --- |
 | `tier1` | Something answered or done without changing project behavior: Git/GitHub housekeeping (commit, push, open a PR, sync, tag, post given results as a comment), a status check, reading a log or searching and reporting, the smallest doc fix (a typo, a link, an ignore entry), or a reply the assistant can give directly, including non-engineering chores and discussion questions | "commit these changes", "is issue 88 still open?", "find every repo still using the old logging package", "close the other apps" |
 | `tier2` | A bounded change or investigation: any code, configuration or CI edit (a rename, a value, a revert), running tests or a build, filing an issue, explaining code that must be read first, most bug fixes, a focused review or benchmark, planning or synthesis that produces documents (phase plans, an ADR, an overview), and answers that settle open decisions for work in progress | "rename tmp to buffer", "run npm test", "the CLI fails with this error, fix it", "1. agree 2. keep the old name" |
-| `tier3` | New or changed behavior delivered with tests, and anything larger: a new flag or format, a behavior change across producers and tests, a large refactor, a dependency upgrade with breakage, a migration, a repository-wide change, a multi-phase implementation, or delegation to several agents | "add a --verbose flag with tests", "migrate to ESM", "dispatch agents to audit every repo" |
+| `tier3` | Work that needs several agents (sub-agents included); a new module or subsystem, a refactor, or a change to a framework or interface definition; or code or design work that spans several repositories or several workflows | "add a plugin system to the CLI", "refactor the parser into modules", "change the plugin interface", "dispatch agents to audit every repo" |
 | `null` | No actionable content: a bare continuation, "do what you said", a vague reaction, a question about what the assistant meant, or pasted status with no request | "go", "yes", "照剛剛說的改", "something is off here" |
 
 Rules:
@@ -31,6 +31,30 @@ Rules:
    reformat or extend the assistant's previous answer without changing files is
    `tier1`. A requested change to code or configuration, however small, is at
    least `tier2`. Only the smallest doc fixes listed in the table stay `tier1`.
+   The line between `tier2` and `tier3` is the owner's: `tier3` when the work
+   needs several agents or sub-agents, adds a feature, refactors, changes a
+   framework or interface definition, or spans several repositories or
+   workflows. Fixing, adjusting, investigating or reviewing existing behavior
+   within one workflow is `tier2`.
+   The owner settled four more lines from the third spot-check:
+   - **Git is `tier1` at any width.** Committing sub-repositories and then the
+     main repository is still housekeeping. "Several repositories" applies to
+     code or design work only.
+   - **Scope, not the word "add".** Adding an option, a function or a parser
+     inside one existing tool or component is `tier2`. A new module or
+     subsystem, or an addition that spans several components, is `tier3`.
+     A bug fix, even with a regression test, is `tier2`.
+   - **Repository-wide or cross-component work is `tier3`.** Replacing or
+     unifying a pattern across the repository (every `console.log`, all error
+     handling, every path in a report), upgrading a dependency and fixing what
+     breaks, a data migration, a CI matrix across platforms, or restructuring a
+     documentation tree is `tier3`, even when each edit is small.
+   - **A short reply follows the work in progress.** A reply that asks to
+     change an output (code, a document, an example) is `tier2`. Only a pure
+     question or a bare acknowledgement stays `tier1`.
+   - **Explicit agents or architecture make `tier3`.** A request for several
+     agents, a team discussion or a verifier pass, or one that redefines an
+     architecture, is `tier3` even when it is phrased as discussion.
 4. Explicit workflow words ("use fable", "no subagents") do not change the gold.
    Precedence is a router policy, not a label.
 5. Never derive gold from the lexical router or any model output. A reviewer may
@@ -102,6 +126,10 @@ a decision on its own.
 | `benchmarks/fixtures/system-one-holdout.json` | Evaluation corpus, rebuilt with `node scripts/system-one-corpus.js build <draft> <reviews> <corpus>` |
 
 The reviewed holdout has 215 cases (11 rejected): en 93 and zh-TW 122; gold
-`tier1` 47, `tier2` 97, `tier3` 51 and `null` 20. It meets the `reviewedHoldout`
-gate. It says nothing about model quality until a `harness-routing-v1`
+`tier1` 48, `tier2` 104, `tier3` 43 and `null` 20. It meets the `reviewedHoldout`
+gate. After the third training spot-check the owner moved nine cases to the new
+lines: `h-long-git-ops-1` to `tier1`, and the `h-add-cli-flag-*`,
+`h-hash-algorithm-support-1/2`, `h-fix-cli-error-log-3/4` and
+`h-fix-failing-test-3` cases to `tier2`. Results recorded before that change
+used 47 `tier1`, 97 `tier2` and 51 `tier3`. It says nothing about model quality until a `harness-routing-v1`
 checkpoint is evaluated on it.