harness-everything 0.29.0 → 0.30.1

package/plugins/harness-everything/skills/harness-everything/scripts/system-one/resident.js

@@ -10,7 +10,12 @@ const { spawn } = require('node:child_process');
 const { Worker } = require('node:worker_threads');
 const { validateRequest } = require('./contract');
 
-const ADAPTER = path.join(__dirname, 'cua_adapter.py');
+const ADAPTER_PATTERN = /^[a-z][a-z0-9_-]{0,63}\.py$/;
+const adapterPath = manifest => {
+  const name = manifest.adapter || 'cua_adapter.py';
+  if (!ADAPTER_PATTERN.test(name)) throw new TypeError('invalid-adapter');
+  return path.join(__dirname, name);
+};
 const LOCK_TTL_MS = 60000;
 const CALL_TIMEOUT_MS = 1000;
 const MAX_REPLY_BYTES = 1024 * 1024;
@@ -18,7 +23,7 @@ const unavailable = reason => ({ status: 'unavailable', reason });
 const sleep = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 const exact = (value, keys) => value && typeof value === 'object' && !Array.isArray(value)
   && Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k));
-const defaultCommand = manifest => [manifest.python, [ADAPTER, '--serve']];
+const defaultCommand = manifest => [manifest.python, [adapterPath(manifest), '--serve']];
 
 function stateFiles(manifest, manifestPath) {
   const digest = createHash('sha256').update(JSON.stringify(manifest)).digest('hex').slice(0, 16);
@@ -197,4 +202,4 @@ if (require.main === module) {
   if (op === 'stop') stop(manifest, manifestPath);
   process.stdout.write(`${JSON.stringify(status(manifest, manifestPath))}\n`);
 }
-module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, LOCK_TTL_MS, CALL_TIMEOUT_MS };
+module.exports = { stateFiles, readState, callSync, callAsync, score, scoreAsync, ensureReady, status, stop, defaultCommand, LOCK_TTL_MS, CALL_TIMEOUT_MS };

package/plugins/harness-everything/skills/harness-everything/scripts/system-one/router.js

@@ -26,7 +26,7 @@ function selectTier(input, env = process.env, scorer = provider.score) {
     const allowed = ['provider-timeout', 'provider-output-limit', 'provider-exit', 'provider-json', 'provider-config', 'provider-unavailable', 'provider-starting'];
     return fallback(allowed.includes(result?.reason) ? result.reason : 'provider-unavailable');
   }
-  const decision = decide(request, result.response, result.acceptance ? { ...result.acceptance } : {});
+  const decision = decide(request, result.response, provider.decisionPolicy(result.acceptance));
   if (decision.selectedId === 'unclassified') Object.assign(decision, { selectedId: null, status: 'abstain', reason: 'unclassified' });
   const accepted = decision.status === 'accepted';
   const belowFloor = accepted && input.floor != null && RANK[decision.selectedId] < RANK[input.floor];

package/sbom.cdx.json

@@ -2,23 +2,23 @@
   "$schema": "https://cyclonedx.org/schema/bom-1.7.schema.json",
   "bomFormat": "CycloneDX",
   "specVersion": "1.7",
-  "serialNumber": "urn:uuid:03c78df6-955e-5903-b3d9-620c6be8e2d4",
+  "serialNumber": "urn:uuid:4d14e861-0454-56ef-a99b-c25f67dc9e83",
   "version": 1,
   "metadata": {
-    "timestamp": "2026-09-28T01:49:53.842Z",
+    "timestamp": "2026-10-01T02:23:01.933Z",
     "tools": [
       {
         "vendor": "Harness Everything",
         "name": "generate-sbom.js",
-        "version": "0.29.0"
+        "version": "0.30.1"
       }
     ],
     "component": {
       "type": "application",
-      "bom-ref": "pkg:npm/harness-everything@0.29.0",
+      "bom-ref": "pkg:npm/harness-everything@0.30.1",
       "name": "harness-everything",
-      "version": "0.29.0",
-      "purl": "pkg:npm/harness-everything@0.29.0"
+      "version": "0.30.1",
+      "purl": "pkg:npm/harness-everything@0.30.1"
     }
   },
   "components": [
@@ -1999,7 +1999,7 @@
       ]
     },
     {
-      "ref": "pkg:npm/harness-everything@0.29.0",
+      "ref": "pkg:npm/harness-everything@0.30.1",
       "dependsOn": [
         "pkg:npm/%40commitlint/cli@19.8.1",
         "pkg:npm/%40commitlint/config-conventional@19.8.1",

package/scripts/evaluate-system-one.js

@@ -4,7 +4,8 @@ const fs = require('node:fs');
 const os = require('node:os');
 const { performance } = require('node:perf_hooks');
 const { validateCorpus, evaluate } = require('../harness-everything/scripts/system-one/evaluate');
-const { scoreAsync, provenance, readManifest } = require('../harness-everything/scripts/system-one/provider');
+const { scoreAsync, provenance, readManifest, decisionPolicy } = require('../harness-everything/scripts/system-one/provider');
+const { CATALOGS, MULTI } = require('../harness-everything/scripts/system-one/catalogs');
 const resident = require('../harness-everything/scripts/system-one/resident');
 const { decide } = require('../harness-everything/scripts/system-one/contract');
 const { run: route } = require('../harness-everything/scripts/tier-router');
@@ -22,6 +23,15 @@ async function run(corpus, manifest) {
   const ready = isResident && resident.ensureReady(config, manifest, 60000);
   let artifactVerified = false;
   if (isNgram) { try { require('../harness-everything/scripts/system-one/ngram').loadModel(config); artifactVerified = true; } catch (_) { /* recorded below */ } }
+  const task = corpus.cases[0].request.task;
+  // The lexical router only knows tiers; other stages compare against the majority gold (ties by catalog order).
+  let majority = null;
+  if (task !== 'tier') {
+    const ids = CATALOGS[task].map(o => o.id);
+    const count = id => corpus.cases.filter(c => c.split === 'holdout' && (c.gold === null ? 'unclassified' : c.gold) === id).length;
+    const top = ids.reduce((best, id) => (count(id) > count(best) ? id : best), ids[0]);
+    majority = top === 'unclassified' ? null : top;
+  }
   const records = [];
   const originalMode = process.env.HARNESS_SYSTEM_ONE_MODE;
   const originalLog = console.log;
@@ -29,17 +39,17 @@ async function run(corpus, manifest) {
     process.env.HARNESS_SYSTEM_ONE_MODE = 'off';
     console.log = () => {};
     for (const c of corpus.cases.filter(item => item.split === 'holdout')) {
-      const tier = route(c.request.context).workflowPlan.tier;
+      const tier = task === 'tier' ? route(c.request.context).workflowPlan.tier : null;
       const runs = [];
       for (let i = 0; i < 2; i++) {
         const start = performance.now();
         const result = await scoreAsync(c.request, manifest);
-        const decision = result.status === 'scored' ? decide(c.request, result.response, result.acceptance ? { ...result.acceptance } : {}) : result;
+        const decision = result.status === 'scored' ? decide(c.request, result.response, decisionPolicy(result.acceptance)) : result;
         if (decision.selectedId === 'unclassified') Object.assign(decision, { status: 'abstain', reason: 'unclassified', selectedId: null });
         const scores = ['accepted', 'abstain'].includes(decision.status) ? result.response.scores.map(s => s.probability) : null;
         runs.push({ decision, scores, latencyMs: performance.now() - start, coldStart: !warm });
       }
-      records.push({ id: c.id, baseline: tier === 'unclassified' ? null : tier, runs });
+      records.push({ id: c.id, baseline: task === 'tier' ? (tier === 'unclassified' ? null : tier) : majority, runs });
     }
   } finally {
     if (isResident && !wasRunning) resident.stop(config, manifest);
@@ -55,7 +65,9 @@ async function run(corpus, manifest) {
   // The ngram transport has no Python package to probe; its provenance is the hash-verified artifact.
   const source = isNgram ? { status: 'recorded', transport: 'ngram', artifactVerified } : provenance(manifest);
   const kind = isNgram ? 'offline-ngram' : isResident ? 'offline-resident' : 'offline-one-shot';
-  return { ...evaluate(corpus, records, source), evidence: { kind, artifact, source,
+  const secondaryThreshold = MULTI.includes(task) ? config?.acceptance?.secondaryThreshold : undefined;
+  return { ...evaluate(corpus, records, source, { secondaryThreshold }), evidence: { kind, artifact, source,
+    ...(MULTI.includes(task) ? { secondaryThreshold } : {}),
     residentReady: isResident ? ready : null,
     environment: { cpu: os.cpus()[0]?.model || 'unknown', os: `${os.platform()} ${os.release()} ${os.arch()}`, node: process.version,
       python: source.status === 'recorded' && source.provenance ? source.provenance.python : null, threads: 1 },

package/scripts/system-one-calibrate-laya.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Calibrate Laya zero-shot scores to teacher relevance (#255 Phase 3 prep).
+
+Per-intent isotonic (PAVA, monotone non-decreasing) mapping from Laya
+zero-shot scores to the sonnet teacher's scaled relevance, plus a null-cutoff
+sweep. Honest estimate via K-fold CV with FIXED derive bands; the shipped
+artifact (knots + cutoff) is fit on all overlap rows and clearly labeled
+in-sample. Stdlib only; no model is loaded -- this runs in seconds.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve()
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+_compare_spec = importlib.util.spec_from_file_location(
+    'laya_compare', HERE.parent / 'system-one-compare-laya.py')
+_compare = importlib.util.module_from_spec(_compare_spec)
+_compare_spec.loader.exec_module(_compare)
+graded = _compare.graded
+
+
+def isotonic_fit(xs, ys):
+    """PAVA isotonic regression. Returns knots as [[x, y], ...] sorted by x."""
+    by_x = {}
+    for x, y in zip(xs, ys):
+        by_x.setdefault(float(x), []).append(float(y))
+    points = sorted((x, sum(v) / len(v)) for x, v in by_x.items())
+    blocks = [[x, y, 1] for x, y in points]  # x, mean, weight
+    i = 0
+    while i < len(blocks) - 1:
+        if blocks[i][1] <= blocks[i + 1][1]:
+            i += 1
+            continue
+        x0, m0, w0 = blocks[i]
+        x1, m1, w1 = blocks[i + 1]
+        blocks[i] = [(x0 * w0 + x1 * w1) / (w0 + w1), (m0 * w0 + m1 * w1) / (w0 + w1), w0 + w1]
+        del blocks[i + 1]
+        if i > 0:
+            i -= 1
+    return [[b[0], b[1]] for b in blocks]
+
+
+def isotonic_predict(knots, x):
+    """Piecewise-linear interpolation over knots, clamped outside the range."""
+    x = float(x)
+    if x <= knots[0][0]:
+        return knots[0][1]
+    if x >= knots[-1][0]:
+        return knots[-1][1]
+    for (x0, y0), (x1, y1) in zip(knots, knots[1:]):
+        if x0 <= x <= x1:
+            t = (x - x0) / (x1 - x0) if x1 > x0 else 0.0
+            return y0 + t * (y1 - y0)
+    return knots[-1][1]
+
+
+def apply_mapping(raw_scores, mappings):
+    """Map raw per-intent scores through per-intent knots (identity when absent)."""
+    out = {}
+    for intent in INTENTS:
+        knots = mappings.get(intent)
+        out[intent] = isotonic_predict(knots, raw_scores[intent]) if knots else float(raw_scores[intent])
+    return out
+
+
+def derive_calibrated(raw_scores, mappings, null_cutoff):
+    """Bands on calibrated scores: abstain when the max is at/below the cutoff."""
+    scores = apply_mapping(raw_scores, mappings)
+    top = max(scores[intent] for intent in INTENTS)
+    if top <= null_cutoff:
+        return {'gold': None, 'secondary': []}
+    gold = max(INTENTS, key=lambda i: (scores[i], -INTENTS.index(i)))
+    secondary = sorted((i for i in INTENTS if i != gold and scores[i] >= 0.4),
+                       key=lambda i: (-scores[i], INTENTS.index(i)))
+    return {'gold': gold, 'secondary': secondary}
+
+
+def fold_of(idx, k):
+    return idx % k
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--cand', required=True, help='candidate labels file')
+    parser.add_argument('--out', required=True, help='calibration artifact json path')
+    parser.add_argument('--folds', type=int, default=5)
+    args = parser.parse_args(argv)
+    data = Path(args.data_dir)
+    teacher = {r['id']: r for r in map(json.loads, (data / args.teacher).read_text(encoding='utf-8').splitlines()) if True}
+    cand = {r['id']: r for r in map(json.loads, (data / args.cand).read_text(encoding='utf-8').splitlines()) if True}
+    ids = sorted(set(teacher) & set(cand))
+    n = len(ids)
+
+    def fit_mappings(train_ids):
+        mappings = {}
+        for intent in INTENTS:
+            xs = [cand[i]['scores'][intent] for i in train_ids]
+            ys = [teacher[i]['scores'][intent] for i in train_ids]
+            mappings[intent] = isotonic_fit(xs, ys)
+        return mappings
+
+    # Honest CV with fixed bands (null iff calibrated max <= 0.2).
+    cv_exact, cv_graded, cv_mae = [], [], []
+    for f in range(args.folds):
+        held = [i for idx, i in enumerate(ids) if fold_of(idx, args.folds) == f]
+        train = [i for idx, i in enumerate(ids) if fold_of(idx, args.folds) != f]
+        mappings = fit_mappings(train)
+        exact = sum(1 for i in held
+                    if derive_calibrated(cand[i]['scores'], mappings, 0.2)['gold'] == teacher[i]['gold'])
+        grades = [graded(derive_calibrated(cand[i]['scores'], mappings, 0.2)['gold'],
+                         derive_calibrated(cand[i]['scores'], mappings, 0.2)['secondary'],
+                         teacher[i]['gold'], teacher[i].get('secondary', [])) for i in held]
+        mae = sum(abs(apply_mapping(cand[i]['scores'], mappings)[k] - teacher[i]['scores'][k])
+                  for i in held for k in INTENTS) / (len(held) * len(INTENTS))
+        cv_exact.append(exact / len(held))
+        cv_graded.append(sum(grades) / len(grades))
+        cv_mae.append(mae)
+
+    # Full-fit artifact + null-cutoff sweep (in-sample, labeled as such).
+    full = fit_mappings(ids)
+    sweep = {}
+    for step in range(0, 75, 5):
+        cutoff = step / 100
+        grades = [graded(*_derive_pair(cand[i]['scores'], full, cutoff, teacher[i])) for i in ids]
+        sweep[f'{cutoff:.2f}'] = sum(grades) / len(grades)
+    best_cutoff = max(sweep, key=lambda c: (sweep[c], -float(c)))
+    mae_full = sum(abs(apply_mapping(cand[i]['scores'], full)[k] - teacher[i]['scores'][k])
+                   for i in ids for k in INTENTS) / (n * len(INTENTS))
+    raw_mae = sum(abs(cand[i]['scores'][k] - teacher[i]['scores'][k])
+                  for i in ids for k in INTENTS) / (n * len(INTENTS))
+
+    artifact = {'candidate': args.cand,
+                'candidateSha256': hashlib.sha256((data / args.cand).read_bytes()).hexdigest(),
+                'teacher': args.teacher,
+                'teacherSha256': hashlib.sha256((data / args.teacher).read_bytes()).hexdigest(),
+                'n': n, 'folds': args.folds,
+                'cv': {'exactMean': sum(cv_exact) / len(cv_exact), 'gradedMean': sum(cv_graded) / len(cv_graded),
+                       'maeMean': sum(cv_mae) / len(cv_mae)},
+                'inSample': {'maeRaw': raw_mae, 'maeCalibrated': mae_full,
+                             'nullCutoffSweep': sweep, 'bestCutoff': best_cutoff},
+                'mappings': full}
+    Path(args.out).write_text(json.dumps(artifact, indent=2), encoding='utf-8')
+    print(json.dumps({'n': n, 'cvExact': round(artifact['cv']['exactMean'], 3),
+                      'cvGraded': round(artifact['cv']['gradedMean'], 3),
+                      'cvMAE': round(artifact['cv']['maeMean'], 4),
+                      'rawMAE': round(raw_mae, 4), 'bestCutoff': best_cutoff,
+                      'bestSweepGraded': round(sweep[best_cutoff], 3)}))
+    return 0
+
+
+def _derive_pair(raw_scores, mappings, cutoff, teacher_row):
+    d = derive_calibrated(raw_scores, mappings, cutoff)
+    return d['gold'], d['secondary'], teacher_row['gold'], teacher_row.get('secondary', [])
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-calibrate.js

@@ -3,40 +3,69 @@
 // Chooses System One acceptance thresholds on validation scores (docs/system-one-training.md).
 // The holdout is never used here.
 const fs = require('node:fs');
+const { CATALOGS } = require('../harness-everything/scripts/system-one/catalogs');
+const { gradedAgreement } = require('../harness-everything/scripts/system-one/intent');
 
 const IDS = ['tier1', 'tier2', 'tier3', 'unclassified'];
 // Same acceptance rule as contract.decide: top probability and top-minus-second margin, unclassified abstains.
-function decideRow(probs, { minConfidence, minMargin }) {
+function decideRow(probs, { minConfidence, minMargin }, ids = IDS) {
   const order = probs.map((p, i) => [p, i]).sort((a, b) => b[0] - a[0] || a[1] - b[1]);
   const [top, second] = order;
   const margin = top[0] - second[0];
-  if (margin === 0 || top[0] < minConfidence || margin < minMargin || IDS[top[1]] === 'unclassified') return null;
-  return IDS[top[1]];
+  if (margin === 0 || top[0] < minConfidence || margin < minMargin || ids[top[1]] === 'unclassified') return null;
+  return ids[top[1]];
 }
-function calibrate(rows, { minPrecision = 0.85 } = {}) {
+// Predicted secondary intents: every other catalog intent at or above the threshold.
+const secondaryOf = (probs, primary, threshold, ids) => ids.filter((id, i) => id !== primary && id !== 'unclassified' && probs[i] >= threshold);
+// Threshold with the best secondary micro-F1 on validation; ties keep the lowest.
+function secondaryThreshold(rows, ids) {
+  let best = null;
+  for (let s = 5; s <= 50; s += 5) {
+    let tp = 0; let fp = 0; let fn = 0;
+    for (const r of rows) {
+      const primary = ids[r.probs.indexOf(Math.max(...r.probs))];
+      const predicted = new Set(secondaryOf(r.probs, primary, s / 100, ids)); const gold = new Set(r.secondary || []);
+      for (const x of predicted) (gold.has(x) ? tp++ : fp++);
+      for (const x of gold) if (!predicted.has(x)) fn++;
+    }
+    const f1 = 2 * tp + fp + fn ? 2 * tp / (2 * tp + fp + fn) : 0;
+    if (!best || f1 > best.f1) best = { threshold: s / 100, f1 };
+  }
+  return best.threshold;
+}
+function calibrate(rows, { minPrecision = 0.85, task = 'tier' } = {}) {
+  const ids = CATALOGS[task].map(o => o.id);
+  const multi = task === 'intent';
+  const sThreshold = multi ? secondaryThreshold(rows, ids) : undefined;
+  // Intent rows score by graded agreement against { gold, secondary }; tier rows by exact match.
+  const score = (r, d) => (multi ? gradedAgreement({ gold: r.gold, secondary: r.secondary || [] }, { gold: d, secondary: secondaryOf(r.probs, d, sThreshold, ids) })
+    : Number(d === (r.gold === null ? 'unclassified' : r.gold)));
   let best = null;
   for (let c = 50; c <= 99; c++) {
     for (let m = 0; m <= 90; m += 5) {
       const t = { minConfidence: c / 100, minMargin: m / 100 };
-      let accepted = 0; let correct = 0;
-      for (const r of rows) { const d = decideRow(r.probs, t); if (d) { accepted++; if (d === (r.gold === null ? 'unclassified' : r.gold)) correct++; } }
-      const precision = accepted ? correct / accepted : 0;
+      let accepted = 0; let points = 0; let exact = 0;
+      for (const r of rows) { const d = decideRow(r.probs, t, ids); if (d) { accepted++; points += score(r, d); if (d === r.gold) exact++; } }
+      const precision = accepted ? points / accepted : 0;
       if (!accepted || precision < minPrecision) continue;
       const coverage = accepted / rows.length;
       // Highest coverage wins; ties keep the first (lowest) thresholds scanned.
-      if (!best || coverage > best.coverage) best = { ...t, coverage, precision, accepted };
+      if (!best || coverage > best.coverage) best = { ...t, coverage, precision, accepted, ...(multi ? { exactPrecision: exact / accepted } : {}) };
     }
   }
-  return best ? { feasible: true, rows: rows.length, minPrecision, ...best }
-    : { feasible: false, rows: rows.length, minPrecision, minConfidence: null, minMargin: null, coverage: 0, precision: null, accepted: 0 };
+  const extra = multi ? { task, secondaryThreshold: sThreshold } : {};
+  return best ? { feasible: true, rows: rows.length, minPrecision, ...extra, ...best }
+    : { feasible: false, rows: rows.length, minPrecision, ...extra, minConfidence: null, minMargin: null, coverage: 0, precision: null, accepted: 0 };
 }
 
 if (require.main === module) {
   try {
-    const [input, output, extra] = process.argv.slice(2);
-    if (!input || !output || extra) throw new Error('usage: system-one-calibrate.js <val-scores.jsonl> <calibration.json>');
+    const args = process.argv.slice(2);
+    const ti = args.indexOf('--task'); const task = ti >= 0 ? args.splice(ti, 2)[1] : 'tier';
+    const [input, output, extra] = args;
+    if (!input || !output || extra || !Object.hasOwn(CATALOGS, task)) throw new Error('usage: system-one-calibrate.js <val-scores.jsonl> <calibration.json> [--task tier|intent]');
     const rows = fs.readFileSync(input, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
-    const result = calibrate(rows);
+    const result = calibrate(rows, { task });
     fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`);
     console.log(JSON.stringify(result));
   } catch (err) { console.error(`system-one-calibrate: ${err.message}`); process.exitCode = 1; }

package/scripts/system-one-collect.js

@@ -176,6 +176,9 @@ function build({ home, appdata, holdoutTexts }) {
     families: new Set(rows.map(r => r.family)).size, zhShare: rows.length ? +(rows.filter(r => /[一-鿿]/.test(r.text)).length / rows.length).toFixed(3) : 0 } };
 }
 
+// Every committed holdout; a training prompt must not near-duplicate any of them.
+const defaultHoldouts = () => ['system-one-holdout.json', 'system-one-intent-holdout.json'].map(f => path.join(REPO, 'benchmarks', 'fixtures', f));
+
 if (require.main === module) {
   const args = process.argv.slice(2);
   const opt = name => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
@@ -183,8 +186,9 @@ if (require.main === module) {
     const out = path.resolve(opt('--out') || path.join(os.homedir(), '.agents', 'harness-everything', 'system-one', 'training'));
     const rel = path.relative(REPO, out);
     if (!rel.startsWith('..') && !path.isAbsolute(rel)) throw new Error('refusing to write the dataset inside the repository');
-    const holdoutFile = opt('--holdout') || path.join(REPO, 'benchmarks', 'fixtures', 'system-one-holdout.json');
-    const holdoutTexts = JSON.parse(fs.readFileSync(holdoutFile, 'utf8')).cases.map(c => c.request.context);
+    const given = args.flatMap((a, i) => (a === '--holdout' && args[i + 1] ? [args[i + 1]] : []));
+    const holdoutTexts = (given.length ? given : defaultHoldouts())
+      .flatMap(file => JSON.parse(fs.readFileSync(file, 'utf8')).cases.map(c => c.request.context));
     const { rows, stats } = build({ home: opt('--home') || os.homedir(), appdata: opt('--appdata') || process.env.APPDATA || '', holdoutTexts });
     fs.mkdirSync(out, { recursive: true });
     const body = rows.map(r => JSON.stringify(r)).join('\n') + '\n';
@@ -197,4 +201,4 @@ if (require.main === module) {
     process.exitCode = 1;
   }
 }
-module.exports = { readSources, keep, redact, dedupe, leakGuard, jaccard, splitFor, build };
+module.exports = { readSources, keep, redact, dedupe, leakGuard, jaccard, splitFor, build, defaultHoldouts };

package/scripts/system-one-compare-laya.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""Compare Laya zero-shot dense labels against the teacher scores (#255 Phase 2 evidence).
+
+Reads the private training dir only; the printed/written report holds aggregate
+metrics, hashes and score distributions -- never prompt text -- so it can join
+the public evidence bundle. Graded agreement mirrors
+harness-everything/scripts/system-one/intent.js gradedAgreement; --self-check
+asserts the mirror against the owner's S1-I05 grade table on every run.
+"""
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+# (gold_a, sec_a, gold_b, sec_b, want) from ci/mechanism-system-one-intent.test.js S1-I05.
+GRADE_TABLE = [
+    (('fix', ['test']), ('fix', ['docs', 'review']), 1),
+    (('fix', ['test']), ('test', ['fix']), 0.6),
+    (('fix', ['test', 'docs', 'review', 'plan']), ('test', ['fix']), 0.3),
+    (('fix', ['test', 'docs', 'review']), ('test', ['fix']), 0.6),
+    (('fix', ['test']), ('test', []), 0.3),
+    (('fix', []), ('docs', ['review']), 0),
+    ((None, []), (None, []), 1),
+    ((None, []), ('fix', []), 0),
+]
+
+
+def graded(a_gold, a_sec, b_gold, b_sec):
+    """Mirror of intent.js gradedAgreement."""
+    if a_gold is None or b_gold is None:
+        return 1 if (a_gold is None and b_gold is None) else 0
+    fa = {a_gold, *a_sec}
+    fb = {b_gold, *b_sec}
+    if a_gold == b_gold:
+        return 1
+    if a_gold in fb and b_gold in fa:
+        return 0.6 if abs(len(a_sec) - len(b_sec)) < 3 else 0.3
+    if a_gold in fb or b_gold in fa:
+        return 0.3
+    return 0
+
+
+def self_check():
+    for (ag, ase), (bg, bse), want in GRADE_TABLE:
+        got = graded(ag, ase, bg, bse)
+        assert got == want, f'graded mirror drift: {(ag, ase, bg, bse)} -> {got}, want {want}'
+        assert graded(bg, bse, ag, ase) == want, 'graded mirror must be symmetric'
+
+
+def is_zh(text):
+    return any('\u4e00' <= ch <= '\u9fff' for ch in text)
+
+
+def spearman(xs, ys):
+    n = len(xs)
+    if n < 2:
+        return None
+    rx = sorted(range(n), key=lambda i: xs[i])
+    ry = sorted(range(n), key=lambda i: ys[i])
+    rank_x = [0] * n
+    rank_y = [0] * n
+    for r, i in enumerate(rx):
+        rank_x[i] = r
+    for r, i in enumerate(ry):
+        rank_y[i] = r
+    mean = (n - 1) / 2
+    cov = sum((rank_x[i] - mean) * (rank_y[i] - mean) for i in range(n))
+    var = sum((rank_x[i] - mean) ** 2 for i in range(n))
+    return cov / var if var else None
+
+
+def load_jsonl(path):
+    return [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()]
+
+
+def main(argv=None):
+    self_check()
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--cand', nargs='+', required=True, help='candidate labels files to compare')
+    parser.add_argument('--out', required=True, help='report json path')
+    args = parser.parse_args(argv)
+    data = Path(args.data_dir)
+    prompts = {r['id']: r['text'] for r in load_jsonl(data / 'prompts.jsonl')}
+    teacher = {r['id']: r for r in load_jsonl(data / args.teacher)}
+    report = {'teacher': args.teacher,
+              'teacherSha256': hashlib.sha256((data / args.teacher).read_bytes()).hexdigest(),
+              'candidates': {}}
+    cands = {name: {r['id']: r for r in load_jsonl(data / name)} for name in args.cand}
+    common = set(teacher) & set(prompts)
+    for name in args.cand:
+        common &= set(cands[name])
+    common = sorted(common)
+    report['compared'] = len(common)
+    for name in args.cand:
+        rows = cands[name]
+        exact = sum(1 for i in common if rows[i]['gold'] == teacher[i]['gold'])
+        grades = [graded(rows[i]['gold'], rows[i].get('secondary', []),
+                         teacher[i]['gold'], teacher[i].get('secondary', [])) for i in common]
+        scored = [rows[i]['scores'] for i in common]
+        dist = {intent: sum(s[intent] for s in scored) / len(scored) for intent in INTENTS}
+        top_mean = sum(max(s.values()) for s in scored) / len(scored)
+        rank_corr = {intent: spearman([rows[i]['scores'][intent] for i in common],
+                                      [teacher[i]['scores'][intent] for i in common]) for intent in INTENTS}
+        slices = {}
+        for lang, pred in (('zh', is_zh), ('en', lambda t: not is_zh(t))):
+            ids = [i for i in common if pred(prompts[i])]
+            if ids:
+                slices[lang] = {'n': len(ids),
+                                'exact': sum(1 for i in ids if rows[i]['gold'] == teacher[i]['gold']) / len(ids),
+                                'graded': sum(graded(rows[i]['gold'], rows[i].get('secondary', []),
+                                                     teacher[i]['gold'], teacher[i].get('secondary', [])) for i in ids) / len(ids)}
+        report['candidates'][name] = {
+            'sha256': hashlib.sha256((data / name).read_bytes()).hexdigest(),
+            'n': len(common), 'primaryExact': exact / len(common),
+            'gradedMean': sum(grades) / len(grades),
+            'meanTopScore': top_mean, 'meanScoreByIntent': dist,
+            'rankCorrByIntent': rank_corr,
+            'nullRate': sum(1 for i in common if rows[i]['gold'] is None) / len(common),
+            'teacherNullRate': sum(1 for i in common if teacher[i]['gold'] is None) / len(common),
+            'slices': slices,
+            'routing': {},
+        }
+        for i in common:
+            routed = (rows[i].get('labeler') or {}).get('routing', 'unknown')
+            report['candidates'][name]['routing'][routed] = \
+                report['candidates'][name]['routing'].get(routed, 0) + 1
+    # Backend parity over shared ids.
+    if len(args.cand) == 2:
+        a, b = (cands[args.cand[0]], cands[args.cand[1]])
+        same_route = sum(1 for i in common
+                         if (a[i].get('labeler') or {}).get('routing') == (b[i].get('labeler') or {}).get('routing'))
+        same_gold = sum(1 for i in common if a[i]['gold'] == b[i]['gold'])
+        deltas = [max(abs(a[i]['scores'][k] - b[i]['scores'][k]) for k in INTENTS) for i in common]
+        report['parity'] = {'pair': list(args.cand), 'n': len(common),
+                            'routingAgreement': same_route / len(common),
+                            'derivedGoldAgreement': same_gold / len(common),
+                            'maxAbsDeltaMean': sum(deltas) / len(deltas),
+                            'maxAbsDeltaP95': sorted(deltas)[int(len(deltas) * 0.95)]}
+    Path(args.out).write_text(json.dumps(report, indent=2), encoding='utf-8')
+    print(json.dumps({k: v for k, v in report.items() if k != 'candidates'}, indent=2))
+    for name, c in report['candidates'].items():
+        print(f"{name}: exact={c['primaryExact']:.3f} graded={c['gradedMean']:.3f} "
+              f"top={c['meanTopScore']:.3f} null={c['nullRate']:.3f} routing={c['routing']} slices={c['slices']}")
+    if 'parity' in report:
+        print('parity:', json.dumps(report['parity']))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())