harness-everything 0.29.0 → 0.30.0

package/scripts/system-one-calibrate-laya.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Calibrate Laya zero-shot scores to teacher relevance (#255 Phase 3 prep).
+
+Per-intent isotonic (PAVA, monotone non-decreasing) mapping from Laya
+zero-shot scores to the sonnet teacher's scaled relevance, plus a null-cutoff
+sweep. Honest estimate via K-fold CV with FIXED derive bands; the shipped
+artifact (knots + cutoff) is fit on all overlap rows and clearly labeled
+in-sample. Stdlib only; no model is loaded -- this runs in seconds.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve()
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+_compare_spec = importlib.util.spec_from_file_location(
+    'laya_compare', HERE.parent / 'system-one-compare-laya.py')
+_compare = importlib.util.module_from_spec(_compare_spec)
+_compare_spec.loader.exec_module(_compare)
+graded = _compare.graded
+
+
+def isotonic_fit(xs, ys):
+    """PAVA isotonic regression. Returns knots as [[x, y], ...] sorted by x."""
+    by_x = {}
+    for x, y in zip(xs, ys):
+        by_x.setdefault(float(x), []).append(float(y))
+    points = sorted((x, sum(v) / len(v)) for x, v in by_x.items())
+    blocks = [[x, y, 1] for x, y in points]  # x, mean, weight
+    i = 0
+    while i < len(blocks) - 1:
+        if blocks[i][1] <= blocks[i + 1][1]:
+            i += 1
+            continue
+        x0, m0, w0 = blocks[i]
+        x1, m1, w1 = blocks[i + 1]
+        blocks[i] = [(x0 * w0 + x1 * w1) / (w0 + w1), (m0 * w0 + m1 * w1) / (w0 + w1), w0 + w1]
+        del blocks[i + 1]
+        if i > 0:
+            i -= 1
+    return [[b[0], b[1]] for b in blocks]
+
+
+def isotonic_predict(knots, x):
+    """Piecewise-linear interpolation over knots, clamped outside the range."""
+    x = float(x)
+    if x <= knots[0][0]:
+        return knots[0][1]
+    if x >= knots[-1][0]:
+        return knots[-1][1]
+    for (x0, y0), (x1, y1) in zip(knots, knots[1:]):
+        if x0 <= x <= x1:
+            t = (x - x0) / (x1 - x0) if x1 > x0 else 0.0
+            return y0 + t * (y1 - y0)
+    return knots[-1][1]
+
+
+def apply_mapping(raw_scores, mappings):
+    """Map raw per-intent scores through per-intent knots (identity when absent)."""
+    out = {}
+    for intent in INTENTS:
+        knots = mappings.get(intent)
+        out[intent] = isotonic_predict(knots, raw_scores[intent]) if knots else float(raw_scores[intent])
+    return out
+
+
+def derive_calibrated(raw_scores, mappings, null_cutoff):
+    """Bands on calibrated scores: abstain when the max is at/below the cutoff."""
+    scores = apply_mapping(raw_scores, mappings)
+    top = max(scores[intent] for intent in INTENTS)
+    if top <= null_cutoff:
+        return {'gold': None, 'secondary': []}
+    gold = max(INTENTS, key=lambda i: (scores[i], -INTENTS.index(i)))
+    secondary = sorted((i for i in INTENTS if i != gold and scores[i] >= 0.4),
+                       key=lambda i: (-scores[i], INTENTS.index(i)))
+    return {'gold': gold, 'secondary': secondary}
+
+
+def fold_of(idx, k):
+    return idx % k
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--cand', required=True, help='candidate labels file')
+    parser.add_argument('--out', required=True, help='calibration artifact json path')
+    parser.add_argument('--folds', type=int, default=5)
+    args = parser.parse_args(argv)
+    data = Path(args.data_dir)
+    teacher = {r['id']: r for r in map(json.loads, (data / args.teacher).read_text(encoding='utf-8').splitlines()) if True}
+    cand = {r['id']: r for r in map(json.loads, (data / args.cand).read_text(encoding='utf-8').splitlines()) if True}
+    ids = sorted(set(teacher) & set(cand))
+    n = len(ids)
+
+    def fit_mappings(train_ids):
+        mappings = {}
+        for intent in INTENTS:
+            xs = [cand[i]['scores'][intent] for i in train_ids]
+            ys = [teacher[i]['scores'][intent] for i in train_ids]
+            mappings[intent] = isotonic_fit(xs, ys)
+        return mappings
+
+    # Honest CV with fixed bands (null iff calibrated max <= 0.2).
+    cv_exact, cv_graded, cv_mae = [], [], []
+    for f in range(args.folds):
+        held = [i for idx, i in enumerate(ids) if fold_of(idx, args.folds) == f]
+        train = [i for idx, i in enumerate(ids) if fold_of(idx, args.folds) != f]
+        mappings = fit_mappings(train)
+        exact = sum(1 for i in held
+                    if derive_calibrated(cand[i]['scores'], mappings, 0.2)['gold'] == teacher[i]['gold'])
+        grades = [graded(derive_calibrated(cand[i]['scores'], mappings, 0.2)['gold'],
+                         derive_calibrated(cand[i]['scores'], mappings, 0.2)['secondary'],
+                         teacher[i]['gold'], teacher[i].get('secondary', [])) for i in held]
+        mae = sum(abs(apply_mapping(cand[i]['scores'], mappings)[k] - teacher[i]['scores'][k])
+                  for i in held for k in INTENTS) / (len(held) * len(INTENTS))
+        cv_exact.append(exact / len(held))
+        cv_graded.append(sum(grades) / len(grades))
+        cv_mae.append(mae)
+
+    # Full-fit artifact + null-cutoff sweep (in-sample, labeled as such).
+    full = fit_mappings(ids)
+    sweep = {}
+    for step in range(0, 75, 5):
+        cutoff = step / 100
+        grades = [graded(*_derive_pair(cand[i]['scores'], full, cutoff, teacher[i])) for i in ids]
+        sweep[f'{cutoff:.2f}'] = sum(grades) / len(grades)
+    best_cutoff = max(sweep, key=lambda c: (sweep[c], -float(c)))
+    mae_full = sum(abs(apply_mapping(cand[i]['scores'], full)[k] - teacher[i]['scores'][k])
+                   for i in ids for k in INTENTS) / (n * len(INTENTS))
+    raw_mae = sum(abs(cand[i]['scores'][k] - teacher[i]['scores'][k])
+                  for i in ids for k in INTENTS) / (n * len(INTENTS))
+
+    artifact = {'candidate': args.cand,
+                'candidateSha256': hashlib.sha256((data / args.cand).read_bytes()).hexdigest(),
+                'teacher': args.teacher,
+                'teacherSha256': hashlib.sha256((data / args.teacher).read_bytes()).hexdigest(),
+                'n': n, 'folds': args.folds,
+                'cv': {'exactMean': sum(cv_exact) / len(cv_exact), 'gradedMean': sum(cv_graded) / len(cv_graded),
+                       'maeMean': sum(cv_mae) / len(cv_mae)},
+                'inSample': {'maeRaw': raw_mae, 'maeCalibrated': mae_full,
+                             'nullCutoffSweep': sweep, 'bestCutoff': best_cutoff},
+                'mappings': full}
+    Path(args.out).write_text(json.dumps(artifact, indent=2), encoding='utf-8')
+    print(json.dumps({'n': n, 'cvExact': round(artifact['cv']['exactMean'], 3),
+                      'cvGraded': round(artifact['cv']['gradedMean'], 3),
+                      'cvMAE': round(artifact['cv']['maeMean'], 4),
+                      'rawMAE': round(raw_mae, 4), 'bestCutoff': best_cutoff,
+                      'bestSweepGraded': round(sweep[best_cutoff], 3)}))
+    return 0
+
+
+def _derive_pair(raw_scores, mappings, cutoff, teacher_row):
+    d = derive_calibrated(raw_scores, mappings, cutoff)
+    return d['gold'], d['secondary'], teacher_row['gold'], teacher_row.get('secondary', [])
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-calibrate.js

@@ -3,40 +3,69 @@
 // Chooses System One acceptance thresholds on validation scores (docs/system-one-training.md).
 // The holdout is never used here.
 const fs = require('node:fs');
+const { CATALOGS } = require('../harness-everything/scripts/system-one/catalogs');
+const { gradedAgreement } = require('../harness-everything/scripts/system-one/intent');
 
 const IDS = ['tier1', 'tier2', 'tier3', 'unclassified'];
 // Same acceptance rule as contract.decide: top probability and top-minus-second margin, unclassified abstains.
-function decideRow(probs, { minConfidence, minMargin }) {
+function decideRow(probs, { minConfidence, minMargin }, ids = IDS) {
   const order = probs.map((p, i) => [p, i]).sort((a, b) => b[0] - a[0] || a[1] - b[1]);
   const [top, second] = order;
   const margin = top[0] - second[0];
-  if (margin === 0 || top[0] < minConfidence || margin < minMargin || IDS[top[1]] === 'unclassified') return null;
-  return IDS[top[1]];
+  if (margin === 0 || top[0] < minConfidence || margin < minMargin || ids[top[1]] === 'unclassified') return null;
+  return ids[top[1]];
 }
-function calibrate(rows, { minPrecision = 0.85 } = {}) {
+// Predicted secondary intents: every other catalog intent at or above the threshold.
+const secondaryOf = (probs, primary, threshold, ids) => ids.filter((id, i) => id !== primary && id !== 'unclassified' && probs[i] >= threshold);
+// Threshold with the best secondary micro-F1 on validation; ties keep the lowest.
+function secondaryThreshold(rows, ids) {
+  let best = null;
+  for (let s = 5; s <= 50; s += 5) {
+    let tp = 0; let fp = 0; let fn = 0;
+    for (const r of rows) {
+      const primary = ids[r.probs.indexOf(Math.max(...r.probs))];
+      const predicted = new Set(secondaryOf(r.probs, primary, s / 100, ids)); const gold = new Set(r.secondary || []);
+      for (const x of predicted) (gold.has(x) ? tp++ : fp++);
+      for (const x of gold) if (!predicted.has(x)) fn++;
+    }
+    const f1 = 2 * tp + fp + fn ? 2 * tp / (2 * tp + fp + fn) : 0;
+    if (!best || f1 > best.f1) best = { threshold: s / 100, f1 };
+  }
+  return best.threshold;
+}
+function calibrate(rows, { minPrecision = 0.85, task = 'tier' } = {}) {
+  const ids = CATALOGS[task].map(o => o.id);
+  const multi = task === 'intent';
+  const sThreshold = multi ? secondaryThreshold(rows, ids) : undefined;
+  // Intent rows score by graded agreement against { gold, secondary }; tier rows by exact match.
+  const score = (r, d) => (multi ? gradedAgreement({ gold: r.gold, secondary: r.secondary || [] }, { gold: d, secondary: secondaryOf(r.probs, d, sThreshold, ids) })
+    : Number(d === (r.gold === null ? 'unclassified' : r.gold)));
   let best = null;
   for (let c = 50; c <= 99; c++) {
     for (let m = 0; m <= 90; m += 5) {
       const t = { minConfidence: c / 100, minMargin: m / 100 };
-      let accepted = 0; let correct = 0;
-      for (const r of rows) { const d = decideRow(r.probs, t); if (d) { accepted++; if (d === (r.gold === null ? 'unclassified' : r.gold)) correct++; } }
-      const precision = accepted ? correct / accepted : 0;
+      let accepted = 0; let points = 0; let exact = 0;
+      for (const r of rows) { const d = decideRow(r.probs, t, ids); if (d) { accepted++; points += score(r, d); if (d === r.gold) exact++; } }
+      const precision = accepted ? points / accepted : 0;
       if (!accepted || precision < minPrecision) continue;
       const coverage = accepted / rows.length;
       // Highest coverage wins; ties keep the first (lowest) thresholds scanned.
-      if (!best || coverage > best.coverage) best = { ...t, coverage, precision, accepted };
+      if (!best || coverage > best.coverage) best = { ...t, coverage, precision, accepted, ...(multi ? { exactPrecision: exact / accepted } : {}) };
     }
   }
-  return best ? { feasible: true, rows: rows.length, minPrecision, ...best }
-    : { feasible: false, rows: rows.length, minPrecision, minConfidence: null, minMargin: null, coverage: 0, precision: null, accepted: 0 };
+  const extra = multi ? { task, secondaryThreshold: sThreshold } : {};
+  return best ? { feasible: true, rows: rows.length, minPrecision, ...extra, ...best }
+    : { feasible: false, rows: rows.length, minPrecision, ...extra, minConfidence: null, minMargin: null, coverage: 0, precision: null, accepted: 0 };
 }
 
 if (require.main === module) {
   try {
-    const [input, output, extra] = process.argv.slice(2);
-    if (!input || !output || extra) throw new Error('usage: system-one-calibrate.js <val-scores.jsonl> <calibration.json>');
+    const args = process.argv.slice(2);
+    const ti = args.indexOf('--task'); const task = ti >= 0 ? args.splice(ti, 2)[1] : 'tier';
+    const [input, output, extra] = args;
+    if (!input || !output || extra || !Object.hasOwn(CATALOGS, task)) throw new Error('usage: system-one-calibrate.js <val-scores.jsonl> <calibration.json> [--task tier|intent]');
     const rows = fs.readFileSync(input, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
-    const result = calibrate(rows);
+    const result = calibrate(rows, { task });
     fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`);
     console.log(JSON.stringify(result));
   } catch (err) { console.error(`system-one-calibrate: ${err.message}`); process.exitCode = 1; }

package/scripts/system-one-collect.js

@@ -176,6 +176,9 @@ function build({ home, appdata, holdoutTexts }) {
     families: new Set(rows.map(r => r.family)).size, zhShare: rows.length ? +(rows.filter(r => /[一-鿿]/.test(r.text)).length / rows.length).toFixed(3) : 0 } };
 }
 
+// Every committed holdout; a training prompt must not near-duplicate any of them.
+const defaultHoldouts = () => ['system-one-holdout.json', 'system-one-intent-holdout.json'].map(f => path.join(REPO, 'benchmarks', 'fixtures', f));
+
 if (require.main === module) {
   const args = process.argv.slice(2);
   const opt = name => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
@@ -183,8 +186,9 @@ if (require.main === module) {
     const out = path.resolve(opt('--out') || path.join(os.homedir(), '.agents', 'harness-everything', 'system-one', 'training'));
     const rel = path.relative(REPO, out);
     if (!rel.startsWith('..') && !path.isAbsolute(rel)) throw new Error('refusing to write the dataset inside the repository');
-    const holdoutFile = opt('--holdout') || path.join(REPO, 'benchmarks', 'fixtures', 'system-one-holdout.json');
-    const holdoutTexts = JSON.parse(fs.readFileSync(holdoutFile, 'utf8')).cases.map(c => c.request.context);
+    const given = args.flatMap((a, i) => (a === '--holdout' && args[i + 1] ? [args[i + 1]] : []));
+    const holdoutTexts = (given.length ? given : defaultHoldouts())
+      .flatMap(file => JSON.parse(fs.readFileSync(file, 'utf8')).cases.map(c => c.request.context));
     const { rows, stats } = build({ home: opt('--home') || os.homedir(), appdata: opt('--appdata') || process.env.APPDATA || '', holdoutTexts });
     fs.mkdirSync(out, { recursive: true });
     const body = rows.map(r => JSON.stringify(r)).join('\n') + '\n';
@@ -197,4 +201,4 @@ if (require.main === module) {
     process.exitCode = 1;
   }
 }
-module.exports = { readSources, keep, redact, dedupe, leakGuard, jaccard, splitFor, build };
+module.exports = { readSources, keep, redact, dedupe, leakGuard, jaccard, splitFor, build, defaultHoldouts };

package/scripts/system-one-compare-laya.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""Compare Laya zero-shot dense labels against the teacher scores (#255 Phase 2 evidence).
+
+Reads the private training dir only; the printed/written report holds aggregate
+metrics, hashes and score distributions -- never prompt text -- so it can join
+the public evidence bundle. Graded agreement mirrors
+harness-everything/scripts/system-one/intent.js gradedAgreement; --self-check
+asserts the mirror against the owner's S1-I05 grade table on every run.
+"""
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+# (gold_a, sec_a, gold_b, sec_b, want) from ci/mechanism-system-one-intent.test.js S1-I05.
+GRADE_TABLE = [
+    (('fix', ['test']), ('fix', ['docs', 'review']), 1),
+    (('fix', ['test']), ('test', ['fix']), 0.6),
+    (('fix', ['test', 'docs', 'review', 'plan']), ('test', ['fix']), 0.3),
+    (('fix', ['test', 'docs', 'review']), ('test', ['fix']), 0.6),
+    (('fix', ['test']), ('test', []), 0.3),
+    (('fix', []), ('docs', ['review']), 0),
+    ((None, []), (None, []), 1),
+    ((None, []), ('fix', []), 0),
+]
+
+
+def graded(a_gold, a_sec, b_gold, b_sec):
+    """Mirror of intent.js gradedAgreement."""
+    if a_gold is None or b_gold is None:
+        return 1 if (a_gold is None and b_gold is None) else 0
+    fa = {a_gold, *a_sec}
+    fb = {b_gold, *b_sec}
+    if a_gold == b_gold:
+        return 1
+    if a_gold in fb and b_gold in fa:
+        return 0.6 if abs(len(a_sec) - len(b_sec)) < 3 else 0.3
+    if a_gold in fb or b_gold in fa:
+        return 0.3
+    return 0
+
+
+def self_check():
+    for (ag, ase), (bg, bse), want in GRADE_TABLE:
+        got = graded(ag, ase, bg, bse)
+        assert got == want, f'graded mirror drift: {(ag, ase, bg, bse)} -> {got}, want {want}'
+        assert graded(bg, bse, ag, ase) == want, 'graded mirror must be symmetric'
+
+
+def is_zh(text):
+    return any('\u4e00' <= ch <= '\u9fff' for ch in text)
+
+
+def spearman(xs, ys):
+    n = len(xs)
+    if n < 2:
+        return None
+    rx = sorted(range(n), key=lambda i: xs[i])
+    ry = sorted(range(n), key=lambda i: ys[i])
+    rank_x = [0] * n
+    rank_y = [0] * n
+    for r, i in enumerate(rx):
+        rank_x[i] = r
+    for r, i in enumerate(ry):
+        rank_y[i] = r
+    mean = (n - 1) / 2
+    cov = sum((rank_x[i] - mean) * (rank_y[i] - mean) for i in range(n))
+    var = sum((rank_x[i] - mean) ** 2 for i in range(n))
+    return cov / var if var else None
+
+
+def load_jsonl(path):
+    return [json.loads(line) for line in Path(path).read_text(encoding='utf-8').splitlines() if line.strip()]
+
+
+def main(argv=None):
+    self_check()
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True)
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--cand', nargs='+', required=True, help='candidate labels files to compare')
+    parser.add_argument('--out', required=True, help='report json path')
+    args = parser.parse_args(argv)
+    data = Path(args.data_dir)
+    prompts = {r['id']: r['text'] for r in load_jsonl(data / 'prompts.jsonl')}
+    teacher = {r['id']: r for r in load_jsonl(data / args.teacher)}
+    report = {'teacher': args.teacher,
+              'teacherSha256': hashlib.sha256((data / args.teacher).read_bytes()).hexdigest(),
+              'candidates': {}}
+    cands = {name: {r['id']: r for r in load_jsonl(data / name)} for name in args.cand}
+    common = set(teacher) & set(prompts)
+    for name in args.cand:
+        common &= set(cands[name])
+    common = sorted(common)
+    report['compared'] = len(common)
+    for name in args.cand:
+        rows = cands[name]
+        exact = sum(1 for i in common if rows[i]['gold'] == teacher[i]['gold'])
+        grades = [graded(rows[i]['gold'], rows[i].get('secondary', []),
+                         teacher[i]['gold'], teacher[i].get('secondary', [])) for i in common]
+        scored = [rows[i]['scores'] for i in common]
+        dist = {intent: sum(s[intent] for s in scored) / len(scored) for intent in INTENTS}
+        top_mean = sum(max(s.values()) for s in scored) / len(scored)
+        rank_corr = {intent: spearman([rows[i]['scores'][intent] for i in common],
+                                      [teacher[i]['scores'][intent] for i in common]) for intent in INTENTS}
+        slices = {}
+        for lang, pred in (('zh', is_zh), ('en', lambda t: not is_zh(t))):
+            ids = [i for i in common if pred(prompts[i])]
+            if ids:
+                slices[lang] = {'n': len(ids),
+                                'exact': sum(1 for i in ids if rows[i]['gold'] == teacher[i]['gold']) / len(ids),
+                                'graded': sum(graded(rows[i]['gold'], rows[i].get('secondary', []),
+                                                     teacher[i]['gold'], teacher[i].get('secondary', [])) for i in ids) / len(ids)}
+        report['candidates'][name] = {
+            'sha256': hashlib.sha256((data / name).read_bytes()).hexdigest(),
+            'n': len(common), 'primaryExact': exact / len(common),
+            'gradedMean': sum(grades) / len(grades),
+            'meanTopScore': top_mean, 'meanScoreByIntent': dist,
+            'rankCorrByIntent': rank_corr,
+            'nullRate': sum(1 for i in common if rows[i]['gold'] is None) / len(common),
+            'teacherNullRate': sum(1 for i in common if teacher[i]['gold'] is None) / len(common),
+            'slices': slices,
+            'routing': {},
+        }
+        for i in common:
+            routed = (rows[i].get('labeler') or {}).get('routing', 'unknown')
+            report['candidates'][name]['routing'][routed] = \
+                report['candidates'][name]['routing'].get(routed, 0) + 1
+    # Backend parity over shared ids.
+    if len(args.cand) == 2:
+        a, b = (cands[args.cand[0]], cands[args.cand[1]])
+        same_route = sum(1 for i in common
+                         if (a[i].get('labeler') or {}).get('routing') == (b[i].get('labeler') or {}).get('routing'))
+        same_gold = sum(1 for i in common if a[i]['gold'] == b[i]['gold'])
+        deltas = [max(abs(a[i]['scores'][k] - b[i]['scores'][k]) for k in INTENTS) for i in common]
+        report['parity'] = {'pair': list(args.cand), 'n': len(common),
+                            'routingAgreement': same_route / len(common),
+                            'derivedGoldAgreement': same_gold / len(common),
+                            'maxAbsDeltaMean': sum(deltas) / len(deltas),
+                            'maxAbsDeltaP95': sorted(deltas)[int(len(deltas) * 0.95)]}
+    Path(args.out).write_text(json.dumps(report, indent=2), encoding='utf-8')
+    print(json.dumps({k: v for k, v in report.items() if k != 'candidates'}, indent=2))
+    for name, c in report['candidates'].items():
+        print(f"{name}: exact={c['primaryExact']:.3f} graded={c['gradedMean']:.3f} "
+              f"top={c['meanTopScore']:.3f} null={c['nullRate']:.3f} routing={c['routing']} slices={c['slices']}")
+    if 'parity' in report:
+        print('parity:', json.dumps(report['parity']))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-corpus.js

@@ -1,16 +1,15 @@
 #!/usr/bin/env node
 'use strict';
-// Assembles the System One tier holdout from a draft plus human review decisions.
-// See docs/system-one-corpus.md. A decision counts only for the exact prompt text it reviewed.
+// Assembles a System One holdout (tier or intent stage) from a draft plus human review decisions.
+// See docs/system-one-corpus.md and docs/system-one-intent.md. A decision counts only for the exact prompt text it reviewed.
 const fs = require('node:fs');
 const { createHash } = require('node:crypto');
 const { createRequest } = require('../harness-everything/scripts/system-one/contract');
-const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const { CATALOGS, MULTI, goldLabels, validSecondary } = require('../harness-everything/scripts/system-one/catalogs');
 const { validateCorpus } = require('../harness-everything/scripts/system-one/evaluate');
 
 const SOURCES = ['synthetic:authored', 'derived:local-history'];
 const LANGUAGES = ['en', 'zh-TW'];
-const GOLD = ['tier1', 'tier2', 'tier3', null];
 const REVIEWER = 'human:repository-owner';
 // Drafts are committed publicly: no email addresses or user home paths.
 const PRIVATE = [/[\w.+-]+@[\w-]+\.[\w.-]+/, /[A-Za-z]:\\Users\\/i, /\/home\/[^/\s]+/, /\/Users\/[^/\s]+/];
@@ -20,7 +19,8 @@ const nonempty = v => typeof v === 'string' && v.trim().length > 0;
 const fail = code => { throw new TypeError(code); };
 const promptHash = text => createHash('sha256').update(text, 'utf8').digest('hex');
 
-function validateDraft(draft) {
+function validateDraft(draft, task = 'tier') {
+  const GOLD = goldLabels(task);
   if (!exact(draft, ['schemaVersion', 'cases']) || draft.schemaVersion !== 1 || !Array.isArray(draft.cases) || !draft.cases.length) fail('invalid-draft');
   const ids = new Set(); const prompts = new Set();
   for (const c of draft.cases) {
@@ -33,24 +33,27 @@ function validateDraft(draft) {
   return true;
 }
 
-function validateReviews(reviews) {
+function validateReviews(reviews, task = 'tier') {
+  const GOLD = goldLabels(task);
   if (!exact(reviews, ['schemaVersion', 'decisions']) || reviews.schemaVersion !== 1 || !Array.isArray(reviews.decisions)) fail('invalid-reviews');
   const ids = new Set();
   for (const d of reviews.decisions) {
-    if (!exact(d, ['id', 'promptHash', 'decision', 'gold', 'reviewer']) || !nonempty(d.id) || ids.has(d.id)
+    const multi = MULTI.includes(task);
+    if (!exact(d, ['id', 'promptHash', 'decision', 'gold', ...(multi ? ['secondary'] : []), 'reviewer']) || !nonempty(d.id) || ids.has(d.id)
       || typeof d.promptHash !== 'string' || !/^[0-9a-f]{64}$/.test(d.promptHash)
       || !['accept', 'relabel', 'reject'].includes(d.decision) || !GOLD.includes(d.gold) || d.reviewer !== REVIEWER
+      || !validSecondary(task, d.gold, d.secondary)
       || (d.decision === 'reject' && d.gold !== null)) fail('invalid-reviews');
     ids.add(d.id);
   }
   return true;
 }
 
-function build(draft, reviews) {
-  validateDraft(draft);
+function build(draft, reviews, task = 'tier') {
+  validateDraft(draft, task);
   const byId = new Map();
   if (reviews !== null) {
-    validateReviews(reviews);
+    validateReviews(reviews, task);
     for (const d of reviews.decisions) {
       const c = draft.cases.find(item => item.id === d.id);
       if (!c) fail('unknown-review-case');
@@ -60,7 +63,8 @@ function build(draft, reviews) {
     }
   }
   const summary = { cases: 0, reviewed: 0, rejected: 0, stale: 0, unreviewed: 0,
-    byLanguage: { en: 0, 'zh-TW': 0 }, byGold: { tier1: 0, tier2: 0, tier3: 0, unclassified: 0 }, reviewedHoldoutGate: false };
+    byLanguage: { en: 0, 'zh-TW': 0 }, byGold: Object.fromEntries(CATALOGS[task].map(o => [o.id, 0])),
+    ...(MULTI.includes(task) ? { bySecondary: {} } : {}), reviewedHoldoutGate: false };
   const cases = [];
   for (const c of draft.cases) {
     const d = byId.get(c.id);
@@ -70,10 +74,13 @@ function build(draft, reviews) {
     const reviewed = Boolean(current);
     if (reviewed) summary.reviewed += 1; else if (!d) summary.unreviewed += 1;
     const gold = reviewed ? d.gold : c.proposedGold;
+    // Drafts propose only the primary intent; secondary intents come from the reviewer.
+    const secondary = MULTI.includes(task) ? { secondary: reviewed ? [...d.secondary] : [] } : {};
     cases.push({ id: c.id, family: c.family, split: 'holdout', language: c.language, source: c.source, reviewed,
-      request: createRequest('tier', c.prompt, TIER_OPTIONS), gold });
+      request: createRequest(task, c.prompt, CATALOGS[task]), gold, ...secondary });
     summary.byLanguage[c.language] += 1;
     summary.byGold[gold === null ? 'unclassified' : gold] += 1;
+    for (const s of secondary.secondary || []) summary.bySecondary[s] = (summary.bySecondary[s] || 0) + 1;
   }
   summary.cases = cases.length;
   const corpus = { schemaVersion: 1, cases };
@@ -85,14 +92,14 @@ function build(draft, reviews) {
 
 if (require.main === module) {
   try {
-    const [op, draftPath, reviewsPath, outPath, extra] = process.argv.slice(2);
+    const [op, draftPath, reviewsPath, outPath, task = 'tier', extra] = process.argv.slice(2);
     if (op !== 'build' || !draftPath || !reviewsPath || !outPath || extra) throw new Error('usage');
     const read = file => JSON.parse(fs.readFileSync(file, 'utf8'));
-    const { corpus, summary } = build(read(draftPath), reviewsPath === '-' ? null : read(reviewsPath));
+    const { corpus, summary } = build(read(draftPath), reviewsPath === '-' ? null : read(reviewsPath), task);
     fs.writeFileSync(outPath, `${JSON.stringify(corpus, null, 2)}\n`);
     console.log(JSON.stringify(summary));
   } catch (_) {
-    console.error('Usage: system-one-corpus.js build <draft.json> <reviews.json|-> <corpus.json> (invalid draft, reviews, or path)');
+    console.error('Usage: system-one-corpus.js build <draft.json> <reviews.json|-> <corpus.json> [tier|intent] (invalid draft, reviews, or path)');
     process.exitCode = 1;
   }
 }

package/scripts/system-one-eval-laya.js

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+'use strict';
+// Research evaluation driver for the fine-tuned Laya bridge (#255 Phase 3).
+// It does NOT touch provider.js/resident.js (still cua-only): it starts the
+// laya resident server itself, scores cases through it, and feeds the
+// production decide()/evaluate() machinery. Usage:
+//   score  <cases.json> <manifest.json> <records.json> [workdir]
+//   report <corpus.json> <records.json> <report.json> [--secondary-threshold T]
+// cases.json: { cases: [{ id, baseline, request }] } (request = holdout-style,
+// context + 13 catalog options). records.json matches evaluate() input.
+const fs = require('node:fs');
+const path = require('node:path');
+const net = require('node:net');
+const { spawn } = require('node:child_process');
+const { performance } = require('node:perf_hooks');
+const { evaluate } = require('../harness-everything/scripts/system-one/evaluate');
+const { decide } = require('../harness-everything/scripts/system-one/contract');
+const { decisionPolicy } = require('../harness-everything/scripts/system-one/provider');
+
+const ADAPTER = path.join(__dirname, '..', 'harness-everything', 'scripts', 'system-one', 'laya_adapter.py');
+const CALL_TIMEOUT_MS = 30000;
+
+function send(port, token, message) {
+  return new Promise(resolve => {
+    const socket = net.connect({ port, host: '127.0.0.1' });
+    const chunks = [];
+    const done = value => { socket.destroy(); resolve(value); };
+    const timer = setTimeout(() => done({ ok: false, reason: 'timeout' }), CALL_TIMEOUT_MS);
+    socket.on('connect', () => socket.end(`${JSON.stringify({ token, ...message })}\n`));
+    socket.on('data', c => chunks.push(c));
+    socket.on('end', () => { clearTimeout(timer);
+      try { done(JSON.parse(Buffer.concat(chunks).toString('utf8'))); }
+      catch (_) { done({ ok: false, reason: 'bad-reply' }); } });
+    socket.on('error', () => { clearTimeout(timer); done({ ok: false, reason: 'io' }); });
+  });
+}
+
+async function withServer(python, manifest, workdir, fn) {
+  const manifestJson = JSON.stringify(manifest);
+  const state = path.join(workdir, 'laya-research-state.json');
+  const lock = path.join(workdir, 'laya-research-state.starting');
+  for (const f of [state, lock]) try { fs.unlinkSync(f); } catch (_) { /* fresh */ }
+  const child = spawn(python, [ADAPTER, '--serve', manifestJson, state, lock], { stdio: 'ignore' });
+  const deadline = Date.now() + 120000;
+  let info = null;
+  while (Date.now() < deadline) {
+    try { info = JSON.parse(fs.readFileSync(state, 'utf8')); if (info.port && info.token) break; } catch (_) { /* starting */ }
+    if (child.exitCode !== null) throw new Error(`server exited during startup (code ${child.exitCode})`);
+    await new Promise(r => setTimeout(r, 200));
+    info = null;
+  }
+  if (!info) { child.kill('SIGKILL'); throw new Error('server did not become ready'); }
+  try {
+    return await fn(info);
+  } finally {
+    try { await send(info.port, info.token, { op: 'shutdown' }); } catch (_) { /* exiting */ }
+    child.kill('SIGKILL');
+  }
+}
+
+async function score(casesPath, manifestPath, recordsPath, workdir) {
+  const { cases } = JSON.parse(fs.readFileSync(casesPath, 'utf8'));
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+  const policy = decisionPolicy(manifest.acceptance);
+  const records = await withServer(manifest.python, manifest, workdir, async info => {
+    const out = [];
+    for (const c of cases) {
+      const runs = [];
+      for (let i = 0; i < 2; i++) {
+        const start = performance.now();
+        const reply = await send(info.port, info.token, { op: 'score', request: c.request });
+        const latencyMs = performance.now() - start;
+        let decision;
+        let scores = null;
+        if (reply.ok) {
+          decision = decide(c.request, reply.response, policy);
+          if (decision.selectedId === 'unclassified') {
+            decision = { ...decision, status: 'abstain', reason: 'unclassified', selectedId: null };
+          }
+          if (['accepted', 'abstain'].includes(decision.status)) scores = reply.response.scores.map(s => s.probability);
+        } else {
+          decision = { status: 'unavailable', reason: reply.reason };
+        }
+        runs.push({ decision, scores, latencyMs, coldStart: false });
+      }
+      out.push({ id: c.id, baseline: c.baseline, runs });
+    }
+    return out;
+  });
+  fs.writeFileSync(recordsPath, `${JSON.stringify(records)}\n`);
+  console.log(JSON.stringify({ cases: records.length }));
+}
+
+function report(corpusPath, recordsPath, reportPath, secondaryThreshold) {
+  const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
+  const records = JSON.parse(fs.readFileSync(recordsPath, 'utf8'));
+  const result = evaluate(corpus, records, null, { secondaryThreshold });
+  fs.writeFileSync(reportPath, `${JSON.stringify(result, null, 2)}\n`);
+  console.log(JSON.stringify({ cases: result.cases, coverage: result.model.coverage,
+    acceptedPrecision: result.model.acceptedPrecision, rolloutReady: result.rolloutReady }));
+}
+
+async function main() {
+  const [mode, a, b, c, d] = process.argv.slice(2);
+  if (mode === 'score' && a && b && c) return score(a, b, c, d || fs.mkdtempSync('/tmp/laya-eval-'));
+  if (mode === 'report' && a && b && c) {
+    const t = d === undefined ? undefined : Number((d.match(/^--secondary-threshold=(.+)$/) || [])[1]);
+    return report(a, b, c, t);
+  }
+  throw new Error('usage: system-one-eval-laya.js (score <cases> <manifest> <records> [workdir] | report <corpus> <records> <report> [--secondary-threshold=T])');
+}
+main().catch(err => { console.error(`laya research eval failed: ${err.message}`); process.exitCode = 1; });