harness-everything 0.29.0 → 0.30.1

package/scripts/system-one-corpus.js

@@ -1,16 +1,15 @@
 #!/usr/bin/env node
 'use strict';
-// Assembles the System One tier holdout from a draft plus human review decisions.
-// See docs/system-one-corpus.md. A decision counts only for the exact prompt text it reviewed.
+// Assembles a System One holdout (tier or intent stage) from a draft plus human review decisions.
+// See docs/system-one-corpus.md and docs/system-one-intent.md. A decision counts only for the exact prompt text it reviewed.
 const fs = require('node:fs');
 const { createHash } = require('node:crypto');
 const { createRequest } = require('../harness-everything/scripts/system-one/contract');
-const { TIER_OPTIONS } = require('../harness-everything/scripts/system-one/router');
+const { CATALOGS, MULTI, goldLabels, validSecondary } = require('../harness-everything/scripts/system-one/catalogs');
 const { validateCorpus } = require('../harness-everything/scripts/system-one/evaluate');
 
 const SOURCES = ['synthetic:authored', 'derived:local-history'];
 const LANGUAGES = ['en', 'zh-TW'];
-const GOLD = ['tier1', 'tier2', 'tier3', null];
 const REVIEWER = 'human:repository-owner';
 // Drafts are committed publicly: no email addresses or user home paths.
 const PRIVATE = [/[\w.+-]+@[\w-]+\.[\w.-]+/, /[A-Za-z]:\\Users\\/i, /\/home\/[^/\s]+/, /\/Users\/[^/\s]+/];
@@ -20,7 +19,8 @@ const nonempty = v => typeof v === 'string' && v.trim().length > 0;
 const fail = code => { throw new TypeError(code); };
 const promptHash = text => createHash('sha256').update(text, 'utf8').digest('hex');
 
-function validateDraft(draft) {
+function validateDraft(draft, task = 'tier') {
+  const GOLD = goldLabels(task);
   if (!exact(draft, ['schemaVersion', 'cases']) || draft.schemaVersion !== 1 || !Array.isArray(draft.cases) || !draft.cases.length) fail('invalid-draft');
   const ids = new Set(); const prompts = new Set();
   for (const c of draft.cases) {
@@ -33,24 +33,27 @@ function validateDraft(draft) {
   return true;
 }
 
-function validateReviews(reviews) {
+function validateReviews(reviews, task = 'tier') {
+  const GOLD = goldLabels(task);
   if (!exact(reviews, ['schemaVersion', 'decisions']) || reviews.schemaVersion !== 1 || !Array.isArray(reviews.decisions)) fail('invalid-reviews');
   const ids = new Set();
   for (const d of reviews.decisions) {
-    if (!exact(d, ['id', 'promptHash', 'decision', 'gold', 'reviewer']) || !nonempty(d.id) || ids.has(d.id)
+    const multi = MULTI.includes(task);
+    if (!exact(d, ['id', 'promptHash', 'decision', 'gold', ...(multi ? ['secondary'] : []), 'reviewer']) || !nonempty(d.id) || ids.has(d.id)
       || typeof d.promptHash !== 'string' || !/^[0-9a-f]{64}$/.test(d.promptHash)
       || !['accept', 'relabel', 'reject'].includes(d.decision) || !GOLD.includes(d.gold) || d.reviewer !== REVIEWER
+      || !validSecondary(task, d.gold, d.secondary)
       || (d.decision === 'reject' && d.gold !== null)) fail('invalid-reviews');
     ids.add(d.id);
   }
   return true;
 }
 
-function build(draft, reviews) {
-  validateDraft(draft);
+function build(draft, reviews, task = 'tier') {
+  validateDraft(draft, task);
   const byId = new Map();
   if (reviews !== null) {
-    validateReviews(reviews);
+    validateReviews(reviews, task);
     for (const d of reviews.decisions) {
       const c = draft.cases.find(item => item.id === d.id);
       if (!c) fail('unknown-review-case');
@@ -60,7 +63,8 @@ function build(draft, reviews) {
     }
   }
   const summary = { cases: 0, reviewed: 0, rejected: 0, stale: 0, unreviewed: 0,
-    byLanguage: { en: 0, 'zh-TW': 0 }, byGold: { tier1: 0, tier2: 0, tier3: 0, unclassified: 0 }, reviewedHoldoutGate: false };
+    byLanguage: { en: 0, 'zh-TW': 0 }, byGold: Object.fromEntries(CATALOGS[task].map(o => [o.id, 0])),
+    ...(MULTI.includes(task) ? { bySecondary: {} } : {}), reviewedHoldoutGate: false };
   const cases = [];
   for (const c of draft.cases) {
     const d = byId.get(c.id);
@@ -70,10 +74,13 @@ function build(draft, reviews) {
     const reviewed = Boolean(current);
     if (reviewed) summary.reviewed += 1; else if (!d) summary.unreviewed += 1;
     const gold = reviewed ? d.gold : c.proposedGold;
+    // Drafts propose only the primary intent; secondary intents come from the reviewer.
+    const secondary = MULTI.includes(task) ? { secondary: reviewed ? [...d.secondary] : [] } : {};
     cases.push({ id: c.id, family: c.family, split: 'holdout', language: c.language, source: c.source, reviewed,
-      request: createRequest('tier', c.prompt, TIER_OPTIONS), gold });
+      request: createRequest(task, c.prompt, CATALOGS[task]), gold, ...secondary });
     summary.byLanguage[c.language] += 1;
     summary.byGold[gold === null ? 'unclassified' : gold] += 1;
+    for (const s of secondary.secondary || []) summary.bySecondary[s] = (summary.bySecondary[s] || 0) + 1;
   }
   summary.cases = cases.length;
   const corpus = { schemaVersion: 1, cases };
@@ -85,14 +92,14 @@ function build(draft, reviews) {
 
 if (require.main === module) {
   try {
-    const [op, draftPath, reviewsPath, outPath, extra] = process.argv.slice(2);
+    const [op, draftPath, reviewsPath, outPath, task = 'tier', extra] = process.argv.slice(2);
     if (op !== 'build' || !draftPath || !reviewsPath || !outPath || extra) throw new Error('usage');
     const read = file => JSON.parse(fs.readFileSync(file, 'utf8'));
-    const { corpus, summary } = build(read(draftPath), reviewsPath === '-' ? null : read(reviewsPath));
+    const { corpus, summary } = build(read(draftPath), reviewsPath === '-' ? null : read(reviewsPath), task);
     fs.writeFileSync(outPath, `${JSON.stringify(corpus, null, 2)}\n`);
     console.log(JSON.stringify(summary));
   } catch (_) {
-    console.error('Usage: system-one-corpus.js build <draft.json> <reviews.json|-> <corpus.json> (invalid draft, reviews, or path)');
+    console.error('Usage: system-one-corpus.js build <draft.json> <reviews.json|-> <corpus.json> [tier|intent] (invalid draft, reviews, or path)');
     process.exitCode = 1;
   }
 }

package/scripts/system-one-eval-laya.js

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+'use strict';
+// Research evaluation driver for the fine-tuned Laya bridge (#255 Phase 3).
+// It does NOT touch provider.js/resident.js (still cua-only): it starts the
+// laya resident server itself, scores cases through it, and feeds the
+// production decide()/evaluate() machinery. Usage:
+//   score  <cases.json> <manifest.json> <records.json> [workdir]
+//   report <corpus.json> <records.json> <report.json> [--secondary-threshold T]
+// cases.json: { cases: [{ id, baseline, request }] } (request = holdout-style,
+// context + 13 catalog options). records.json matches evaluate() input.
+const fs = require('node:fs');
+const path = require('node:path');
+const net = require('node:net');
+const { spawn } = require('node:child_process');
+const { performance } = require('node:perf_hooks');
+const { evaluate } = require('../harness-everything/scripts/system-one/evaluate');
+const { decide } = require('../harness-everything/scripts/system-one/contract');
+const { decisionPolicy } = require('../harness-everything/scripts/system-one/provider');
+
+const ADAPTER = path.join(__dirname, '..', 'harness-everything', 'scripts', 'system-one', 'laya_adapter.py');
+const CALL_TIMEOUT_MS = 30000;
+
+function send(port, token, message) {
+  return new Promise(resolve => {
+    const socket = net.connect({ port, host: '127.0.0.1' });
+    const chunks = [];
+    const done = value => { socket.destroy(); resolve(value); };
+    const timer = setTimeout(() => done({ ok: false, reason: 'timeout' }), CALL_TIMEOUT_MS);
+    socket.on('connect', () => socket.end(`${JSON.stringify({ token, ...message })}\n`));
+    socket.on('data', c => chunks.push(c));
+    socket.on('end', () => { clearTimeout(timer);
+      try { done(JSON.parse(Buffer.concat(chunks).toString('utf8'))); }
+      catch (_) { done({ ok: false, reason: 'bad-reply' }); } });
+    socket.on('error', () => { clearTimeout(timer); done({ ok: false, reason: 'io' }); });
+  });
+}
+
+async function withServer(python, manifest, workdir, fn) {
+  const manifestJson = JSON.stringify(manifest);
+  const state = path.join(workdir, 'laya-research-state.json');
+  const lock = path.join(workdir, 'laya-research-state.starting');
+  for (const f of [state, lock]) try { fs.unlinkSync(f); } catch (_) { /* fresh */ }
+  const child = spawn(python, [ADAPTER, '--serve', manifestJson, state, lock], { stdio: 'ignore' });
+  const deadline = Date.now() + 120000;
+  let info = null;
+  while (Date.now() < deadline) {
+    try { info = JSON.parse(fs.readFileSync(state, 'utf8')); if (info.port && info.token) break; } catch (_) { /* starting */ }
+    if (child.exitCode !== null) throw new Error(`server exited during startup (code ${child.exitCode})`);
+    await new Promise(r => setTimeout(r, 200));
+    info = null;
+  }
+  if (!info) { child.kill('SIGKILL'); throw new Error('server did not become ready'); }
+  try {
+    return await fn(info);
+  } finally {
+    try { await send(info.port, info.token, { op: 'shutdown' }); } catch (_) { /* exiting */ }
+    child.kill('SIGKILL');
+  }
+}
+
+async function score(casesPath, manifestPath, recordsPath, workdir) {
+  const { cases } = JSON.parse(fs.readFileSync(casesPath, 'utf8'));
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+  const policy = decisionPolicy(manifest.acceptance);
+  const records = await withServer(manifest.python, manifest, workdir, async info => {
+    const out = [];
+    for (const c of cases) {
+      const runs = [];
+      for (let i = 0; i < 2; i++) {
+        const start = performance.now();
+        const reply = await send(info.port, info.token, { op: 'score', request: c.request });
+        const latencyMs = performance.now() - start;
+        let decision;
+        let scores = null;
+        if (reply.ok) {
+          decision = decide(c.request, reply.response, policy);
+          if (decision.selectedId === 'unclassified') {
+            decision = { ...decision, status: 'abstain', reason: 'unclassified', selectedId: null };
+          }
+          if (['accepted', 'abstain'].includes(decision.status)) scores = reply.response.scores.map(s => s.probability);
+        } else {
+          decision = { status: 'unavailable', reason: reply.reason };
+        }
+        runs.push({ decision, scores, latencyMs, coldStart: false });
+      }
+      out.push({ id: c.id, baseline: c.baseline, runs });
+    }
+    return out;
+  });
+  fs.writeFileSync(recordsPath, `${JSON.stringify(records)}\n`);
+  console.log(JSON.stringify({ cases: records.length }));
+}
+
+function report(corpusPath, recordsPath, reportPath, secondaryThreshold) {
+  const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
+  const records = JSON.parse(fs.readFileSync(recordsPath, 'utf8'));
+  const result = evaluate(corpus, records, null, { secondaryThreshold });
+  fs.writeFileSync(reportPath, `${JSON.stringify(result, null, 2)}\n`);
+  console.log(JSON.stringify({ cases: result.cases, coverage: result.model.coverage,
+    acceptedPrecision: result.model.acceptedPrecision, rolloutReady: result.rolloutReady }));
+}
+
+async function main() {
+  const [mode, a, b, c, d] = process.argv.slice(2);
+  if (mode === 'score' && a && b && c) return score(a, b, c, d || fs.mkdtempSync('/tmp/laya-eval-'));
+  if (mode === 'report' && a && b && c) {
+    const t = d === undefined ? undefined : Number((d.match(/^--secondary-threshold=(.+)$/) || [])[1]);
+    return report(a, b, c, t);
+  }
+  throw new Error('usage: system-one-eval-laya.js (score <cases> <manifest> <records> [workdir] | report <corpus> <records> <report> [--secondary-threshold=T])');
+}
+main().catch(err => { console.error(`laya research eval failed: ${err.message}`); process.exitCode = 1; });

package/scripts/system-one-export-laya.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Export soft-target RLCD training items for Laya fine-tuning (#255 Phase 3).
+
+Each train prompt with teacher relevance scores becomes 12 `noul` items, one
+per owner intent (catalog order, v1 wording shared with system-one-label-laya).
+Teacher relevance p maps to gold probabilities {true: p, false: 1-p}, matching
+the upstream fine-tune notebook (LocalLLaMA/typed-decisions, noul target
+[false, true], qtype 2, label = argmax).
+
+Output is reviewable jsonl (one item per line); the training host converts it
+to train_items.pt with torch.save (never commit .pt). Sequence building goes
+through laya.common.build_sequence so tokenization matches inference exactly.
+Stdlib only at import time; `laya` is imported lazily inside the default
+builder so contract tests run without model weights.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import random
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve()
+_label_spec = importlib.util.spec_from_file_location(
+    'laya_labeler', HERE.parent / 'system-one-label-laya.py')
+_label = importlib.util.module_from_spec(_label_spec)
+_label_spec.loader.exec_module(_label)
+INTENTS = _label.INTENTS
+QUESTIONS_VERSION = _label.QUESTIONS_VERSION
+build_questions = _label.build_questions
+
+# laya.common.QTYPES = {"choice": 0, "score": 1, "noul": 2}.
+QTYPE_NOUL = 2
+
+
+def build_item(state_text, instructions, probs, builder):
+    """One upstream-format training item from gold {true, false} probabilities."""
+    pf, pt = float(probs['false']), float(probs['true'])
+    if not (pf >= 0 and pt >= 0 and abs(pf + pt - 1.0) < 1e-6):
+        raise ValueError(f'probabilities must sum to 1, got {probs!r}')
+    built = builder(state_text, instructions, {'true': pt, 'false': pf})
+    if built is None:
+        return None
+    target = [built['target'][0], built['target'][1]]
+    if abs(sum(target) - 1.0) >= 1e-6:
+        raise ValueError(f'builder target must sum to 1, got {target!r}')
+    return {'ids': built['ids'], 'markers': built['markers'], 'qtype': QTYPE_NOUL,
+            'target': target, 'label': target.index(max(target))}
+
+
+def parse_oversample(spec):
+    """Parse 'intent:factor,...' into {intent: factor}; factor 1 is rejected."""
+    table = {}
+    for part in (spec or '').split(','):
+        if not part.strip():
+            continue
+        try:
+            intent, factor = part.split(':')
+            factor = int(factor)
+        except ValueError:
+            raise ValueError(f'oversample: bad entry {part!r}, want intent:factor')
+        if intent not in INTENTS or factor < 2:
+            raise ValueError(f'oversample: bad entry {part!r}, want known intent and factor >= 2')
+        table[intent] = factor
+    return table
+
+
+def default_builder_factory(checkpoint='convaiinnovations/laya', subfolder='multilingual'):
+    """Real sequence builder through laya.common (lazy import)."""
+    from huggingface_hub import snapshot_download  # noqa: PLC0415
+    from transformers import AutoTokenizer  # noqa: PLC0415
+    from laya.agent import _fix_tokenizer_config  # noqa: PLC0415
+    from laya.common import build_sequence, render_options  # noqa: PLC0415
+
+    model_dir = Path(snapshot_download(checkpoint))
+    revision = model_dir.name
+    base = model_dir / subfolder if subfolder else model_dir
+    _fix_tokenizer_config(str(base))
+    tok = AutoTokenizer.from_pretrained(str(base / 'tokenizer'))
+    cfg = json.loads((base / 'rl_agent_config.json').read_text(encoding='utf-8'))
+    questions = build_questions()
+
+    def builder(state_text, instructions, probs):
+        question = {'t': 'noul', 'ins': instructions, 'crit': {}}
+        seq, markers = build_sequence(tok, state_text, question, cfg['max_len'], cfg['head_max_len'])
+        if len(markers) != len(render_options(question)):
+            return None
+        return {'ids': seq, 'markers': markers,
+                'target': [probs['false'], probs['true']]}
+
+    provenance = {'checkpoint': checkpoint, 'revision': revision, 'subfolder': subfolder or 'root',
+                  'questions': QUESTIONS_VERSION,
+                  'tokenizerDir': str(base / 'tokenizer')}
+    return builder, provenance
+
+
+def export(data_dir, split, out_path, manifest_path, builder, seed=0, forbid_path=None,
+           teacher_file='labels-intent-scores.jsonl', provenance=None, oversample=None):
+    """Export one prompts.jsonl split to upstream-format items."""
+    if split not in ('train', 'validation'):
+        raise ValueError(f'split must be train or validation, got {split!r}')
+    oversample = dict(oversample or {})
+    for intent, factor in oversample.items():
+        if intent not in INTENTS or not isinstance(factor, int) or factor < 2:
+            raise ValueError(f'oversample: bad entry {intent!r}:{factor!r}')
+    data_dir, out_path, manifest_path = Path(data_dir), Path(out_path), Path(manifest_path)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    teacher_raw = (data_dir / teacher_file).read_text(encoding='utf-8')
+    teacher = {r['id']: r for r in
+               (json.loads(line) for line in teacher_raw.splitlines() if line.strip())}
+    rows = sorted((r for r in (json.loads(line) for line in prompts_raw.splitlines() if line.strip())
+                   if r.get('split') == split), key=lambda r: r['id'])
+    if forbid_path is not None:
+        forbidden = set(json.loads(Path(forbid_path).read_text(encoding='utf-8')))
+        clash = sorted({r['id'] for r in rows} & forbidden)
+        if clash:
+            raise ValueError(f'overlap with forbidden ids: {clash[:5]} (n={len(clash)})')
+    if seed:
+        random.Random(seed).shuffle(rows)
+    questions = build_questions()
+    items, skipped_no_teacher, skipped_markers = 0, 0, 0
+    with out_path.open('w', encoding='utf-8') as fh:
+        for prompt in rows:
+            trow = teacher.get(prompt['id'])
+            if trow is None:
+                skipped_no_teacher += 1
+                continue
+            for intent in INTENTS:
+                p = float(trow['scores'].get(intent, 0.0))
+                repeat = oversample.get(intent, 1) if p >= 0.4 else 1
+                item = build_item(prompt['text'], questions[intent]['instructions'],
+                                  {'true': p, 'false': 1.0 - p}, builder)
+                if item is None:
+                    skipped_markers += repeat
+                    continue
+                line = json.dumps({'promptId': prompt['id'], 'intent': intent, **item}) + '\n'
+                for _ in range(repeat):
+                    fh.write(line)
+                items += repeat
+    manifest = {'split': split, 'seed': seed, 'questions': QUESTIONS_VERSION,
+                'builder': provenance or {'builder': 'injected'},
+                'oversample': oversample,
+                'items': items, 'prompts': len(rows),
+                'skippedNoTeacher': skipped_no_teacher, 'skippedMarkers': skipped_markers,
+                'promptsSha256': hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                'teacherSha256': hashlib.sha256(teacher_raw.encode('utf-8')).hexdigest(),
+                'teacherFile': teacher_file,
+                'outputSha256': hashlib.sha256(out_path.read_bytes()).hexdigest()}
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
+    return {'items': items, 'skippedNoTeacher': skipped_no_teacher,
+            'skippedMarkers': skipped_markers, 'manifest': manifest}
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True, help='private corpus dir')
+    parser.add_argument('--split', choices=['train', 'validation'], default='train')
+    parser.add_argument('--out', required=True, help='output items jsonl')
+    parser.add_argument('--manifest', required=True, help='provenance manifest json')
+    parser.add_argument('--checkpoint', default='convaiinnovations/laya')
+    parser.add_argument('--subfolder', default='multilingual')
+    parser.add_argument('--seed', type=int, default=0, help='0 keeps id order; nonzero shuffles')
+    parser.add_argument('--forbid', default=None, help='json list of forbidden ids (overlap guard)')
+    parser.add_argument('--teacher', default='labels-intent-scores.jsonl')
+    parser.add_argument('--oversample', default='', help='intent:factor,... to repeat positive intent items')
+    parser.add_argument('--limit', type=int, default=0, help='export only the first N prompts (0 = all)')
+    args = parser.parse_args(argv)
+    oversample = parse_oversample(args.oversample)
+    builder, provenance = default_builder_factory(args.checkpoint, args.subfolder)
+    print(json.dumps({'builder': provenance}))
+    # --limit applies before export by staging a filtered view; export() keeps split semantics.
+    if args.limit > 0:
+        import tempfile  # noqa: PLC0415
+        data_dir = Path(args.data_dir)
+        prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+        rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+        todo = sorted((r for r in rows if r.get('split') == args.split), key=lambda r: r['id'])[:args.limit]
+        with tempfile.TemporaryDirectory() as tmp:
+            view = Path(tmp) / 'view'
+            view.mkdir()
+            (view / 'prompts.jsonl').write_text(
+                '\n'.join(json.dumps(r) for r in todo) + '\n', encoding='utf-8')
+            (view / args.teacher).write_text((data_dir / args.teacher).read_text(encoding='utf-8'), encoding='utf-8')
+            summary = export(view, args.split, args.out, args.manifest, builder,
+                             seed=args.seed, forbid_path=args.forbid, teacher_file=args.teacher,
+                             provenance=provenance, oversample=oversample)
+    else:
+        summary = export(args.data_dir, args.split, args.out, args.manifest, builder,
+                         seed=args.seed, forbid_path=args.forbid, teacher_file=args.teacher,
+                         provenance=provenance, oversample=oversample)
+    print(json.dumps({k: v for k, v in summary.items() if k != 'manifest'}))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/system-one-label-laya.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""Zero-shot dense intent labeling with Laya (#255 Phase 1+2).
+
+Maps each of the 12 owner intents (catalogs.js order, frozen) to one Laya
+`noul` question and records the yes-probability as the per-intent relevance
+score. Output rows match the `labels-intent-scores.jsonl` trainer contract
+{id, gold, secondary, scores}, but MUST be written to a separate file: the
+sonnet-scored labels are never overwritten. Derived gold/secondary are
+mechanical bootstrap only (labeler.derived=true), never owner truth.
+
+Stdlib only at import time; `laya` / `laya_mlx` are imported lazily inside
+the backend constructors so contract tests run without model weights.
+"""
+import argparse
+import hashlib
+import json
+import math
+import statistics
+import sys
+import time
+from pathlib import Path
+
+# Owner catalog order (harness-everything/scripts/system-one/catalogs.js).
+# This order is the scorer output order; changing it invalidates weights.
+INTENTS = ['explain', 'discuss', 'git', 'fix', 'edit', 'feature', 'refactor',
+           'review', 'test', 'docs', 'plan', 'investigate']
+
+QUESTIONS_VERSION = 'v1'
+
+# The sonnet-scored labels file. The labeler refuses to write to it.
+PROTECTED_OUTPUT = 'labels-intent-scores.jsonl'
+
+_PROPOSITIONS = {
+    'explain': 'The request asks for an explanation of how something works or an answer to a question.',
+    'discuss': 'The request asks to weigh options, give an opinion, or settle a decision.',
+    'git': 'The request asks for Git or GitHub housekeeping: commit, push, branch, or pull request.',
+    'fix': 'The request asks to repair wrong behavior: a bug, crash, failing test, or build.',
+    'edit': 'The request asks to change an existing value, setting, text, or behavior.',
+    'feature': 'The request asks to add new behavior: a command, option, format, or script.',
+    'refactor': 'The request asks to restructure or unify existing code without adding new behavior.',
+    'review': 'The request asks to evaluate an existing artifact against a standard.',
+    'test': 'The request asks to run tests or builds, or to write tests as the main deliverable.',
+    'docs': 'The request asks to write or edit documentation as the main deliverable.',
+    'plan': 'The request asks to produce a plan, specification, or breakdown before implementation.',
+    'investigate': 'The request asks to find facts or a cause and report back without changing anything.',
+}
+
+# v2 adds contrastive exclusions aimed at the measured top-2 co-firing pairs
+# (edit/refactor, fix/investigate, investigate/review/explain, docs/edit,
+# feature/edit, plan/discuss). Same 12 intents, same semantics: exclusions
+# only sharpen boundaries, they never add or remove a category.
+_PROPOSITIONS_V2 = {
+    'explain': 'The request asks for an explanation of how something works or an answer to a question. Not investigating an unknown cause (investigate).',
+    'discuss': 'The request asks to weigh options, give an opinion, or settle a decision. Not producing an implementation plan or breakdown (plan).',
+    'git': 'The request asks for Git or GitHub housekeeping: commit, push, branch, or pull request.',
+    'fix': 'The request asks to repair wrong behavior: a bug, crash, failing test, or build. Not merely finding the cause (investigate); only when a repair is requested.',
+    'edit': 'The request asks to change an existing value, setting, text, or behavior. Not a bug repair (fix), not a restructuring (refactor), not new behavior (feature), not documentation as the deliverable (docs).',
+    'feature': 'The request asks to add new behavior: a command, option, format, or script. Not modifying existing behavior (edit).',
+    'refactor': 'The request asks to restructure or unify existing code without adding new behavior. Not a value change (edit), not a bug repair (fix).',
+    'review': 'The request asks to evaluate an existing artifact against a standard. Not fact-finding (investigate).',
+    'test': 'The request asks to run tests or builds, or to write tests as the main deliverable.',
+    'docs': 'The request asks to write or edit documentation as the main deliverable. Not incidental text edits alongside other work (edit).',
+    'plan': 'The request asks to produce a plan, specification, or breakdown before implementation. Not weighing options in discussion (discuss).',
+    'investigate': 'The request asks to find facts or a cause and report back without changing anything. Not repairing (fix), not judging quality (review), not explaining how things work (explain).',
+}
+
+_QUESTIONS = {'v1': _PROPOSITIONS, 'v2': _PROPOSITIONS_V2}
+
+
+def build_questions(version='v1'):
+    """One `noul` question per intent, in catalog order."""
+    if version not in _QUESTIONS:
+        raise ValueError(f'unknown questions version: {version!r}')
+    table = _QUESTIONS[version]
+    return {intent: {'type': 'noul', 'instructions': table[intent]} for intent in INTENTS}
+
+
+def validate_scores(scores):
+    """True when scores cover exactly the 12 intents with finite 0..1 floats."""
+    if not isinstance(scores, dict) or set(scores) != set(INTENTS):
+        raise ValueError(f'scores keys must be exactly the 12 intents, got {sorted(scores) if isinstance(scores, dict) else type(scores)}')
+    for intent in INTENTS:
+        v = scores[intent]
+        if not isinstance(v, float) or not math.isfinite(v) or not 0.0 <= v <= 1.0:
+            raise ValueError(f'scores[{intent!r}] out of range: {v!r}')
+    return True
+
+
+def derive_label(scores):
+    """Mechanical {gold, secondary} from bands (bootstrap only, derived=true).
+
+    Mirrors the scoredLabel bands: secondary is every other intent at 0.4+;
+    a max at or below 0.2 abstains to null. Ties break by catalog order.
+    """
+    validate_scores(scores)
+    top = max(scores[intent] for intent in INTENTS)
+    if top <= 0.2:
+        return {'gold': None, 'secondary': [], 'derived': True}
+    gold = max(INTENTS, key=lambda i: (scores[i], -INTENTS.index(i)))
+    secondary = sorted((i for i in INTENTS if i != gold and scores[i] >= 0.4),
+                       key=lambda i: (-scores[i], INTENTS.index(i)))
+    return {'gold': gold, 'secondary': secondary, 'derived': True}
+
+
+def output_row(prompt, scores, backend, out_name, routing=None, questions_version='v1'):
+    """One trainer-contract row for a labeled prompt."""
+    if Path(out_name).name == PROTECTED_OUTPUT:
+        raise ValueError(f'protected output: refusing to overwrite {PROTECTED_OUTPUT}')
+    derived = derive_label(scores)
+    labeler = {'task': 'intent', 'target': 'scores', 'backend': backend,
+               'questions': questions_version, 'derived': True}
+    if routing is not None:
+        labeler['routing'] = routing
+    return {'id': prompt['id'], 'gold': derived['gold'], 'secondary': derived['secondary'],
+            'scores': {intent: scores[intent] for intent in INTENTS},
+            'labeler': labeler}
+
+
+def torch_backend(questions_version='v1'):
+    """Official PyTorch Laya backend (lazy import). Returns (fn, provenance)."""
+    import laya  # noqa: PLC0415
+    router = laya.Router()
+    questions = build_questions(questions_version)
+
+    def run(text):
+        started = time.monotonic()
+        result = router.predict(text, questions)
+        elapsed_ms = (time.monotonic() - started) * 1000.0
+        answers = result['answers']
+        scores = {intent: float(answers[intent]['noul']) for intent in INTENTS}
+        return scores, result['routing']['model'], elapsed_ms
+
+    provenance = {'package': 'laya', 'version': laya.__version__, 'questions': questions_version}
+    return run, provenance
+
+
+def mlx_backend(questions_version='v1'):
+    """Native MLX Laya port backend (lazy import, inference only)."""
+    import laya_mlx  # noqa: PLC0415
+    from laya_mlx import Router  # noqa: PLC0415
+    router = Router()
+    questions = build_questions(questions_version)
+
+    def run(text):
+        started = time.monotonic()
+        result = router.predict(text, questions)
+        elapsed_ms = (time.monotonic() - started) * 1000.0
+        answers = result['answers']
+        scores = {intent: float(answers[intent]['noul']) for intent in INTENTS}
+        return scores, result['routing']['model'], elapsed_ms
+
+    provenance = {'package': 'laya-mlx', 'version': laya_mlx.__version__, 'questions': questions_version}
+    return run, provenance
+
+
+def label_rows(rows, split, input_sha, out_path, manifest_path, backend_fn, backend_name, resume=False,
+               questions_version='v1'):
+    """Label explicit rows; returns a summary dict."""
+    out_path, manifest_path = Path(out_path), Path(manifest_path)
+    if out_path.name == PROTECTED_OUTPUT:
+        raise ValueError(f'protected output: refusing to overwrite {PROTECTED_OUTPUT}')
+    done = set()
+    if resume and out_path.exists():
+        for line in out_path.read_text(encoding='utf-8').splitlines():
+            if line.strip():
+                done.add(json.loads(line)['id'])
+        rows = [r for r in rows if r['id'] not in done]
+    routing, latencies, this_run = {}, [], 0
+    with out_path.open('a' if (resume and done) else 'w', encoding='utf-8') as fh:
+        for prompt in rows:
+            scores, routed, elapsed_ms = backend_fn(prompt['text'])
+            fh.write(json.dumps(output_row(prompt, scores, backend_name, out_path.name, routed,
+                                              questions_version)) + '\n')
+            routing[routed] = routing.get(routed, 0) + 1
+            latencies.append(elapsed_ms)
+            this_run += 1
+    total, full_routing = this_run, routing
+    if resume:
+        # The file is the source of truth: recount everything for cumulative totals.
+        full_routing = {}
+        total = 0
+        for line in out_path.read_text(encoding='utf-8').splitlines():
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            routed = (row.get('labeler') or {}).get('routing', 'unknown')
+            full_routing[routed] = full_routing.get(routed, 0) + 1
+            total += 1
+    manifest = {'backend': backend_name, 'split': split, 'questions': questions_version,
+                'labeled': total, 'thisRun': this_run, 'resumed': bool(resume and done),
+                'inputSha256': input_sha, 'routing': full_routing,
+                'latencyMs': {'p50': statistics.median(latencies) if latencies else 0.0,
+                              'p95': statistics.quantiles(latencies, n=100)[94] if len(latencies) >= 100 else (max(latencies) if latencies else 0.0)}}
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding='utf-8')
+    return manifest
+
+
+def label_split(data_dir, split, out_path, manifest_path, backend_fn, backend_name, questions_version='v1'):
+    """Label one prompts.jsonl split; returns a summary dict."""
+    if split not in ('train', 'validation'):
+        raise ValueError(f'split must be train or validation, got {split!r}')
+    data_dir = Path(data_dir)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+    todo = [r for r in rows if r.get('split') == split]
+    return label_rows(todo, split, hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                      out_path, manifest_path, backend_fn, backend_name,
+                      questions_version=questions_version)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--data-dir', required=True, help='private corpus dir (holds prompts.jsonl)')
+    parser.add_argument('--split', choices=['train', 'validation'], default='validation')
+    parser.add_argument('--out', required=True, help='output jsonl (never labels-intent-scores.jsonl)')
+    parser.add_argument('--manifest', required=True, help='provenance manifest json')
+    parser.add_argument('--backend', choices=['torch', 'mlx'], required=True)
+    parser.add_argument('--questions', choices=['v1', 'v2'], default='v1')
+    parser.add_argument('--limit', type=int, default=0, help='label only the first N rows (0 = all)')
+    parser.add_argument('--resume', action='store_true', help='skip ids already present in --out')
+    args = parser.parse_args(argv)
+    run, provenance = (torch_backend(args.questions) if args.backend == 'torch'
+                       else mlx_backend(args.questions))
+    print(json.dumps({'backend': args.backend, 'provenance': provenance}))
+
+    data_dir = Path(args.data_dir)
+    prompts_raw = (data_dir / 'prompts.jsonl').read_text(encoding='utf-8')
+    rows = [json.loads(line) for line in prompts_raw.splitlines() if line.strip()]
+    todo = [r for r in rows if r.get('split') == args.split]
+    if args.limit > 0:
+        todo = todo[:args.limit]
+    summary = label_rows(todo, args.split,
+                         hashlib.sha256(prompts_raw.encode('utf-8')).hexdigest(),
+                         args.out, args.manifest, run, args.backend, resume=args.resume,
+                         questions_version=args.questions)
+    summary['limit'] = args.limit
+    Path(args.manifest).write_text(json.dumps(summary, indent=2), encoding='utf-8')
+    print(json.dumps(summary))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())