create-pmos 0.46.0 → 0.48.0

package/workspace/lib/engine/pmos-model.js

@@ -7,6 +7,10 @@
  *   node pmos-model.js recommend "the work" [--skill x] [--agent claude]
  *   node pmos-model.js resolve [--skill x] [--tier fast|balanced|deep] [--agent claude]
  *   node pmos-model.js buckets                      evaluator models seen in evals.jsonl (calibration is per model, D40)
+ *   node pmos-model.js route "the prompt" [--skill x] [--source pm|agent|schedule] [--explain]
+ *                                                   route ANY prompt (prompt-model-routing): rules, then keyword routes. The Jev
+ *                                                   classification stage runs only in the workspace bridge, which holds the key.
+ *   node pmos-model.js bakeoff --labels <file>      replay PM-labelled prompts through each router and both baselines (R8–R10)
  *
  * PMOS routes on TIERS, never on a model name: the PM maps a tier to a model and an effort in
  * <state>/models/policy.json, and the defaults use the harness's own aliases (haiku, sonnet, opus) so they
@@ -37,6 +41,7 @@ const DEFAULT_POLICY = {
     { match: { skill: 'backlog-triage' }, tier: 'fast', why: 'sorting, not deciding' },
     { match: { skill: 'skill-discover' }, tier: 'balanced', why: 'judging relevance and safety of text found on the web' },
     { match: { skill: 'skill-generate' }, tier: 'balanced', why: 'drafting a skill from evidence in the repository' },
+    { match: { skill: 'live-notes' }, tier: 'balanced', why: 'placing talking points and writing short cited notes from the initiatives\' own text' },
     { match: { skill: 'gather-requirements' }, tier: 'balanced', why: 'reading many sources and judging what is a requirement' },
     { match: { skill: 'report-weekly' }, tier: 'balanced', why: 'a weekly synthesis' },
     { match: { skill: 'report-biweekly' }, tier: 'balanced', why: 'a two-week synthesis' },
@@ -80,6 +85,8 @@ function validate(p) {
     if (!r.match || (typeof r.match.skill !== 'string' && !Array.isArray(r.match.words))) errs.push(`route ${i}: match needs a skill or a list of words`);
     if (r.pinned !== undefined && typeof r.pinned !== 'boolean') errs.push(`route ${i}: pinned must be true or false`);
   });
+  if (p.version !== undefined && ![1, 2].includes(p.version)) errs.push('`version` must be 1 or 2');
+  if (p.router !== undefined) errs.push(...validateRouter(p.router));
   return errs;
 }
 function loadPolicy(root) {
@@ -198,10 +205,204 @@ function evaluatorNote(root, model) {
   return b[m] ? null : `'${m}' has no recorded evaluator verdicts here: its calibration starts at zero and the Review Gate stays advisory for it (D40).`;
 }
 
+// ── the prompt router (prompt-model-routing) ─────────────────────────────────────────────────
+/* A prompt gets a tier from the first stage that can decide it, and the answer NAMES that stage:
+ *   rules      an explicit choice, a schedule or skill tier, a pinned route, a long prompt
+ *   jev        a classification the workspace bridge obtained from TypeSafe under its gates, handed in as data
+ *   keywords   the D75 routes, unchanged
+ * This module makes no request. A classification arrives as an argument; without one the answer is `keywords`.
+ * Doubt goes UP a tier, never down; a pinned route is raised, never lowered. In a running session the answer
+ * is only a suggestion; spawn applies it as D75 §3 says. */
+const ROUTER_DEFAULTS = {
+  enabled: true,             // the keyword/rule router is always available; this only gates the Jev stage
+  shadow: false,             // true: compute and log, show nothing, apply nothing
+  jev_model: 'jev-1.13.0',   // a NAMED version from the vendor's models page, never the alias — a version change must miss the cache
+  confidence_floor: 0.6,     // below this the tier answer is doubtful and goes up one tier
+  stakes_threshold: 0.5,     // P(a wrong answer is costly) at or above this sets `deep`
+  closed_threshold: 0.65,    // P(closed-set sorting task) at or above this may allow `fast`
+  max_prompt_chars: 6000,    // what the bridge may send; longer prompts are truncated first
+  long_prompt_chars: 8000,   // a prompt this long is never routed below `balanced`
+  min_labels: 50,            // the bakeoff says "unmeasured" below this many labelled prompts (R9)
+  max_under_rate: 0.05,      // a router may be applied automatically only at or below this under-routing rate (R9)
+  rates: null,               // optional { fast, balanced, deep }: the PM's own cost per 1000 tokens; absent, cost_rank is used
+  jev_rate: null,            // optional: the PM's own cost per 1000 Jev tokens, so the router's cost counts against its savings
+};
+const ROUTER_KEYS = Object.keys(ROUTER_DEFAULTS);
+function validateRouter(r) {
+  if (!r || typeof r !== 'object' || Array.isArray(r)) return ['`router` must be an object'];
+  const errs = [];
+  for (const k of Object.keys(r)) if (!ROUTER_KEYS.includes(k)) errs.push(`router: unknown key '${k}'`);
+  for (const k of ['enabled', 'shadow']) if (r[k] !== undefined && typeof r[k] !== 'boolean') errs.push(`router.${k} must be true or false`);
+  if (r.jev_model !== undefined && (typeof r.jev_model !== 'string' || !MODEL_RE.test(r.jev_model))) errs.push('router.jev_model must be a plain version token like "jev-1.13.0"');
+  for (const k of ['confidence_floor', 'stakes_threshold', 'closed_threshold']) if (r[k] !== undefined && !(Number.isFinite(r[k]) && r[k] >= 0 && r[k] <= 1)) errs.push(`router.${k} must be a number from 0 to 1`);
+  if (r.min_labels !== undefined && !(Number.isInteger(r.min_labels) && r.min_labels >= 1 && r.min_labels <= 100000)) errs.push('router.min_labels must be a whole number of at least 1');
+  if (r.max_under_rate !== undefined && !(Number.isFinite(r.max_under_rate) && r.max_under_rate >= 0 && r.max_under_rate <= 1)) errs.push('router.max_under_rate must be a number from 0 to 1');
+  if (r.rates !== undefined && r.rates !== null && !(typeof r.rates === 'object' && !Array.isArray(r.rates) && TIERS.every((t) => Number.isFinite(r.rates[t]) && r.rates[t] >= 0) && Object.keys(r.rates).every((k) => TIERS.includes(k)))) errs.push('router.rates must give a number of 0 or more for each of fast, balanced and deep');
+  if (r.jev_rate !== undefined && r.jev_rate !== null && !(Number.isFinite(r.jev_rate) && r.jev_rate >= 0)) errs.push('router.jev_rate must be a number of 0 or more');
+  for (const k of ['max_prompt_chars', 'long_prompt_chars']) if (r[k] !== undefined && !(Number.isInteger(r[k]) && r[k] >= 200 && r[k] <= 60000)) errs.push(`router.${k} must be a whole number from 200 to 60000`);
+  return errs;
+}
+function routerConfig(policy) { return { ...ROUTER_DEFAULTS, ...((policy && policy.router) || {}) }; }
+
+const up = (t) => TIERS[Math.min(TIERS.indexOf(t) + 1, TIERS.length - 1)];
+const higher = (a, b) => (TIERS.indexOf(a) >= TIERS.indexOf(b) ? a : b);
+const sha = (t) => require('crypto').createHash('sha256').update(String(t)).digest('hex');
+
+/* classification (optional, from the bridge): { a:{tier,p}, b:{tier,p}, stakes:p, closed:p, model_asked, model_answered, tokens_in, tokens_out }
+ * a and b are the same tier question asked in two phrasings; p is the probability of the tier given. */
+function route(root, prompt, ctx = {}, classification = null) {
+  const text = String(prompt == null ? '' : prompt);
+  if (!text.trim() && !ctx.skill) return { ok: false, error: 'say what the prompt is, or name a skill' };
+  const loaded = loadPolicy(root);
+  if (!loaded.ok) return { ok: false, error: loaded.error };
+  const policy = loaded.policy; const cfg = routerConfig(policy);
+  const source = ['pm', 'agent', 'schedule'].includes(ctx.source) ? ctx.source : 'agent';
+  const kw = routeFor(policy, { skill: ctx.skill, text });
+  const done = (tier, stage, confidence, why, extra = {}) => {
+    const x = policy.tiers[tier];
+    return { ok: true, engine: 'claude', tier, model: x.model, effort: x.effort, confidence, stage, why, source, pinned: !!(kw && kw.pinned),
+      shadow: !!cfg.shadow, applies: ctx.running ? 'suggestion — the PM presses to switch' : 'at spawn', policy_source: loaded.source,
+      prompt_hash: sha(text), prompt_len: text.length, ...extra };
+  };
+  // stage 1 — rules
+  const ex = ctx.explicit || {};
+  const ruled = ex.tier || ctx.scheduleTier || ctx.skillTier;
+  let floor = null; let floorWhy = '';
+  if (kw && kw.pinned) { floor = kw.tier; floorWhy = 'review work is pinned (D40)'; }
+  if (text.length >= cfg.long_prompt_chars) { floor = floor ? higher(floor, 'balanced') : 'balanced'; floorWhy = floorWhy || 'a long prompt is never routed below balanced'; }
+  if (ruled) {
+    const t = ex.tier || ctx.scheduleTier || ctx.skillTier;
+    if (!TIERS.includes(t)) return { ok: false, error: `'${t}' is not a tier — one of ${TIERS.join(', ')}` };
+    if (kw && kw.pinned && TIERS.indexOf(t) < TIERS.indexOf(kw.tier)) return { ok: false, error: `'${ctx.skill || 'this work'}' is pinned to the ${kw.tier} tier (D40) — ${t} would lower it` };
+    return done(t, 'rules', 'rule', ex.tier ? 'a tier was chosen for this prompt' : ctx.scheduleTier ? "the schedule's tier" : "the skill's tier");
+  }
+  if (kw && kw.pinned) return done(kw.tier, 'rules', 'rule', kw.why || 'a pinned route');
+  // stage 2 — the Jev classification, when the bridge supplied one
+  if (cfg.enabled && classification && classification.a && classification.b && TIERS.includes(classification.a.tier) && TIERS.includes(classification.b.tier)) {
+    const c = classification; const signals = { a: c.a, b: c.b, stakes: c.stakes, closed: c.closed };
+    const agree = c.a.tier === c.b.tier;
+    let tier = agree ? c.a.tier : up(higher(c.a.tier, c.b.tier));
+    const lowP = Math.min(Number(c.a.p) || 0, Number(c.b.p) || 0);
+    let confidence = 'high'; const notes = [];
+    if (!agree) { confidence = 'unsure'; notes.push('the two phrasings of the tier question disagreed, so it goes up one tier'); }
+    else if (lowP < cfg.confidence_floor) { confidence = 'unsure'; tier = up(tier); notes.push(`the tier answer was below the confidence floor (${cfg.confidence_floor}), so it goes up one tier`); }
+    if (Number(c.stakes) >= cfg.stakes_threshold) { tier = 'deep'; notes.push('a wrong answer here could change a decision, a spec, a security rule or something irreversible'); }
+    const closedSet = Number(c.closed) >= cfg.closed_threshold && !(Number(c.stakes) >= cfg.stakes_threshold);
+    if (floor) tier = higher(tier, floor);
+    return done(tier, 'jev', confidence, notes.length ? notes.join('; ') : 'Jev classified the prompt and both phrasings agreed', { signals, closed_set: closedSet,
+      jev_model_asked: c.model_asked || null, jev_model_answered: c.model_answered || null, tokens_in: c.tokens_in ?? null, tokens_out: c.tokens_out ?? null });
+  }
+  // stage 3 — the D75 keyword routes, unchanged
+  let tier = kw ? kw.tier : 'balanced';
+  if (floor) tier = higher(tier, floor);
+  return done(tier, 'keywords', kw ? 'rule' : 'unsure', kw ? (kw.why || 'a keyword route matched') : 'no rule matched — balanced is the safe middle; say what is at stake if it should be deep', { matched: !!kw });
+}
+
+
+/* ── the bakeoff (prompt-model-routing R8–R10) ───────────────────────────────────────────────
+ * Replays a labelled prompt set through each router and two baselines. A LABEL is the cheapest tier whose output the
+ * PM accepted when reading the outputs of every tier blind. A row: { prompt, label, tokens?, classification?, router_ms? }.
+ *   prompt          the text, so the keyword routes can run (the file is the PM's, kept where the PM chooses)
+ *   label           fast | balanced | deep
+ *   tokens          one number, or { fast, balanced, deep }: what the work cost at each tier; absent, 1000
+ *   classification  a recorded Jev classification (as route() takes it); absent, the Jev router has no answer for the row
+ *   router_ms       the recorded delay of that classification
+ * Under-routing (a tier BELOW the label) is the costly mistake: the PM gets work they would not accept. Nothing here
+ * makes a request or fetches a price; a cost is tokens times the PM's own rate, or the tier's cost_rank. */
+const ROUTERS = ['keywords', 'jev', 'always-balanced', 'always-deep'];
+function readLabels(file) {
+  let raw; try { raw = fs.readFileSync(file, 'utf8'); } catch (e) { return { ok: false, error: `the labelled set could not be read — ${String((e && e.code) || e)}` }; }
+  const rows = []; const bad = [];
+  raw.split('\n').forEach((l, i) => {
+    if (!l.trim()) return;
+    let o; try { o = JSON.parse(l); } catch { bad.push(i + 1); return; }
+    if (!o || typeof o.prompt !== 'string' || !TIERS.includes(o.label)) { bad.push(i + 1); return; }
+    rows.push(o);
+  });
+  if (bad.length) return { ok: false, error: `line(s) ${bad.slice(0, 10).join(', ')} are not { "prompt": text, "label": fast|balanced|deep }` };
+  return { ok: true, rows };
+}
+const pct = (xs, q) => { if (!xs.length) return null; const s = xs.slice().sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.ceil(q * s.length) - 1)]; };
+function bakeoff(root, rows) {
+  const loaded = loadPolicy(root); if (!loaded.ok) return { ok: false, error: loaded.error };
+  const policy = loaded.policy; const cfg = routerConfig(policy);
+  const unit = cfg.rates ? 'the PM\'s rates per 1000 tokens' : 'cost_rank × tokens / 1000 (no rates set)';
+  const rate = (t) => (cfg.rates ? cfg.rates[t] : rank(policy, t));
+  const tokensAt = (row, t) => (row.tokens && typeof row.tokens === 'object' ? Number(row.tokens[t]) : Number(row.tokens)) || 1000;
+  const costOf = (row, t) => (tokensAt(row, t) / 1000) * rate(t);
+  const ix = (t) => TIERS.indexOf(t);
+  const pick = (name, row) => {
+    if (name === 'always-balanced') return { tier: 'balanced', ms: 0 };
+    if (name === 'always-deep') return { tier: 'deep', ms: 0 };
+    if (name === 'keywords') { const r = route(root, row.prompt, {}, null); return r.ok ? { tier: r.tier, ms: 0 } : null; }
+    if (!row.classification) return null;
+    const r = route(root, row.prompt, {}, row.classification);
+    if (!r.ok || r.stage !== 'jev') return null;
+    const jt = (Number(row.classification.tokens_in) || 0) + (Number(row.classification.tokens_out) || 0);
+    return { tier: r.tier, ms: Number.isFinite(row.router_ms) ? row.router_ms : null, routerCost: cfg.jev_rate != null ? (jt / 1000) * cfg.jev_rate : 0 };
+  };
+  const results = {};
+  for (const name of ROUTERS) {
+    let n = 0, under = 0, over = 0, exact = 0, cost = 0, base = 0, deepCost = 0, routerCost = 0; const ms = []; let missing = 0;
+    for (const row of rows) {
+      const p = pick(name, row);
+      if (!p) { missing++; continue; }
+      n++;
+      if (ix(p.tier) < ix(row.label)) under++; else if (ix(p.tier) > ix(row.label)) over++; else exact++;
+      cost += costOf(row, p.tier); base += costOf(row, 'balanced'); deepCost += costOf(row, 'deep'); routerCost += p.routerCost || 0;
+      if (p.ms != null) ms.push(p.ms);
+    }
+    const total = cost + routerCost;
+    results[name] = { n, missing, under, over, exact, under_rate: n ? under / n : null, over_rate: n ? over / n : null,
+      cost: Math.round(total * 1000) / 1000, router_cost: Math.round(routerCost * 1000) / 1000, router_cost_counted: name !== 'jev' || cfg.jev_rate != null,
+      vs_balanced: n && base ? Math.round((total / base - 1) * 1000) / 10 : null, vs_deep: n && deepCost ? Math.round((total / deepCost - 1) * 1000) / 10 : null,
+      router_ms_p50: pct(ms, 0.5), router_ms_p95: pct(ms, 0.95) };
+  }
+  const verdicts = {};
+  for (const name of ['keywords', 'jev']) {
+    const r = results[name]; const b = results['always-balanced'];
+    let verdict; let why;
+    if (r.n < cfg.min_labels) { verdict = 'unmeasured'; why = `${r.n} labelled prompt(s), fewer than ${cfg.min_labels}: suggestion only`; }
+    else if (r.under_rate > cfg.max_under_rate) { verdict = 'refused'; why = `under-routes ${(r.under_rate * 100).toFixed(1)}% of prompts, above the ${(cfg.max_under_rate * 100).toFixed(1)}% limit${r.cost < b.cost ? ' — it is cheaper, and that does not outweigh it' : ''}`; }
+    else if (!(r.cost < b.cost)) { verdict = 'not-cheaper'; why = 'it does not cost less than always-balanced, so routing buys nothing'; }
+    else { verdict = 'may-apply'; why = `under-routes ${(r.under_rate * 100).toFixed(1)}% (limit ${(cfg.max_under_rate * 100).toFixed(1)}%) and costs ${r.vs_balanced}% against always-balanced`; }
+    verdicts[name] = { verdict, why };
+  }
+  return { ok: true, labels: rows.length, unit, min_labels: cfg.min_labels, max_under_rate: cfg.max_under_rate, results, verdicts,
+    note: 'A may-apply verdict is evidence for the PM, not a switch: changing the policy stays the PM\'s edit (R10).' };
+}
+
+// What the router would do, written out for the PM. No prompt is involved.
+function explain(root) {
+  const loaded = loadPolicy(root); if (!loaded.ok) return { ok: false, error: loaded.error };
+  const cfg = routerConfig(loaded.policy);
+  return { ok: true, source: loaded.source, version: loaded.policy.version || 1, router: cfg, router_in_file: !!loaded.policy.router,
+    tiers: Object.fromEntries(TIERS.map((t) => [t, loaded.policy.tiers[t]])), routes: (loaded.policy.routes || []).length,
+    stages: ['rules (explicit, schedule, skill, pinned, long prompt)', cfg.enabled ? 'jev classification (only in the workspace bridge, only when its gates hold)' : 'jev classification (off in this policy)', 'keyword routes'] };
+}
+
+/* The routed-prompt ledger lines. A WHITELIST: only these keys are written, so the text of a prompt or of an
+ * answer cannot reach the file by being passed in by mistake. The hash and length stand in for the prompt. */
+const ROUTE_KEYS = ['source', 'stage', 'confidence', 'suggested_tier', 'applied_tier', 'overridden', 'shadow', 'jev_model_asked', 'jev_model_answered', 'tokens_in', 'tokens_out', 'router_ms', 'prompt_hash', 'prompt_len', 'skill'];
+function logRoute(root, r, extra = {}) {
+  const e = { kind: 'route', id: extra.id || sha(r.prompt_hash + ':' + Date.now() + ':' + Math.random()).slice(0, 16), suggested_tier: r.tier, applied_tier: extra.applied_tier ?? null, overridden: false,
+    source: r.source, stage: r.stage, confidence: r.confidence, shadow: r.shadow, jev_model_asked: r.jev_model_asked ?? null, jev_model_answered: r.jev_model_answered ?? null,
+    tokens_in: r.tokens_in ?? null, tokens_out: r.tokens_out ?? null, router_ms: extra.router_ms ?? null, prompt_hash: r.prompt_hash, prompt_len: r.prompt_len, skill: extra.skill || null };
+  const clean = { kind: e.kind, id: e.id }; for (const k of ROUTE_KEYS) clean[k] = e[k];
+  return logDecision(root, clean) ? e.id : null;
+}
+const OUTCOMES = ['accepted', 'overridden', 'rerun_higher', 'corrected'];
+function attachOutcome(root, id, o = {}) {
+  if (!id || !OUTCOMES.includes(o.outcome)) return false;
+  const e = { kind: 'route-outcome', ref: String(id).slice(0, 40), outcome: o.outcome };
+  if (o.applied_tier && TIERS.includes(o.applied_tier)) e.applied_tier = o.applied_tier;
+  return logDecision(root, e);
+}
+
 // ── CLI ──────────────────────────────────────────────────────────────────────────────────────
 function main(argv) {
   const a = argv.slice(2); const cmd = a[0]; const opt = {}; const rest = [];
-  for (let i = 1; i < a.length; i++) { if (a[i].startsWith('--')) { const k = a[i].slice(2); if (k === 'json') opt.json = true; else opt[k] = a[++i]; } else rest.push(a[i]); }
+  for (let i = 1; i < a.length; i++) { if (a[i].startsWith('--')) { const k = a[i].slice(2); if (k === 'json') opt.json = true; else if (k === 'explain') opt.explain = true; else opt[k] = a[++i]; } else rest.push(a[i]); }
   const root = path.resolve(opt.root || process.cwd());
   const out = (o, human) => { process.stdout.write(opt.json ? JSON.stringify(o, null, 2) + '\n' : human + '\n'); };
   if (!cmd || cmd === 'help' || cmd === '--help') { process.stdout.write('usage: pmos-model.js show | init | recommend "the work" [--skill x] | resolve [--skill x] [--tier t] | buckets   [--root <repo>] [--json]\n'); return cmd ? 0 : 1; }
@@ -225,15 +426,45 @@ function main(argv) {
     out({ ...r, launch }, `${r.tier} → ${r.model} (effort ${r.effort})${r.pinned ? ' · pinned' : ''}\n  why: ${r.why}${launch.state === 'no-model-control' ? `\n  note: '${opt.agent}' has no verified model control in PMOS` : ''}`);
     return 0;
   }
+  if (cmd === 'route' && opt.explain) {
+    const x = explain(root); if (!x.ok) { process.stderr.write(x.error + '\n'); return 2; }
+    out(x, `policy: ${x.source} (version ${x.version}${x.router_in_file ? '' : ', no router section — defaults'})\n  stages: ${x.stages.join(' → ')}\n  router: ${JSON.stringify(x.router)}\n  ${x.routes} keyword route(s); first match wins`);
+    return 0;
+  }
   if (cmd === 'resolve') {
     const r = resolve(root, { skill: opt.skill, text: opt.text, agent: opt.agent || 'claude', explicit: opt.tier ? { tier: opt.tier } : (opt.model ? { model: opt.model } : {}) });
     if (!r.ok) { process.stderr.write(r.error + '\n'); return 2; }
     out(r, r.tier ? `${r.tier} → ${r.model} (effort ${r.effort}) · decided by ${r.decidedBy} · ${r.state}` : 'not asked — no rule matched, so no flags (the harness default)');
     return 0;
   }
+  if (cmd === 'route') {
+    const prompt = rest.join(' ') || opt.text;
+    const t0 = Date.now();
+    const r = route(root, prompt, { skill: opt.skill, source: opt.source || 'agent', running: opt.running === 'true' });
+    if (!r.ok) { process.stderr.write(r.error + '\n'); return 2; }
+    const note = 'the Jev stage did not run: it lives in the workspace bridge, which holds the key';
+    const id = logRoute(root, r, { router_ms: Date.now() - t0, skill: opt.skill });
+    if (!id) process.stderr.write('(no decisions ledger kept: run `pmos-model init` to opt in)\n');
+    if (r.shadow) { out({ ok: true, shadow: true, id }, 'shadow mode: the suggestion was logged and is not shown'); return 0; }
+    out({ ...r, id, note }, `${r.tier} → ${r.model} (effort ${r.effort}) · stage ${r.stage} · ${r.confidence}${r.pinned ? ' · pinned' : ''}\n  why: ${r.why}\n  ${r.applies}\n  note: ${note}`);
+    return 0;
+  }
+  if (cmd === 'route-outcome') {
+    const okk = attachOutcome(root, rest[0], { outcome: opt.outcome, applied_tier: opt.tier });
+    if (!okk) { process.stderr.write(`usage: route-outcome <id> --outcome ${OUTCOMES.join('|')} [--tier fast|balanced|deep] (needs the decisions ledger)\n`); return 2; }
+    out({ ok: true }, 'recorded'); return 0;
+  }
+  if (cmd === 'bakeoff') {
+    if (!opt.labels) { process.stderr.write('usage: bakeoff --labels <file.jsonl> — one { "prompt", "label" } per line; keep the file where you choose, outside the repository\n'); return 2; }
+    const l = readLabels(path.resolve(opt.labels)); if (!l.ok) { process.stderr.write(l.error + '\n'); return 2; }
+    const b = bakeoff(root, l.rows); if (!b.ok) { process.stderr.write(b.error + '\n'); return 2; }
+    const row = (k) => { const r = b.results[k]; return `  ${k.padEnd(16)} n ${String(r.n).padStart(4)}  under ${r.under_rate == null ? '  —  ' : (r.under_rate * 100).toFixed(1).padStart(5) + '%'}  over ${r.over_rate == null ? '  —  ' : (r.over_rate * 100).toFixed(1).padStart(5) + '%'}  cost ${String(r.cost).padStart(9)}  vs balanced ${r.vs_balanced == null ? '—' : r.vs_balanced + '%'}  vs deep ${r.vs_deep == null ? '—' : r.vs_deep + '%'}${r.router_ms_p95 != null ? `  router p50 ${r.router_ms_p50} ms p95 ${r.router_ms_p95} ms` : ''}${r.router_cost_counted ? '' : '  (router cost not counted: no jev_rate)'}`; };
+    out(b, `${b.labels} labelled prompt(s); cost in ${b.unit}\n${ROUTERS.map(row).join('\n')}\n  keywords: ${b.verdicts.keywords.verdict} — ${b.verdicts.keywords.why}\n  jev:      ${b.verdicts.jev.verdict} — ${b.verdicts.jev.why}\n  ${b.note}`);
+    return 0;
+  }
   if (cmd === 'buckets') { const b = evaluatorBuckets(root); out(b, Object.keys(b).length ? Object.entries(b).sort((x, y) => y[1] - x[1]).map(([m, n]) => `  ${m}  ${n} verdict(s)`).join('\n') : 'no evaluator verdicts recorded'); return 0; }
   process.stderr.write(`unknown command '${cmd}'\n`); return 2;
 }
 
-module.exports = { DEFAULT_POLICY, TIERS, EFFORTS, MODEL_RE, validate, loadPolicy, routeFor, resolve, recommend, launchFor, logDecision, evaluatorBuckets, evaluatorNote, modelsDir, stateDir, LAUNCH };
+module.exports = { bakeoff, readLabels, ROUTERS, ROUTER_DEFAULTS, validateRouter, routerConfig, route, explain, logRoute, attachOutcome, OUTCOMES, ROUTE_KEYS, DEFAULT_POLICY, TIERS, EFFORTS, MODEL_RE, validate, loadPolicy, routeFor, resolve, recommend, launchFor, logDecision, evaluatorBuckets, evaluatorNote, modelsDir, stateDir, LAUNCH };
 if (require.main === module) process.exit(main(process.argv));

package/workspace/lib/engine/status-facts.js

@@ -0,0 +1,204 @@
+'use strict';
+/*
+ * PMOS status facts — three things the ladder cannot see, computed by code from git and the records.
+ *
+ *   since_approval  spec files (PRD, rubric, brief) with a commit AFTER the latest contract approval
+ *                   or the latest acceptance first appeared in git
+ *   coverage        requirement ids in the PRD that the rubric never cites, and ids the rubric cites
+ *                   that the PRD does not have
+ *   record_says     the record's free-text `status:` names a stage the derived ladder contradicts
+ *
+ * WHAT THESE ARE NOT. They are not step states and they are not gates. None of them can mark a step
+ * `done`, change the stage, change a colour or enter a problem count. They sit beside the ladder as
+ * flags with their basis (a commit, a file, a line), so the PM can look. A false `done` is worse than
+ * a false `pending`; a flag can only send someone to look.
+ *
+ * WHAT THEY NEVER WRITE. This module reads git and files. It appends to no ledger, edits no record
+ * and edits no document. Contract approval, acceptance, stop/hold/cancel, outcome verdicts, anchors
+ * and reviewer identity stay the PM's (D12, D70, D57).
+ *
+ * HONESTY. A check that cannot run reports `unknown` with the reason (a shallow clone, a row that is
+ * not in committed history, a rubric that cites no requirement id). It never reports a pass.
+ * Order comes from POSITION IN THE COMMIT LIST, never from clock time, as workflow-state.js does.
+ *
+ * Zero dependencies, stock Node. It runs in CI, in a terminal and inside the workspace server.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+// Same line shape the claims ledger uses for a PRD requirement (workspace/lib/claims.js REQ_RE), minus
+// the prefixes PMOS PRDs use for something else: Q for an open question and S for a story.
+const REQ_LINE = /^\s*[-*]\s+\*\*([A-PR-RT-Z]{1,3}\d{1,3})\b[^*]*\*\*/;
+const CITE_ID = /\b([A-PR-RT-Z]{1,3}\d{1,3})\b/g;
+const CAP = 20;
+
+function git(root, args) {
+  try {
+    return execFileSync('git', ['-C', root, ...args], {
+      encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 128 * 1024 * 1024,
+    });
+  } catch { return null; }
+}
+
+// ── history: one pass over the commits and one over the ledgers, cached per root + head ─────────
+const _hist = new Map();   // root -> { head, value }
+
+function ledgerRels(root, L) {
+  const rel = (f) => path.relative(root, path.join(L.runsDir, f)).split(path.sep).join('/');
+  return { runs: rel('agent-runs.jsonl'), acceptances: rel('acceptances.jsonl') };
+}
+
+function history(root, L) {
+  const head = (git(root, ['rev-parse', '--verify', '--quiet', 'HEAD']) || '').trim();
+  const hit = _hist.get(root);
+  if (hit && hit.head === head && head) return hit.value;
+  let value;
+  if (!head) value = { ok: false, reason: 'no commits yet' };
+  else if ((git(root, ['rev-parse', '--is-shallow-repository']) || '').trim() === 'true') {
+    value = { ok: false, reason: 'shallow clone — no history to order commits by (CI needs fetch-depth: 0)' };
+  } else {
+    const log = git(root, ['log', '--reverse', '--format=%x1e%H %ct%x1f%s%x1f', '--name-only', 'HEAD']);
+    if (log == null) value = { ok: false, reason: 'git history unreadable' };
+    else {
+      const shas = []; const dates = []; const subjects = []; const changed = [];
+      for (const chunk of log.split('\x1e')) {
+        if (!chunk.trim()) continue;
+        const parts = chunk.split('\x1f');
+        const [sha, ct] = (parts[0] || '').trim().split(/\s+/);
+        if (!sha) continue;
+        shas.push(sha);
+        dates.push(Number.isFinite(parseInt(ct, 10)) ? parseInt(ct, 10) : null);
+        subjects.push(String(parts[1] || '').replace(/\s+/g, ' ').trim());
+        changed.push(new Set(String(parts[2] || '').split('\n').map((s) => s.trim()).filter(Boolean)));
+      }
+      const index = new Map(); shas.forEach((s, i) => index.set(s, i));
+      // First commit that ADDED a line carrying the approval itself — not a line that merely names
+      // the run. A contract is approved by `"PM_approved": true`, an acceptance by `"accepted": true`.
+      const rels = ledgerRels(root, L);
+      const approvals = new Map();   // `${kind}\0${run_id}` -> sha
+      const plog = git(root, ['log', '--reverse', '--format=%x1e%H', '-p', '-U0', '--no-color', 'HEAD', '--', rels.runs, rels.acceptances]);
+      if (plog != null) {
+        for (const chunk of plog.split('\x1e')) {
+          if (!chunk.trim()) continue;
+          const lines = chunk.split('\n');
+          const sha = (lines.shift() || '').trim();
+          let file = null;
+          for (const ln of lines) {
+            if (ln.startsWith('+++ ')) { file = ln.slice(6).trim(); continue; }
+            if (!ln.startsWith('+') || ln.startsWith('+++')) continue;
+            const m = /"run_id"\s*:\s*"([^"]+)"/.exec(ln);
+            if (!m) continue;
+            let kind = null;
+            if (file === rels.runs && /"PM_approved"\s*:\s*true/.test(ln)) kind = 'contract';
+            else if (file === rels.acceptances && /"accepted"\s*:\s*true/.test(ln)) kind = 'acceptance';
+            if (!kind) continue;
+            const key = `${kind}\0${m[1]}`;
+            if (!approvals.has(key)) approvals.set(key, sha);
+          }
+        }
+      }
+      value = { ok: true, shas, dates, subjects, changed, index, approvals };
+    }
+  }
+  _hist.set(root, { head, value });
+  return value;
+}
+
+const norm = (p) => String(p || '').trim().replace(/^\.\//, '').replace(/^\/+/, '');
+
+// ── since_approval ──────────────────────────────────────────────────────────────────────────────
+function sinceApproval({ root, L, fm, runs, accepted }) {
+  const files = ['prd_path', 'rubric_path', 'brief_path'].map((k) => norm(fm[k])).filter((p) => p && !/^[<(…]/.test(p));
+  if (!files.length) return null;
+  const contractRun = [...(runs || [])].reverse().find((r) => r && r.pre_run_contract && r.pre_run_contract.PM_approved === true);
+  const acceptRow = [...(accepted || [])].reverse().find((a) => a && a.accepted === true);
+  if (!contractRun && !acceptRow) return null;
+
+  const h = history(root, L);
+  const out = { status: 'ok', approvals: [] };
+  const todo = [];
+  if (contractRun) todo.push({ kind: 'contract', run_id: contractRun.run_id });
+  if (acceptRow) todo.push({ kind: 'acceptance', run_id: acceptRow.run_id });
+  if (!h.ok) return { status: 'unknown', reason: h.reason, approvals: todo.map((t) => ({ ...t, status: 'unknown' })) };
+
+  for (const t of todo) {
+    const sha = t.run_id ? h.approvals.get(`${t.kind}\0${t.run_id}`) : undefined;
+    if (!sha) {
+      out.approvals.push({ ...t, status: 'unknown', reason: 'the approval row is not in committed history (or has no run_id)' });
+      continue;
+    }
+    const at = h.index.get(sha);
+    const changed = [];
+    for (let i = at + 1; i < h.shas.length; i++) {
+      for (const f of files) {
+        if (!h.changed[i].has(f)) continue;
+        changed.push({ file: f, sha: h.shas[i].slice(0, 12), date: h.dates[i] != null ? new Date(h.dates[i] * 1000).toISOString().slice(0, 10) : null, subject: h.subjects[i].slice(0, 90) });
+      }
+    }
+    out.approvals.push({
+      ...t, status: 'ok', approved_sha: sha.slice(0, 12), changed: changed.slice(0, CAP), changed_total: changed.length,
+    });
+  }
+  if (out.approvals.some((a) => a.status === 'unknown')) out.status = out.approvals.every((a) => a.status === 'unknown') ? 'unknown' : 'partial';
+  return out;
+}
+
+// ── coverage ────────────────────────────────────────────────────────────────────────────────────
+function readText(root, rel) { try { return fs.readFileSync(path.join(root, rel), 'utf8'); } catch { return null; } }
+
+function coverage({ root, fm }) {
+  const prd = norm(fm.prd_path); const rub = norm(fm.rubric_path);
+  if (!prd || !rub) return null;
+  const pt = readText(root, prd); const rt = readText(root, rub);
+  if (pt == null || rt == null) return { status: 'unknown', reason: 'the PRD or the rubric could not be read' };
+  const reqs = new Set();
+  for (const line of pt.split('\n')) { const m = REQ_LINE.exec(line); if (m) reqs.add(m[1]); }
+  if (!reqs.size) return { status: 'unknown', reason: 'the PRD lists no requirement ids' };
+  const cited = new Set();
+  for (const m of rt.matchAll(CITE_ID)) cited.add(m[1]);
+  // Only an id shaped like a requirement counts as a citation; a rubric that cites none says
+  // nothing about coverage either way, and is reported as undecidable rather than as all-missing.
+  const citedReqLike = [...cited].filter((id) => reqs.has(id));
+  if (!citedReqLike.length) return { status: 'unknown', reason: 'the rubric cites no requirement id of this PRD, so coverage is undecidable by id' };
+  const uncited = [...reqs].filter((id) => !cited.has(id));
+  const orphan = [...cited].filter((id) => /^R\d{1,3}$/.test(id) && !reqs.has(id));
+  return { status: 'ok', requirements: reqs.size, uncited, orphan_citations: orphan, basis: 'cited anywhere in the rubric, which is not the same as graded' };
+}
+
+// ── record_says ─────────────────────────────────────────────────────────────────────────────────
+const RANK = { intake: 0, spec: 1, contract: 2, build: 3, eval: 4, accepted: 5 };
+const STAGE_WORD = /^\s*(intake|spec|contract|build|eval|accepted)\b/i;
+
+function recordSays({ fm, lane, stage }) {
+  if (lane === 'chore') return null;                       // for a chore, status: IS the intent
+  const prose = String(fm.status || '').trim();
+  const m = STAGE_WORD.exec(prose);
+  if (!m) return null;                                     // no stage word: nothing to compare
+  const claimed = m[1].toLowerCase();
+  const d = RANK[stage]; const c = RANK[claimed];
+  if (d == null || c == null) return null;
+  // Conservative on purpose. The prose is free text and the ladder's `eval` follows from any run
+  // being logged, so "build in progress" beside `eval` is not a contradiction. What is flagged:
+  //   · the prose claims acceptance and no acceptance row exists (the lie that matters most),
+  //   · the prose sits two or more stages BEHIND the ladder, or says anything but accepted on an
+  //     accepted initiative.
+  let reason = null;
+  if (claimed === 'accepted' && stage !== 'accepted') reason = `says accepted, but there is no acceptance row (the ladder is at ${stage})`;
+  else if (stage === 'accepted' && claimed !== 'accepted') reason = `says ${claimed}, but the PM accepted this initiative`;
+  else if (d - c >= 2) reason = `says ${claimed}, but the ladder is at ${stage}`;
+  if (!reason) return null;
+  return { status: 'mismatch', claimed, derived: stage, reason, quote: prose.slice(0, 120) };
+}
+
+function compute(c) {
+  const { root, L, fm, lane, stage } = c;
+  let since = null; let cov = null; let says = null;
+  try { since = sinceApproval(c); } catch (e) { since = { status: 'unknown', reason: `since-approval check failed: ${e.message}` }; }
+  try { cov = lane === 'feature' ? coverage({ root, fm }) : null; } catch (e) { cov = { status: 'unknown', reason: `coverage check failed: ${e.message}` }; }
+  try { says = recordSays({ fm, lane, stage }); } catch { says = null; }
+  return { since_approval: since, coverage: cov, record_says: says };
+}
+
+module.exports = { compute, sinceApproval, coverage, recordSays, history };

package/workspace/lib/engine/workflow-state.js

@@ -54,6 +54,13 @@ const anchors = require('./anchor-resolve');
  * server. It is shared now, for the same reason `anchor-resolve` is.
  */
 const delivery = require('./delivery-index');
+/*
+ * Three flags the ladder cannot see (status-from-artifacts, Run 1): a spec file changed after an
+ * approval, requirement ids the rubric never cites, and `status:` prose the ladder contradicts. They
+ * sit BESIDE the steps. They never change a step, the stage, a colour or a count, and they write
+ * nothing. See scripts/status-facts.js.
+ */
+const statusFacts = require('./status-facts');
 
 // The OKR documents and the budget registry are read once per root, not once per initiative: the
 // board derives ~100 ladders in a single pass and re-reading the same three files each time is the
@@ -1235,10 +1242,15 @@ function computeWorkflowDerived(id, opts) {
     || steps.find((s) => s.status === 'pending' && s.next)
     || null;
 
+  // `opts.facts === false` exists so a test can prove the flags change nothing else in the ladder.
+  const facts = (opts && opts.facts === false) ? {}
+    : statusFacts.compute({ root, L, id, fm, lane: effLane, stage, runs, accepted });
+
   return {
     ok: true, id, lane, laneKnown: knownLane, legacy, in_change: inChange,
     last_activity: lastActivity, accepted: accepted.length > 0, stage, record: recRel,
     title: fm.title || id, anchor: fm.parent_kr || null, profile: 'product-code',
+    ...facts,
     steps, blockers: blockers.map((s) => s.key), gate_failures: gateFailures.map((s) => s.rule || s.key),
     next: next ? { step: next.key, why: next.evidence, action: next.next || next.evidence } : null,
     range: range.ok ? { base: range.base, commits: range.shas.length, whole_history: !!range.wholeHistory } : { error: range.reason },
@@ -1390,6 +1402,44 @@ function brief(s, n) {
   return t.length > n ? `${t.slice(0, n - 1)}…` : t;
 }
 
+/**
+ * The three status-from-artifacts lines, one plain sentence each with its basis. A flag, never a
+ * step: nothing here is a state, and a missing basis prints nothing (never a pass).
+ */
+function renderFacts(w) {
+  const out = [];
+  const sa = w.since_approval;
+  if (sa) {
+    if (sa.status === 'unknown') out.push(`  ${c(90, '?')} since approval ${c(90, `cannot tell — ${sa.reason}`)}`);
+    else {
+      for (const a of sa.approvals) {
+        const what = a.kind === 'contract' ? 'the contract was approved' : 'the PM accepted';
+        if (a.status === 'unknown') out.push(`  ${c(90, '?')} since approval ${c(90, `${what} (${a.run_id || 'no run id'}) — ${a.reason}`)}`);
+        else if (!a.changed.length) out.push(`  ${c(90, '·')} since approval ${c(90, `no spec file changed since ${what} (${a.approved_sha})`)}`);
+        else {
+          const files = [...new Set(a.changed.map((x) => x.file))].join(', ');
+          const last = a.changed[a.changed.length - 1];
+          out.push(`  ${c(33, '!')} since approval ${files} changed after ${what} (${a.approved_sha}); ${a.changed_total} commit(s), latest ${last.sha} ${last.date || ''}`);
+        }
+      }
+    }
+  }
+  const cv = w.coverage;
+  if (cv) {
+    if (cv.status === 'unknown') out.push(`  ${c(90, '?')} coverage      ${c(90, `cannot tell — ${cv.reason}`)}`);
+    else if (!cv.uncited.length && !cv.orphan_citations.length) out.push(`  ${c(90, '·')} coverage      ${c(90, `all ${cv.requirements} requirement ids are cited in the rubric`)}`);
+    else {
+      const bits = [];
+      if (cv.uncited.length) bits.push(`${cv.uncited.join(', ')} not cited in the rubric`);
+      if (cv.orphan_citations.length) bits.push(`${cv.orphan_citations.join(', ')} cited but not in the PRD`);
+      out.push(`  ${c(33, '!')} coverage      ${bits.join('; ')} (cited is not graded)`);
+    }
+  }
+  const rs = w.record_says;
+  if (rs) out.push(`  ${c(33, '!')} record says    status: ${rs.reason}`);
+  return out;
+}
+
 function renderLadder(w) {
   if (!w.ok) return c(31, `✗ ${w.error}`);
   const out = [];
@@ -1403,6 +1453,8 @@ function renderLadder(w) {
     out.push(`  ${mark(s.status)} ${s.status === 'na' || s.status === 'pending' ? c(90, label) : label} ${brief(s.evidence, 110)}`);
     if (s.next) out.push(`    ${c(90, '→')} ${c(s.status === 'blocked' ? 33 : 90, brief(s.next, 150))}`);
   }
+  const flags = renderFacts(w);
+  if (flags.length) { out.push(`  ${c(90, '─'.repeat(78))}`); out.push(...flags); }
   out.push('');
   if (w.next) out.push(`  ${c(1, 'next:')} ${w.next.action}`);
   else out.push(`  ${c(32, 'nothing blocking — the ladder is complete')}`);