@golden-frijoles/kit 0.7.0 → 0.8.0

package/dist/jev-eval.mjs

@@ -0,0 +1,369 @@
+#!/usr/bin/env node
+// jev-eval.mjs — the labelled eval set for the Jev guards, and the rot guard on shadow mode
+// (jev-semantic-guards S1.4, D8).
+//
+//   node scripts/jev-eval.mjs                 offline (CI): replay every fixture's RECORDED Jev answers
+//                                             through the real judge, assert the decision still matches the
+//                                             recording, and fail if any rail is in shadow past its expiry.
+//   node scripts/jev-eval.mjs --live          re-ask Jev for every fixture, REWRITE the recordings, and print
+//                                             accuracy against the labels — regex vs Jev, per rail/family.
+//   node scripts/jev-eval.mjs --rail prose    limit to one rail.
+//   node scripts/jev-eval.mjs --live --limit 10
+//                                             the SETUP PROOF: ask Jev for only the first n fixtures of each
+//                                             rail, print per-rail agreement with the labels, and WRITE
+//                                             NOTHING. A partial run must never rewrite the committed
+//                                             recordings (they would then cover n fixtures, not all), so
+//                                             --limit is refused without --live and never touches the file.
+//                                             It still needs `jev.egress` true and TYPESAFE_API_KEY.
+//
+// Why offline replay exists: a model bump, a threshold change or an edit to a judge's decide logic must
+// show up as a red CI run, not as a quietly different verdict on the next PR. Why --live exists: the
+// recordings are only as current as the model that produced them — run it before bumping `model`.
+//
+// The rot guard: a rail in `shadow` past `shadowExpires` fails this script, so shadow cannot quietly become
+// the permanent "regex and Jev both" the product owner ruled out.
+//
+// Zero deps — Node 18+.
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadJevConfig, parseJevConfig, RAILS, readApiKey, repoRoot } from './lib/jev.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const FIXTURES_PATH = join(__dirname, 'jev-eval.fixtures.json');
+
+/** Shadow is a short, expiring measurement: at most this many days out, ever. */
+export const MAX_SHADOW_DAYS = 21;
+/** A judge that exists must be proven on at least this many labelled cases (S1.4 acceptance). */
+export const MIN_FIXTURES = 30;
+
+const addDays = (ymd, n) =>
+  new Date(Date.parse(`${ymd}T00:00:00Z`) + n * 86_400_000).toISOString().slice(0, 10);
+
+/**
+ * Shadow rails that fail the rot guard. Pure. `today` is the UTC date, YYYY-MM-DD (CI runs in UTC).
+ * Past its date is expired; more than MAX_SHADOW_DAYS out is refused too — `2099-01-01` would otherwise
+ * be a permanent shadow wearing an expiry date, and the gap only ever shrinks, so the cap is safe.
+ */
+export function expiredShadowRails(config, today) {
+  const cap = addDays(today, MAX_SHADOW_DAYS);
+  return Object.entries(config.rails)
+    .filter(
+      ([, r]) => r.mode === 'shadow' && r.shadowExpires && (r.shadowExpires < today || r.shadowExpires > cap)
+    )
+    .map(([name, r]) => ({
+      rail: name,
+      shadowExpires: r.shadowExpires,
+      why: r.shadowExpires < today ? 'past its shadowExpires' : `more than ${MAX_SHADOW_DAYS} days out`,
+    }));
+}
+
+/**
+ * Coverage failures: fixtures with no judge to replay them (a renamed judge must not turn CI green), and a
+ * judge with fewer than MIN_FIXTURES labelled cases. Pure.
+ */
+export function coverageFailures(fixtures, rails) {
+  const out = [];
+  for (const name of RAILS) {
+    const n = (fixtures[name] ?? []).length;
+    if (!rails[name] && n)
+      out.push(`${name}: ${n} fixture(s) but no judge to replay them — was the judge renamed?`);
+    if (rails[name] && n < MIN_FIXTURES)
+      out.push(`${name}: only ${n} labelled fixture(s); a judge needs ≥${MIN_FIXTURES}`);
+  }
+  return out;
+}
+
+/** An `ask` that answers ONLY from a recording — a missing answer is could-not-look, i.e. a stale fixture. */
+export function replayAsk(recorded) {
+  return async ({ questions }) => {
+    const ids = Object.keys(questions);
+    const missing = ids.filter((id) => !recorded?.answers?.[id]);
+    if (missing.length)
+      return {
+        ok: false,
+        state: 'could-not-look',
+        error: `no recording for ${missing.slice(0, 3).join(', ')}`,
+      };
+    return {
+      ok: true,
+      answers: Object.fromEntries(ids.map((id) => [id, recorded.answers[id]])),
+      usage: null,
+      model: recorded.model,
+    };
+  };
+}
+
+/**
+ * An `ask` that forwards to live Jev and captures every answer, so the recording can be rewritten. A
+ * could-not-look is RECORDED as a failure: the judge would fall back to the regex, and scoring that as
+ * Jev's answer — or baking it into the recording — would corrupt the flip gate (fresh review, PR #34).
+ */
+function recordingAsk(ask, sink) {
+  return async (req) => {
+    const r = await ask(req);
+    if (r.ok) {
+      Object.assign(sink.answers, r.answers);
+      sink.model = r.model;
+    } else sink.errors.push(r.error);
+    return r;
+  };
+}
+
+const SEMANTIC_CODES = [
+  'unsupported-fix-claim',
+  'invented-beneficiary',
+  'flag-state-claim',
+  'invented-commitment',
+];
+const sortedCodes = (findings) =>
+  [...new Set((findings ?? []).map((f) => f.code).filter((c) => SEMANTIC_CODES.includes(c)))].sort();
+
+/**
+ * The rails this harness can evaluate. Each judge is looked up by name so a rail whose judge has not
+ * landed yet is SKIPPED loudly instead of failing the import.
+ */
+export async function loadRails() {
+  const rails = {};
+  const review = await import('./lib/review-guard.mjs');
+  if (typeof review.judgeReviewOutput === 'function')
+    rails.review = {
+      run: (fx, deps) => review.judgeReviewOutput(fx.text, {}, deps),
+      regex: (fx) => review.assertReviewOutput(fx.text).ok,
+      predicted: (d) => d.ok,
+      expected: (fx) => fx.label,
+      summary: (d) => ({ ok: d.ok, decider: d.decider }),
+    };
+  const prose = await import('./lib/prose-guard.mjs');
+  if (typeof prose.judgeProse === 'function')
+    rails.prose = {
+      run: (fx, deps) => prose.judgeProse(fx.draft, fx.evidence ?? {}, deps),
+      regex: (fx) => sortedCodes(prose.checkProse(fx.draft, fx.evidence ?? {}).findings),
+      predicted: (d) => sortedCodes(d.findings),
+      expected: (fx) => [...fx.label].sort(),
+      summary: (d) => ({ codes: sortedCodes(d.findings), decider: d.decider }),
+    };
+  return rails;
+}
+
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+/**
+ * Jev config for evaluation: the rail forced to `jev` so Jev's answer — not the regex — is what is scored.
+ *
+ * `egress` is forced to `true` too — for REPLAY, which sends nothing; `evaluate` refuses a live run without a
+ * committed `true` — for the same reason `mode` is forced (golden-frijoles-plugin D12,
+ * S5.4): this harness measures what Jev WOULD decide against a replayed or live answer, never gated by
+ * whether this checkout's OWN `jev.config.json` has answered the egress question yet. `projectRoot()`
+ * resolves to `template/` for a script run as `node template/scripts/…` (D2's documented copied-mode
+ * root for this repo's own dogfooding), so `template/jev.config.json`'s tri-state `egress` IS this
+ * process's live config — and its new default (`null`, unanswered) would otherwise score every fixture
+ * as if the rail were off, which is a fact about this repo's own config, not about Jev's accuracy.
+ */
+const evalConfig = (base, rail) =>
+  parseJevConfig({
+    model: base.model,
+    egress: true,
+    rails: { ...base.rails, [rail]: { ...base.rails[rail], mode: 'jev', shadowExpires: null } },
+  });
+
+/**
+ * Evaluate. Returns { failures, report, fixtures } — pure over its deps apart from the judge calls.
+ * live=false replays recordings; live=true re-asks through deps.ask and rewrites them.
+ */
+export async function evaluate({
+  fixtures,
+  rails,
+  config,
+  live = false,
+  ask = null,
+  only = null,
+  limit = null,
+}) {
+  // Replay forces egress on (evalConfig) because it sends nothing. LIVE sends every fixture to TypeSafe, so it needs
+  // the project's explicit yes: `egress: true`, never null (unanswered) or false (D12; cross-review of #50 — the
+  // CLI refused this, but a caller of this export did not).
+  if (live && config?.egress !== true)
+    throw new Error(
+      `jev-eval live: egress is ${JSON.stringify(config?.egress ?? null)}, not true — refusing to send fixtures to Jev.`
+    );
+  const failures = [];
+  const report = {};
+  for (const [name, rail] of Object.entries(rails)) {
+    if (only && only !== name) continue;
+    const cases = limit == null ? (fixtures[name] ?? []) : (fixtures[name] ?? []).slice(0, limit);
+    const cfg = evalConfig(config, name);
+    const tally = { n: cases.length, jevRight: 0, regexRight: 0, disagreements: 0, families: {} };
+    for (const fx of cases) {
+      const sink = { answers: {}, model: null, errors: [] };
+      const deps = {
+        config: cfg,
+        key: 'eval',
+        log: () => {},
+        ask: live ? recordingAsk(ask, sink) : replayAsk(fx.recorded),
+      };
+      // A recording answers for the model that produced it. Replaying it under a bumped `model` would pass
+      // while proving nothing about the new one (codex, PR #34): re-record with --live first.
+      if (!live && fx.recorded?.model !== config.model) {
+        failures.push(
+          `${name}/${fx.id}: recorded by ${fx.recorded?.model ?? 'nothing'}, config pins ${config.model} — run --live`
+        );
+        continue;
+      }
+      const decision = await rail.run(fx, deps);
+      const summary = rail.summary(decision);
+      if (live && sink.errors.length) {
+        failures.push(
+          `${name}/${fx.id}: jev could not look (${sink.errors[0]}) — not scored, recording kept`
+        );
+        tally.n--;
+        continue;
+      }
+      if (live) {
+        // A draft the judge needed no answers for (e.g. a heading-only unit, which is never asked about) is
+        // recorded as answered by the pinned model with no answers — replaying it asks nothing.
+        fx.recorded = { model: sink.model ?? cfg.model, answers: sink.answers };
+        fx.decision = summary;
+      } else if (!same(summary, fx.decision)) {
+        failures.push(
+          `${name}/${fx.id}: replay gave ${JSON.stringify(summary)}, recorded ${JSON.stringify(fx.decision)}`
+        );
+      }
+      const expected = rail.expected(fx);
+      const jevRight = same(rail.predicted(decision), expected);
+      const regexRight = same(rail.regex(fx), expected);
+      tally.jevRight += jevRight;
+      tally.regexRight += regexRight;
+      tally.disagreements += !same(rail.predicted(decision), rail.regex(fx));
+      if (name === 'prose')
+        for (const code of SEMANTIC_CODES) {
+          const f = (tally.families[code] ??= { jevRight: 0, regexRight: 0, n: 0 });
+          const want = expected.includes(code);
+          f.n++;
+          f.jevRight += rail.predicted(decision).includes(code) === want;
+          f.regexRight += rail.regex(fx).includes(code) === want;
+        }
+    }
+    report[name] = tally;
+  }
+  return { failures, report, fixtures };
+}
+
+const pct = (a, n) => (n ? `${((100 * a) / n).toFixed(1)}%` : 'n/a');
+
+export function formatReport(report) {
+  const lines = [];
+  for (const [rail, t] of Object.entries(report)) {
+    lines.push(
+      `${rail}: ${t.n} labelled · jev ${pct(t.jevRight, t.n)} · regex ${pct(t.regexRight, t.n)} · ${t.disagreements} disagreement(s)`
+    );
+    for (const [code, f] of Object.entries(t.families))
+      lines.push(`  ${code.padEnd(24)} jev ${pct(f.jevRight, f.n)} · regex ${pct(f.regexRight, f.n)}`);
+  }
+  return lines.join('\n');
+}
+
+/** The one line that refuses a live run, or null when it may go. Egress first, and the key is not even read until the yes is in. */
+export function liveRefusal(config, getKey) {
+  if (config.egress !== true)
+    // false: no text leaves this machine; null: nobody has said yes yet (D12). `--live` sends fixtures.
+    return `jev-eval --live: jev.egress is ${JSON.stringify(config.egress)}, not true — refusing to send fixtures to Jev. Say yes with \`gf-kit config set jev.egress true\` first.`;
+  if (!getKey())
+    return 'jev-eval --live needs TYPESAFE_API_KEY: put TYPESAFE_API_KEY=… in .env.local (or the environment).';
+  return null;
+}
+
+/** Pure — `--limit <n>` as a positive integer, null when absent, or { error } when malformed. */
+export function parseLimit(argv) {
+  const ix = argv.indexOf('--limit');
+  if (ix < 0) return null;
+  const n = Number(argv[ix + 1]);
+  if (!/^\d+$/.test(argv[ix + 1] ?? '') || n < 1) return { error: '--limit needs a positive whole number' };
+  return n;
+}
+
+/**
+ * The CLI, with every side effect injected so a spec can watch it: returns the exit code. `io` carries
+ * { root, config, fixtures, rails, key, makeAsk, writeFixtures, stdout, stderr, today }.
+ */
+export async function run(argv, io = {}) {
+  const { config, fixtures, rails, stdout, stderr } = io;
+  const live = argv.includes('--live');
+  const railIx = argv.indexOf('--rail');
+  const only = railIx >= 0 ? argv[railIx + 1] : null;
+  if (railIx >= 0 && !RAILS.includes(only)) {
+    stderr(`jev-eval: --rail must be one of ${RAILS.join(', ')}\n`);
+    return 2;
+  }
+  const limit = parseLimit(argv);
+  if (limit?.error) {
+    stderr(`jev-eval: ${limit.error}\n`);
+    return 2;
+  }
+  if (limit != null && !live) {
+    stderr('jev-eval: --limit only makes sense with --live (a setup proof against real Jev).\n');
+    return 2;
+  }
+  const expired = expiredShadowRails(config, io.today ?? new Date().toISOString().slice(0, 10));
+  for (const name of ['review', 'prose'])
+    if (!rails[name]) stdout(`${name}: no judge in this checkout yet — skipped\n`);
+
+  let ask = null;
+  if (live) {
+    const refusal = liveRefusal(config, io.key);
+    if (refusal) {
+      stderr(`${refusal}\n`);
+      return 2;
+    }
+    ask = io.makeAsk({ key: io.key(), model: config.model });
+  }
+
+  const { failures, report } = await evaluate({ fixtures, rails, config, live, ask, only, limit });
+  // A partial run is not held to the per-rail fixture floor: that floor is about the committed set.
+  if (limit == null) failures.push(...coverageFailures(fixtures, rails));
+  const n = Object.values(report).reduce((s, t) => s + t.n, 0);
+  if (live && limit != null) {
+    stdout(
+      `live proof: asked Jev for the first ${limit} fixture(s) of each rail (${n} scored) against ${config.model}; ` +
+        'nothing written — the committed recordings are untouched.\n'
+    );
+  } else if (live && failures.length) {
+    stderr('jev-eval --live: failures below — the recordings were NOT rewritten.\n');
+  } else if (live) {
+    io.writeFixtures(fixtures);
+    stdout(`live: re-scored ${n} fixtures against ${config.model}; recordings rewritten.\n`);
+  } else {
+    stdout(`offline: ${n - failures.length}/${n} fixtures match recordings\n`);
+  }
+  stdout(`${formatReport(report)}\n`);
+  for (const f of failures) stderr(`✗ ${f}\n`);
+  for (const e of expired)
+    stderr(`✗ rails.${e.rail} is in shadow ${e.why} (${e.shadowExpires}) — promote it to jev or set it off.\n`);
+  return failures.length || expired.length ? 1 : 0;
+}
+
+async function main() {
+  const root = repoRoot();
+  const { askJev } = await import('./lib/jev.mjs');
+  const code = await run(process.argv.slice(2), {
+    root,
+    config: loadJevConfig({ root }),
+    fixtures: JSON.parse(readFileSync(FIXTURES_PATH, 'utf8')),
+    rails: await loadRails(),
+    key: () => readApiKey({ root }),
+    makeAsk: ({ key, model }) => (req) => askJev(req, { key, model }),
+    writeFixtures: (fx) => writeFileSync(FIXTURES_PATH, `${JSON.stringify(fx, null, 2)}\n`),
+    stdout: (t) => process.stdout.write(t),
+    stderr: (t) => process.stderr.write(t),
+    today: new Date().toISOString().slice(0, 10),
+  });
+  if (code) process.exit(code);
+}
+
+const isMain = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain)
+  main().catch((e) => {
+    process.stderr.write(`jev-eval: ${e.message}\n`);
+    process.exit(1);
+  });

package/dist/lib/config-registry.mjs

@@ -116,7 +116,7 @@ export const REGISTRY = Object.freeze([
     module: 'Operate',
     askWhen: 'first-report',
     default: null,
-    question: 'Where should standups and recaps go (Telegram, Slack, or print to the terminal)? Secrets stay in .env.local.',
+    question: 'Where should standups and recaps go: Telegram (the scheduled reports post there), or print to the terminal? Slack works for test and ad-hoc messages only, not the scheduled reports. Secrets stay in .env.local.',
   },
   {
     key: 'deploy.vercelProject',

package/dist/lib/jev.mjs

@@ -136,7 +136,10 @@ export function loadJevConfig({ root = repoRoot(), read = readFileSync, exists =
   });
   // Absent everywhere → the defaults (every rail off). A PRESENT legacy file holding JSON null is malformed: the
   // parser throws.
-  if (!present) return parseJevConfig({});
+  // No config at all is UNANSWERED, not "yes" (distribute-what-we-use D7). parseJevConfig({}) keeps
+  // egress:true for a legacy FILE that never mentions it, but a stranger with no file has told us nothing,
+  // and D12's promise is that nothing leaves the machine before an explicit `egress: true`.
+  if (!present) return parseJevConfig({ egress: null });
   // D12: a section that never gives egress a non-null value is UNANSWERED, never `true`. readSection treats a
   // `null` in golden-frijoles.config.json as unset, so without this a new-file `egress: null` (or a migrated
   // template config) reached parseJevConfig as a MISSING key and became `true` — sending with nobody's yes
@@ -147,7 +150,7 @@ export function loadJevConfig({ root = repoRoot(), read = readFileSync, exists =
 }
 
 /** Parse `KEY=value` lines. Enough for .env.local; quotes stripped. */
-function envFileValue(text, key) {
+export function envFileValue(text, key) {
   for (const line of String(text).split('\n')) {
     const m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
     if (m && m[1] === key) return m[2].trim().replace(/^(['"])(.*)\1$/, '$2') || null;
@@ -183,10 +186,12 @@ export function readApiKey({
  */
 export function effectiveMode(config, rail, { key } = {}) {
   const configured = config.rails[rail].mode;
-  if (configured === 'off') return { mode: 'off', configured, why: 'configured off' };
-  // Unanswered (D12): distinct from a deliberate `false` so the caller can ask once instead of
-  // staying silently off forever. `unanswered: true` is jevContext's cue to fire the D11 protocol.
+  // Unanswered (D12) is decided FIRST, before the rail's mode: every rail defaults to `off`, so checking
+  // the mode first made the ask unreachable for exactly the user it was written for (D7). Distinct from a
+  // deliberate `false` so the caller can ask once instead of staying silently off forever;
+  // `unanswered: true` is jevContext's cue to fire the D11 protocol. Nothing is sent either way.
   if (config.egress === null) return { mode: 'off', configured, why: 'egress not answered', unanswered: true };
+  if (configured === 'off') return { mode: 'off', configured, why: 'configured off' };
   if (!config.egress) return { mode: 'off', configured, why: 'egress disabled (jev.egress: false)' };
   if (!key) return { mode: 'off', configured, why: 'no TYPESAFE_API_KEY' };
   return { mode: configured, configured, why: `configured ${configured}` };

package/dist/lib/notification-text.mjs

@@ -0,0 +1,41 @@
+// Cross-channel notification text rules.
+//
+// Telegram HTML and Slack mrkdwn reserve the same three characters and require the same byte-for-
+// byte substitutions. Keeping the expansion-aware truncation here makes "fits before escaping"
+// impossible in either channel. Platform modules own only their distinct hard limit and the name
+// their renderer uses for this same operation.
+
+/** Escape the three characters reserved by both Telegram HTML and Slack mrkdwn. */
+export function escapeEntities(s) {
+  return String(s ?? '')
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+}
+
+/** Truncate raw text on a word boundary. Never call this after entity escaping. */
+export function truncateWords(text, max) {
+  const s = String(text ?? '').trim();
+  if (s.length <= max) return s;
+  const cut = s.slice(0, max - 1);
+  const lastSpace = cut.lastIndexOf(' ');
+  return `${(lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).trimEnd()}…`;
+}
+
+/**
+ * Escape raw text while guaranteeing that the escaped result fits `maxEscaped`.
+ *
+ * Escaping can expand one raw character to five. Truncating before escaping can therefore
+ * overshoot, while truncating after escaping can split an entity. Converge on a raw budget,
+ * re-measuring the escaped result each pass.
+ */
+export function escapeToFit(text, maxEscaped) {
+  let budget = maxEscaped;
+  for (let i = 0; i < 24; i++) {
+    const escaped = escapeEntities(truncateWords(text, budget));
+    if (escaped.length <= maxEscaped) return escaped;
+    const ratio = escaped.length / Math.max(1, maxEscaped);
+    budget = Math.max(1, Math.floor(budget / Math.max(1.25, ratio)));
+  }
+  return escapeEntities(truncateWords(text, 40));
+}

package/dist/lib/slack-text.mjs

@@ -0,0 +1,21 @@
+// Slack text helpers — platform naming + limit over the shared notification text rule.
+//
+// Slack's own mrkdwn escaping guide asks for the SAME three substitutions Telegram's HTML
+// parse_mode needs: `&` -> `&amp;`, `<` -> `&lt;`, `>` -> `&gt;`. (Slack isn't rendering HTML —
+// it just reserves those three characters for its own `<url|label>` link syntax and treats them
+// the same way HTML does.) Because the escaping rule is identical, the truncate-then-escape
+// convergence logic below is a straight port of escapeToFit's reasoning: escaping EXPANDS the
+// string, so truncating the raw text to a budget and escaping afterwards can overshoot, and
+// escaping first and truncating second can cut mid-entity. Same bug class, same fix.
+//
+// The shared implementation lives in notification-text.mjs because the substitution table is
+// byte-identical. Slack keeps its own conservative limit here; a shared limit would erase a real
+// platform/product decision.
+
+// Conservative on purpose: Slack's Incoming Webhook `text` field tolerates far more than this in
+// practice, but Block Kit's per-section text object caps at 3000, and keeping the ping terse and
+// pager-friendly is the actual goal here (matching Telegram's terse-by-design pings), not "as much
+// as Slack will technically accept."
+export const SLACK_LIMIT = 3000;
+
+export { escapeEntities as escapeMrkdwn, escapeToFit, truncateWords } from './notification-text.mjs';