faberun 0.23.1 → 0.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "faberun",
-  "version": "0.23.1",
+  "version": "0.24.0",
   "description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
   "type": "module",
   "bin": {
@@ -24,7 +24,7 @@
     "node": ">=22"
   },
   "scripts": {
-    "check": "node -e \"const{readdirSync}=require('node:fs');const{spawnSync}=require('node:child_process');const roots=['bin','.claude/hooks','src','evals','test'];const files=roots.flatMap(r=>readdirSync(r,{recursive:true}).map(String).filter(p=>p.endsWith('.mjs')).map(p=>r+'/'+p));for(const f of files)if(spawnSync(process.execPath,['--check',f],{stdio:'inherit'}).status!==0)process.exit(1);console.log(files.length+' files checked')\"",
+    "check": "node -e \"const{readdirSync}=require('node:fs');const{spawnSync}=require('node:child_process');const roots=['bin','.claude/hooks','src','evals','test'];const files=roots.flatMap(r=>readdirSync(r,{recursive:true}).map(String).filter(p=>p.endsWith('.mjs')&&!p.includes('.runs')).map(p=>r+'/'+p));for(const f of files)if(spawnSync(process.execPath,['--check',f],{stdio:'inherit'}).status!==0)process.exit(1);console.log(files.length+' files checked')\"",
     "typecheck": "tsc",
     "check:campaign-brief-render": "node src/report/campaign-brief-html.mjs",
     "test": "node --test --import ./test/scoped-home.mjs --import ./test/git-env.mjs --import ./test/setup.mjs test/*.test.mjs test/*/*.test.mjs",

package/src/campaign/metrics.mjs

@@ -51,6 +51,7 @@ const UNKNOWN_COST_PROVENANCE = "unknown";
 /** @typedef {"down"|"up"|"informative"} Direction */
 /** @typedef {{value: number|null, direction: Direction, count: number, numerator?: number, denominator?: number, excludedRunIds?: string[], missingSources?: string[]}} Indicator */
 /** @typedef {Indicator & {unknownCount: number, unknownCountByReason: Record<string, number>, unknownFractionByReason: Record<string, number>}} CostIndicator */
+/** @typedef {Indicator & {unknownCount: number}} JudgeCostShareIndicator */
 /** @typedef {{value: Record<string, number>|null, direction: Direction, count: number, numerator?: Record<string, number>, denominator?: Record<string, number>, excludedRunIds?: string[], missingSources?: string[]}} GroupedIndicator */
 /** @typedef {{atMs: number, index: number, event: JsonObject}} RunEvent */
 /** @typedef {{runId: string, id: string, status: string, attempt?: number|null, revisions?: number|null, review?: string|null}} RunNode */
@@ -79,6 +80,8 @@ const UNKNOWN_COST_PROVENANCE = "unknown";
  *   usageTokensByKind: GroupedIndicator,
  *   usageTokensByKindByRuntime: GroupedIndicator,
  *   usageCostUsd: CostIndicator,
+ *   judgeFindingRate: Indicator,
+ *   judgeCostShare: JudgeCostShareIndicator,
  *   blockingJudgeFirstPassRate: GroupedIndicator,
  *   notifyReceiptRate: Indicator,
  *   silentStallRate: Indicator,
@@ -101,6 +104,10 @@ export function projectMetrics({ events = [], usageRecords = [], notifications =
   const excluded = excludedRunIdsFor(campaign, excludedRunIds);
   const eligibleNodes = nodes.filter((node) => !excluded.includes(node.runId));
   const eligibleEvents = eventList.filter((event) => typeof event.runId !== "string" || !excluded.includes(event.runId));
+  const eligibleUsageRecords = usageRecords.filter((record) => {
+    const object = jsonObjectOf(record);
+    return object === null || typeof object.runId !== "string" || !excluded.includes(object.runId);
+  });
   const eligibleNotifications = notifications.filter((record) => {
     const object = jsonObjectOf(record);
     return object === null || typeof object.runId !== "string" || !excluded.includes(object.runId);
@@ -110,6 +117,7 @@ export function projectMetrics({ events = [], usageRecords = [], notifications =
   const closedCheckpoints = closedCheckpointsOf(nodes);
   const span = eventSpanOf(eventList);
   const usage = usageTotalsOf(usageRecords);
+  const judgeYield = judgeYieldOf(eligibleEvents, eligibleUsageRecords);
   const gates = blockingJudgeFirstPassRateOf(eligibleNodes, eligibleEvents);
   const notify = notifyReceiptRateOf(eligibleNotifications);
   const stalls = silentStallRateOf(eligibleNodes, eligibleEvents);
@@ -129,6 +137,11 @@ export function projectMetrics({ events = [], usageRecords = [], notifications =
       unknownCountByReason: usage.unknownCountByReason,
       unknownFractionByReason: usage.unknownFractionByReason,
     },
+    judgeFindingRate: rate("up", judgeYield.findings, judgeYield.judged, excluded),
+    judgeCostShare: {
+      ...measured("down", usage.costCount, usage.costUsd === null ? null : usage.judgeCostUsd / usage.costUsd),
+      unknownCount: usage.unknownJudgeCount,
+    },
     blockingJudgeFirstPassRate: groupedRate("up", gates, excluded),
     notifyReceiptRate: rate("up", notify.satisfied, notify.count, excluded),
     silentStallRate: rate("down", stalls.stalled, stalls.activeHours, excluded, stalls.activeIntervals),
@@ -162,6 +175,8 @@ function applyMissingSources(metrics, missingSources) {
     usageTokensByKind: withMissing(metrics.usageTokensByKind, ["usage.jsonl"]),
     usageTokensByKindByRuntime: withMissing(metrics.usageTokensByKindByRuntime, ["usage.jsonl"]),
     usageCostUsd: withMissing(metrics.usageCostUsd, ["usage.jsonl"]),
+    judgeFindingRate: withMissing(metrics.judgeFindingRate, ["events.jsonl", "usage.jsonl"]),
+    judgeCostShare: withMissing(metrics.judgeCostShare, ["usage.jsonl"]),
     blockingJudgeFirstPassRate: withMissing(metrics.blockingJudgeFirstPassRate, ["nodes.json", "events.jsonl"]),
     notifyReceiptRate: withMissing(metrics.notifyReceiptRate, ["notify.jsonl"]),
     silentStallRate: withMissing(metrics.silentStallRate, ["nodes.json", "events.jsonl"]),
@@ -397,7 +412,7 @@ function eventSpanOf(events) {
  * invocation is counted separately rather than folded into a measured zero.
  *
  * @param {unknown[]} usageRecords
- * @returns {{tokensByKind: Record<string, number>, tokensByKindByRuntime: Record<string, number>, tokenCount: number, costUsd: number|null, costCount: number, unknownCount: number, unknownCountByReason: Record<string, number>, unknownFractionByReason: Record<string, number>}}
+ * @returns {{tokensByKind: Record<string, number>, tokensByKindByRuntime: Record<string, number>, tokenCount: number, costUsd: number|null, judgeCostUsd: number, costCount: number, unknownCount: number, unknownJudgeCount: number, unknownCountByReason: Record<string, number>, unknownFractionByReason: Record<string, number>}}
  */
 function usageTotalsOf(usageRecords) {
   const tokensByKind = { inputTokens: 0, cacheReadInputTokens: 0, outputTokens: 0 };
@@ -405,8 +420,10 @@ function usageTotalsOf(usageRecords) {
   const tokensByKindByRuntime = {};
   let tokenCount = 0;
   let costUsd = 0;
+  let judgeCostUsd = 0;
   let costCount = 0;
   let unknownCount = 0;
+  let unknownJudgeCount = 0;
   /** @type {Record<string, number>} */
   const unknownCountByReason = {};
   const kinds = /** @type {("inputTokens"|"cacheReadInputTokens"|"outputTokens")[]} */ (["inputTokens", "cacheReadInputTokens", "outputTokens"]);
@@ -425,9 +442,11 @@ function usageTotalsOf(usageRecords) {
     if (measuredAny) tokenCount += 1;
     if (typeof record.costUsd === "number" && Number.isFinite(record.costUsd) && record.costProvenance !== UNKNOWN_COST_PROVENANCE) {
       costUsd += record.costUsd;
+      if (record.role === "judge") judgeCostUsd += record.costUsd;
       costCount += 1;
     } else {
       unknownCount += 1;
+      if (record.role === "judge") unknownJudgeCount += 1;
       const reason = typeof record.unknownReason === "string" && UNKNOWN_COST_REASONS.includes(record.unknownReason)
         ? record.unknownReason
         : "legacy";
@@ -443,13 +462,51 @@ function usageTotalsOf(usageRecords) {
     tokensByKindByRuntime,
     tokenCount,
     costUsd: costCount === 0 ? null : costUsd,
+    judgeCostUsd,
     costCount,
     unknownCount,
+    unknownJudgeCount,
     unknownCountByReason,
     unknownFractionByReason,
   };
 }
 
+/**
+ * Count logical nodes that reached a judge and those whose judge recorded at
+ * least one finding. A failed judge verdict is the durable ledger projection
+ * of a non-empty findings array; a complete-phase fail is included for
+ * advisory reviews that settle the node after reporting their finding.
+ *
+ * @param {JsonObject[]} events
+ * @param {unknown[]} usageRecords
+ * @returns {{judged: number, findings: number}}
+ */
+function judgeYieldOf(events, usageRecords) {
+  const judged = new Set();
+  const findings = new Set();
+  for (const raw of usageRecords) {
+    const record = jsonObjectOf(raw);
+    if (record === null || record.role !== "judge" || typeof record.nodeId !== "string") continue;
+    judged.add(logicalNodeKey(record.runId, record.nodeId));
+  }
+  for (const event of events) {
+    if (typeof event.node !== "string") continue;
+    const key = logicalNodeKey(event.runId, event.node);
+    if (event.phase === "judge") judged.add(key);
+    const hasFindingArray = Array.isArray(event.findings) && event.findings.length > 0;
+    if ((event.phase === "judge" || event.phase === "complete") && (event.verdict === "fail" || hasFindingArray)) {
+      judged.add(key);
+      findings.add(key);
+    }
+  }
+  return { judged: judged.size, findings: findings.size };
+}
+
+/** @param {unknown} runId @param {string} nodeId @returns {string} */
+function logicalNodeKey(runId, nodeId) {
+  return `${typeof runId === "string" ? runId : ""}:${nodeId}`;
+}
+
 /**
  * Fraction of blocking-gated checkpoints whose first recorded verdict passed,
  * per lane (the judge runtime that produced it). A node under `advisory` or

package/src/cli/contract.mjs

@@ -6,6 +6,7 @@
  */
 import { existsSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
+import { parseArgs as parseFlags } from "node:util";
 import { loadPersistedContract, validateContract } from "../contract/index.mjs";
 
 /** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
@@ -13,7 +14,7 @@ import { loadPersistedContract, validateContract } from "../contract/index.mjs";
 /** Flags are scoped to the operation that declares them; all others are rejected. */
 /** @type {Record<string, import("node:util").ParseArgsOptionsConfig>} */
 const OPERATION_OPTIONS = {
-  validate: {},
+  validate: { "strict-traceability": { type: "boolean" } },
 };
 
 /**
@@ -23,9 +24,17 @@ const OPERATION_OPTIONS = {
 export function contractCli(args) {
   const operation = args[0];
   if (!operation || !Object.hasOwn(OPERATION_OPTIONS, operation)) return usage();
-  const target = args[1];
-  if (!target || args.length > 2) return usage();
-  if (operation === "validate") validateContractFile(resolve(target));
+  let parsed;
+  try {
+    parsed = parseFlags({ args: args.slice(1), options: OPERATION_OPTIONS[operation], allowPositionals: true, strict: true });
+  } catch (error) {
+    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+    return usage();
+  }
+  const target = parsed.positionals[0];
+  if (!target || parsed.positionals.length > 1) return usage();
+  const values = /** @type {{"strict-traceability"?: boolean}} */ (parsed.values);
+  if (operation === "validate") validateContractFile(resolve(target), { strictTraceability: values["strict-traceability"] === true });
 }
 
 /**
@@ -38,19 +47,29 @@ export function contractCli(args) {
  * authoring validation, tree reads included.
  *
  * @param {string} path
+ * @param {{strictTraceability?: boolean}} [options]
  * @returns {ValidatedContract}
  */
-export function validateContractFile(path) {
+export function validateContractFile(path, { strictTraceability = false } = {}) {
   const runJsonPath = join(dirname(path), "run.json");
   const contract = existsSync(runJsonPath)
     ? loadPersistedContract(path, readRunDigest(runJsonPath))
     : validateContract(JSON.parse(readFileSync(path, "utf8")), path);
   const count = contract.warnings.length;
-  process.stdout.write(`valid${count ? ` (${count} warning${count === 1 ? "" : "s"})` : ""}\n`);
+  const blockingTraceability = strictTraceability
+    ? contract.warnings.filter((warning) => isTraceabilityFinding(warning))
+    : [];
+  process.stdout.write(`${blockingTraceability.length > 0 ? "invalid" : "valid"}${count ? ` (${count} warning${count === 1 ? "" : "s"})` : ""}\n`);
   for (const warning of contract.warnings) process.stdout.write(`[warn] ${warning}\n`);
+  if (blockingTraceability.length > 0) process.exitCode = 1;
   return contract;
 }
 
+/** @param {string} warning @returns {boolean} */
+function isTraceabilityFinding(warning) {
+  return warning.includes("judgment_without_reason") || warning.includes("judgment_beside_mechanical_proof");
+}
+
 /**
  * @param {string} runJsonPath
  * @returns {string|undefined}
@@ -62,7 +81,7 @@ function readRunDigest(runJsonPath) {
 
 /** @returns {void} */
 function usage() {
-  process.stderr.write("usage: faberun contract validate <contract.json>\n");
+  process.stderr.write("usage: faberun contract validate <contract.json> [--strict-traceability]\n");
   process.exitCode = 2;
 }
 

package/src/contract/definition-of-done.mjs

@@ -9,7 +9,7 @@ import { rejectUnknown, requireId, requireString } from "./assert.mjs";
 
 /** @typedef {{kind: "command"|"path"|"verification", ref: string}} DefinitionOfDoneProof */
 
-/** @typedef {{id: string, text: string, proof?: DefinitionOfDoneProof, judgment?: true}} DefinitionOfDoneItem */
+/** @typedef {{id: string, text: string, proof?: DefinitionOfDoneProof, judgment?: true, reason?: string}} DefinitionOfDoneItem */
 
 /**
  * @param {unknown} value
@@ -28,12 +28,16 @@ export function validateDefinitionOfDone(value, label, options = {}) {
       throw new TypeError(`${itemLabel} must be an object with id, text, and proof or judgment: true`);
     }
     const record = /** @type {Record<string, unknown>} */ (item);
-    rejectUnknown(record, new Set(["id", "text", "proof", "judgment"]), itemLabel);
+    rejectUnknown(record, new Set(["id", "text", "proof", "judgment", "reason"]), itemLabel);
     requireId(record.id, `${itemLabel}.id`);
     requireString(record.text, `${itemLabel}.text`);
     if (record.judgment !== undefined && record.judgment !== true) {
       throw new TypeError(`${itemLabel}.judgment must be true when present`);
     }
+    if (record.reason !== undefined) {
+      if (record.judgment !== true) throw new TypeError(`${itemLabel}.reason requires judgment: true`);
+      requireString(record.reason, `${itemLabel}.reason`);
+    }
     const proof = record.proof === undefined
       ? undefined
       : validateProof(record.proof, `${itemLabel}.proof`, options);
@@ -45,10 +49,34 @@ export function validateDefinitionOfDone(value, label, options = {}) {
       text: /** @type {string} */ (record.text),
       ...(proof === undefined ? {} : { proof }),
       ...(record.judgment === true ? { judgment: true } : {}),
+      ...(record.reason === undefined ? {} : { reason: /** @type {string} */ (record.reason) }),
     };
   });
 }
 
+/**
+ * Report judgment items that do not explain what mechanical verification cannot
+ * observe. These remain advisory because `contract validate` has no strict
+ * traceability flag; a later strict contract command can promote these codes.
+ *
+ * @param {DefinitionOfDoneItem[]} items
+ * @param {number} index
+ * @returns {string[]}
+ */
+export function judgmentReasonWarnings(items, index) {
+  const judgmentItems = items.filter((item) => item.judgment === true);
+  const warnings = items.flatMap((item, itemIndex) => item.judgment === true && item.reason === undefined
+    ? [`nodes[${index}] (definitionOfDone[${itemIndex}]): judgment_without_reason: judgment item must say what no command verifies`]
+    : []);
+  const hasMechanicalProof = items.some((item) => item.proof !== undefined);
+  if (judgmentItems.length > 0 && hasMechanicalProof && judgmentItems.every((item) => item.reason === undefined)) {
+    warnings.push(
+      `nodes[${index}]: judgment_beside_mechanical_proof: every judgment item shares this node with mechanical-proof items but declares no reason`,
+    );
+  }
+  return warnings;
+}
+
 /**
  * @param {unknown} value
  * @param {string} label
@@ -144,4 +172,3 @@ export function unquotedFilterValueWarnings(items, index) {
 
 /** The node:test filter flags a `kind: "command"` proof's shell can split on an unquoted value. */
 const TEST_FILTER_FLAGS = ["--test-name-pattern", "--test-skip-pattern"];
-

package/src/contract/index.mjs

@@ -3,7 +3,7 @@ import { readFileSync, statSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { loadTaskPacket, renderWorkerPrompt } from "./task-packet.mjs";
 import { RESERVED_ARTICLES } from "./articles.mjs";
-import { unquotedFilterValueWarnings, validateDefinitionOfDone } from "./definition-of-done.mjs";
+import { judgmentReasonWarnings, unquotedFilterValueWarnings, validateDefinitionOfDone } from "./definition-of-done.mjs";
 import { validateFinalVerification, validateSharedVerification } from "./final-verification.mjs";
 import { VERIFICATION_LIMITS, requirementProofWarnings } from "./verification.mjs";
 import {
@@ -378,6 +378,7 @@ export function validateContract(raw, contractPath, options = {}) {
     ...nodes.flatMap((node, index) => [
       ...commandCoverageWarnings(node, index),
       ...unquotedFilterValueWarnings(node.definitionOfDone ?? [], index),
+      ...judgmentReasonWarnings(node.definitionOfDone ?? [], index),
       ...(persisted ? [] : mirrorCoverageWarnings(node, index, cwd, contractCommands, contractWrites)),
       ...(persisted ? [] : unsnapshottedWriteWarnings(node, index, cwd)),
       ...(persisted ? [] : ignoreSourceWriteWarnings(node, index)),

package/src/plan/proof-run.mjs

@@ -25,6 +25,13 @@ import { NOTIFY_SESSION_ENV } from "../notify/session.mjs";
 /** @typedef {{requirementId: string|null, kind: "command"|"path", ref: string, pass: boolean, detail: string}} RequirementProofResult */
 
 export const MEASURE_TIMEOUT_MS = 30_000;
+/**
+ * A proof is a test run, not a fact probe. Measured 2026-09-24: R6 of
+ * `evals-with-a-budget` takes 98 s pinned to its file, and under the 30 s fact
+ * bound the runner died on SIGTERM with exit 1, read as a failing proof. Ten
+ * minutes has no measurement behind it beyond covering the slowest proof seen.
+ */
+const PROOF_TIMEOUT_MS = 600_000;
 export const MEASURE_OUTPUT_CAP_BYTES = 4096;
 
 /** How much of a failing proof's own output a finding carries: enough to name the failure, never the whole log. */
@@ -82,9 +89,10 @@ export function runShellCapture(cwd, command, probes = {}) {
  * @param {string} cwd
  * @param {string} command
  * @param {MeasureProbes} [probes]
- * @returns {{output: string, exitCode: number|null, truncated: boolean}}
+ * @param {number} [timeoutMs]
+ * @returns {{output: string, exitCode: number|null, truncated: boolean, timedOut: boolean}}
  */
-function runShellFull(cwd, command, probes = {}) {
+function runShellFull(cwd, command, probes = {}, timeoutMs = MEASURE_TIMEOUT_MS) {
   const run = probes.run ?? spawnSync;
   const env = { ...process.env };
   for (const key of MEASURE_SIDE_EFFECT_ENV_KEYS) delete env[key];
@@ -92,9 +100,10 @@ function runShellFull(cwd, command, probes = {}) {
   // child and prints no result at all (measured 2026-09-21, AGENTS.md), so a
   // proof run from inside a test runner would have nothing to show.
   delete env.NODE_TEST_CONTEXT;
-  const result = run(command, { shell: true, cwd, timeout: MEASURE_TIMEOUT_MS, encoding: "utf8", env });
+  const result = run(command, { shell: true, cwd, timeout: timeoutMs, encoding: "utf8", env });
   const combined = `${result.stdout ?? ""}${result.stderr ?? ""}`;
-  return { output: combined, exitCode: result.status, truncated: Buffer.byteLength(combined, "utf8") > MEASURE_OUTPUT_CAP_BYTES };
+  const timedOut = /** @type {NodeJS.ErrnoException|undefined} */ (result.error)?.code === "ETIMEDOUT";
+  return { output: combined, exitCode: result.status, truncated: Buffer.byteLength(combined, "utf8") > MEASURE_OUTPUT_CAP_BYTES, timedOut };
 }
 
 /** `--test-name-pattern` in a shell command, quoted or bare, `=` or space. */
@@ -146,9 +155,11 @@ export function proveRequirements(cwd, requirements, probes = {}) {
     const proof = requirement.proof;
     if (!proof?.ref) continue;
     if (proof.kind === "command") {
-      const { output, exitCode } = runShellFull(cwd, proof.ref, probes);
+      const { output, exitCode, timedOut } = runShellFull(cwd, proof.ref, probes, PROOF_TIMEOUT_MS);
       const matched = exitCode === 0 ? patternMatchedATest(proof.ref, output) : null;
-      const detail = exitCode !== 0
+      const detail = timedOut
+        ? `timed out after ${PROOF_TIMEOUT_MS / 1000} s`
+        : exitCode !== 0
         ? `exit ${exitCode ?? "no exit code (killed or never started)"}${output.trim() ? `: ${output.trim().slice(0, PROOF_DETAIL_CAP)}` : ""}`
         : matched === false ? "exit 0, but its --test-name-pattern matched no test: the proof ran nothing" : "exit 0";
       results.push({ requirementId: requirement.id, kind: "command", ref: proof.ref, pass: exitCode === 0 && matched !== false, detail });

package/src/plan/template.mjs

@@ -111,7 +111,7 @@ const REQUIRED_INPUTS = Object.freeze({
 // run had already succeeded. The id charset is requireId's
 // (contract/assert.mjs) verbatim, because an id that is present but invalid
 // fails that same validator just as late.
-const PLAN_OUTPUT_SHAPE = '{nodes: [{id, objective, taskKind, riskTier, dependsOn, readFiles, writeFiles, scopeAcknowledged, definitionOfDone: [{id, text, proof?: {kind: "command"|"path"|"verification", ref}, judgment?: true}], verification: [{argv: [string], cwd?, timeoutSec?, repeat?, env?, mutation?: {threshold}}], expectedTurns?}], phases?: [{id, requirementIds: [string], nodeIds: [string], deliverable}], justification?}; every id in it (node, phase, node assignment, and definitionOfDone item) must match [A-Za-z0-9._-]+ and never be exactly "." or ".."';
+const PLAN_OUTPUT_SHAPE = '{nodes: [{id, objective, taskKind, riskTier, dependsOn, readFiles, writeFiles, scopeAcknowledged, definitionOfDone: [{id, text, proof?: {kind: "command"|"path"|"verification", ref}, judgment?: true, reason?: string}], verification: [{argv: [string], cwd?, timeoutSec?, repeat?, env?, mutation?: {threshold}}], expectedTurns?}], phases?: [{id, requirementIds: [string], nodeIds: [string], deliverable}], justification?}; every id in it (node, phase, node assignment, and definitionOfDone item) must match [A-Za-z0-9._-]+ and never be exactly "." or ".."';
 /**
  * The size guidance every draft and revise carries. measured 2026-09-20 over
  * stored runs: median 49 provider requests per worker turn; cost per turn