ai-foreman 1.7.20 → 1.7.21

package/dist/workflowReader.d.ts

@@ -1,3 +1,4 @@
+import { type GraphRecordKind } from "./graph/storage.js";
 import { type ReadinessProcess } from "./readinessCleanup.js";
 import type { PendingHumanDecision, RecoveryAttemptReceipt, ResolvedAutonomyPolicy, SupervisorState, WorkflowIssue } from "rafi-spec";
 import { type OperationRecord, type RoleMutationLease, type ProjectLease, type WorkflowRunSnapshot } from "./workflowDb.js";
@@ -8,12 +9,118 @@ export declare class WorkflowReader {
     readonly path: string;
     private readonly db?;
     constructor(projectDir: string, path?: string);
+    qaPreparation(runId: string): {
+        mode: string;
+        works: Array<{
+            workId: string;
+            state: string;
+            digest?: string;
+            depth?: number;
+            detail?: string;
+        }>;
+    };
+    qaPreparationMetrics(runId: string): {
+        withheldEvents?: number | undefined;
+        limitation?: string | undefined;
+        approved: number;
+        started: number;
+        comparableStarted: number;
+        excludedNonEquivalentReviews: number;
+        approvedNeverStarted: number;
+        preparationFailures: number;
+        firstReviews: number;
+        firstReviewPasses: number;
+        firstReviewPassRate: number | null;
+        endToEndFirstAttempt: number;
+        endToEndRate: number | null;
+        omissionWorks: number;
+        omissionRounds: number;
+        omissionIncidence: number | null;
+        omissionRoundsPerStarted: number | null;
+        causes: Record<import("./qaPreparationMetrics.js").FindingCause, number>;
+        statuses: Record<string, number>;
+        quality: {
+            day7: {
+                observed: number;
+                reopened: number;
+                escaped: number;
+                unknown: number;
+            };
+            day30: {
+                observed: number;
+                reopened: number;
+                escaped: number;
+                unknown: number;
+            };
+        };
+        phaseCosts: Record<string, {
+            knownUsd: number;
+            knownSamples: number;
+            unknownSamples: number;
+            durationMs: number;
+        }>;
+        outsideContractEvents: number;
+        formatRepairs: number;
+        providerRetries: number;
+        byMode: {
+            [k: string]: {
+                works: number;
+                started: number;
+            };
+        };
+        samples: {
+            eventId: string;
+            workKey: string;
+            mode: "legacy" | "shadow" | "enforce";
+            level: number | undefined;
+            cohort: string;
+            at: string;
+            costUsd: number | null;
+            durationMs: number | null;
+            evidenceRefs: string[];
+            contractDigest?: string | undefined;
+            reviewDigest?: string | undefined;
+            assessor?: string | undefined;
+            checkUsefulness?: {
+                checkId: string;
+                result: string;
+                duplicate?: boolean;
+                irrelevant?: boolean;
+                observation: string;
+            }[] | undefined;
+            excessiveInvestigation?: string;
+            coverageAdequate?: boolean | undefined;
+        }[];
+        cohortSizes: {
+            [k: string]: number;
+        };
+    };
+    qaPreparationEvents(runId: string): Array<{
+        eventId: string;
+        workId: string;
+        kind: string;
+        value: unknown;
+    }>;
+    graphHostProjectIdentity(): string | undefined;
+    graphEvidenceAllowed(digest: string): boolean;
     close(): void;
+    graphRecord<T>(kind: GraphRecordKind, id: string): {
+        revision: number;
+        value: T;
+    } | undefined;
+    graphReceiptSummaries(runId: string): unknown[];
+    graphRecentJobs(): Array<{
+        id: string;
+        state: string;
+        reason?: string;
+        generationId?: string;
+    }>;
+    graphGenerationForCorpus<T>(corpusDigest: string, policyDigest: string): T | undefined;
     readinessProcesses(): ReadinessProcess[];
     available(): boolean;
     getRun(runId: string): WorkflowRunSnapshot | undefined;
     activeRuns(): WorkflowRunSnapshot[];
-    buildRuns(): WorkflowRunSnapshot[];
+    buildRuns(strict?: boolean): WorkflowRunSnapshot[];
     events(runId: string): Array<{
         sequence: number;
         type: string;
@@ -22,7 +129,7 @@ export declare class WorkflowReader {
         at: string;
     }>;
     issues(runId: string): WorkflowIssue[];
-    operations(runId: string): OperationRecord[];
+    operations(runId: string, strict?: boolean): OperationRecord[];
     currentLease(): ProjectLease | undefined;
     pendingQaTicketIds(runId: string): string[];
     preparationSuccessor(runId: string): string;

package/dist/workflowReader.js

@@ -1,3 +1,7 @@
+import { bytesDigest } from "./graph/util.js";
+import { measureQaPreparation } from "./qaPreparationMetrics.js";
+import { graphDerivedAllowed } from "./graph/derived.js";
+import { readGraphRecord } from "./graph/storage.js";
 import { checkBuildOwnershipSchema, canonicalProject } from "./buildAdmission.js";
 import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
@@ -23,7 +27,43 @@ export class WorkflowReader {
             throw error;
         }
     }
+    qaPreparation(runId) {
+        if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='qa_preparation_policy'").get())
+            return { mode: "legacy", works: [] };
+        const policy = this.db.prepare("SELECT record_json FROM qa_preparation_policy WHERE run_id=?").get(runId);
+        const rows = this.db.prepare("SELECT h.work_id,h.state,h.digest,h.detail,c.record_json FROM qa_contract_heads h LEFT JOIN qa_verification_contracts c ON c.digest=h.digest WHERE h.run_id=?").all(runId);
+        return { mode: policy ? JSON.parse(policy.record_json).mode : "legacy", works: rows.map(row => ({ workId: row.work_id, state: row.state, digest: row.digest, detail: row.detail && !this.graphEvidenceAllowed(bytesDigest(row.detail)) ? "Graph-derived preparation detail withheld; source-based recovery required" : row.detail, depth: row.record_json && this.graphEvidenceAllowed(bytesDigest(row.record_json)) ? JSON.parse(row.record_json).depthDecision?.level : undefined })) };
+    }
+    qaPreparationMetrics(runId) {
+        const events = this.qaPreparationEvents(runId), withheld = events.filter(event => event.kind === "evidence-unavailable").length;
+        return { ...measureQaPreparation(events.filter(event => event.kind === "metric").map(event => event.value)), ...(withheld ? { withheldEvents: withheld, limitation: "Metrics are incomplete: graph-derived events are unavailable" } : {}) };
+    }
+    qaPreparationEvents(runId) {
+        if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='qa_preparation_events'").get())
+            return [];
+        return this.db.prepare("SELECT event_id,work_id,kind,record_json FROM qa_preparation_events WHERE run_id=? ORDER BY rowid").all(runId).map(row => ({ eventId: row.event_id, workId: row.work_id, kind: this.graphEvidenceAllowed(bytesDigest(row.record_json)) ? row.kind : "evidence-unavailable", value: this.graphEvidenceAllowed(bytesDigest(row.record_json)) ? JSON.parse(row.record_json) : { unavailable: "graph-access-revoked", originalDigest: bytesDigest(row.record_json) } }));
+    }
+    graphHostProjectIdentity() { if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE name='build_project_identity'").get())
+        return; return this.db.prepare("SELECT project_id FROM build_project_identity WHERE singleton=1").get()?.project_id; }
+    graphEvidenceAllowed(digest) { return this.db ? graphDerivedAllowed(this.db, this.projectDir, digest) : true; }
     close() { this.db?.close(); }
+    graphRecord(kind, id) { return readGraphRecord(this.db, kind, id); }
+    graphReceiptSummaries(runId) {
+        if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE name='graph_records'").get())
+            return [];
+        return this.db.prepare("SELECT id AS receiptId,json_extract(value,'$.purpose') AS purpose,json_extract(value,'$.decision') AS decision,json_extract(value,'$.deliveredAt') AS deliveredAt,json_extract(value,'$.generationIds') AS generationIds FROM graph_records WHERE kind='receipt' AND json_extract(value,'$.runId')=? ORDER BY rowid DESC LIMIT 20").all(runId);
+    }
+    graphRecentJobs() {
+        if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE name='graph_records'").get())
+            return [];
+        return this.db.prepare("SELECT id,json_extract(value,'$.state') AS state,json_extract(value,'$.reason') AS reason,json_extract(value,'$.generationId') AS generationId FROM graph_records WHERE kind='job' ORDER BY rowid DESC LIMIT 15").all();
+    }
+    graphGenerationForCorpus(corpusDigest, policyDigest) {
+        if (!this.db || !this.db.prepare("SELECT 1 FROM sqlite_master WHERE name='graph_records'").get())
+            return;
+        const row = this.db.prepare("SELECT value FROM graph_records WHERE kind='generation' AND json_extract(value,'$.corpusDigest')=? AND json_extract(value,'$.policyDigest')=? ORDER BY json_extract(value,'$.createdAt') DESC LIMIT 1").get(corpusDigest, policyDigest);
+        return row ? JSON.parse(row.value) : undefined;
+    }
     readinessProcesses() {
         if (!this.db)
             return [];
@@ -53,13 +93,15 @@ export class WorkflowReader {
             return [];
         }
     }
-    buildRuns() {
+    buildRuns(strict = false) {
         if (!this.db)
             return [];
         try {
             return this.db.prepare("SELECT * FROM workflow_runs WHERE kind='build' ORDER BY updated_at DESC").all().map(toRun);
         }
-        catch {
+        catch (error) {
+            if (strict)
+                throw error;
             return [];
         }
     }
@@ -83,13 +125,15 @@ export class WorkflowReader {
             return [];
         }
     }
-    operations(runId) {
+    operations(runId, strict = false) {
         if (!this.db)
             return [];
         try {
             return this.db.prepare("SELECT * FROM operation_journal WHERE run_id=? ORDER BY created_at,idempotency_key").all(runId).map(row => ({ idempotencyKey: String(row.idempotency_key), runId: String(row.run_id), kind: String(row.kind), status: String(row.status), intent: JSON.parse(String(row.intent_json)), ...(row.result_json ? { result: JSON.parse(String(row.result_json)) } : {}), ...(row.external_id ? { externalId: String(row.external_id) } : {}), ...(row.error ? { error: String(row.error) } : {}), createdAt: String(row.created_at), updatedAt: String(row.updated_at) }));
         }
-        catch {
+        catch (error) {
+            if (strict)
+                throw error;
             return [];
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-foreman",
-  "version": "1.7.20",
+  "version": "1.7.21",
   "description": "An orchestrator that keeps Codex CLI / Claude Code builders moving through their step list unattended. Part of Rafi.",
   "license": "Apache-2.0",
   "type": "module",
@@ -17,10 +17,38 @@
     "ai-foreman": "dist/index.js"
   },
   "exports": {
+    "./build-assignment.js": {
+      "types": "./dist/buildAssignment.d.ts",
+      "default": "./dist/buildAssignment.js"
+    },
+    "./build-ownership-reconciliation.js": {
+      "types": "./dist/buildOwnershipReconciliation.d.ts",
+      "default": "./dist/buildOwnershipReconciliation.js"
+    },
+    "./manager-evidence.js": {
+      "types": "./dist/managerEvidence.d.ts",
+      "default": "./dist/managerEvidence.js"
+    },
+    "./qa-evidence-reader.js": {
+      "types": "./dist/qaEvidenceReader.d.ts",
+      "default": "./dist/qaEvidenceReader.js"
+    },
+    "./manager-evidence-artifacts.js": {
+      "types": "./dist/managerEvidenceArtifacts.d.ts",
+      "default": "./dist/managerEvidenceArtifacts.js"
+    },
+    "./qa-timeline.js": {
+      "types": "./dist/qaTimeline.d.ts",
+      "default": "./dist/qaTimeline.js"
+    },
     "./activity.js": {
       "types": "./dist/activity.d.ts",
       "default": "./dist/activity.js"
     },
+    "./terminal-style.js": {
+      "types": "./dist/terminalStyle.d.ts",
+      "default": "./dist/terminalStyle.js"
+    },
     "./agent-run.js": {
       "types": "./dist/agentRun.d.ts",
       "default": "./dist/agentRun.js"
@@ -188,21 +216,69 @@
     "./state-transfer.js": {
       "types": "./dist/stateTransfer.d.ts",
       "default": "./dist/stateTransfer.js"
-    }
+    },
+    "./human-decision.js": {
+      "types": "./dist/humanDecision.d.ts",
+      "default": "./dist/humanDecision.js"
+    },
+    "./build-approval.js": {
+      "types": "./dist/buildApproval.d.ts",
+      "default": "./dist/buildApproval.js"
+    },
+    "./build-work-admission.js": {
+      "types": "./dist/buildWorkAdmission.d.ts",
+      "default": "./dist/buildWorkAdmission.js"
+    },
+    "./build-work-migration.js": {
+      "types": "./dist/buildWorkMigration.d.ts",
+      "default": "./dist/buildWorkMigration.js"
+    },
+    "./build-interventions.js": {
+      "types": "./dist/buildInterventions.d.ts",
+      "default": "./dist/buildInterventions.js"
+    },
+    "./build-ownership-repair.js": {
+      "types": "./dist/buildOwnershipRepair.d.ts",
+      "default": "./dist/buildOwnershipRepair.js"
+    },
+    "./graph.js": {
+      "types": "./dist/graph/turn.d.ts",
+      "default": "./dist/graph/turn.js"
+    },
+    "./graph-maintenance.js": {
+      "types": "./dist/graph/maintenance.d.ts",
+      "default": "./dist/graph/maintenance.js"
+    },
+    "./cli/graph.js": {
+      "types": "./dist/cli/graph.d.ts",
+      "default": "./dist/cli/graph.js"
+    },
+    "./qa-preparation-policy.js": {
+      "types": "./dist/qaPreparationPolicy.d.ts",
+      "default": "./dist/qaPreparationPolicy.js"
+    },
+    "./qa-effective-config.js": {
+      "types": "./dist/qaEffectiveConfig.d.ts",
+      "default": "./dist/qaEffectiveConfig.js"
+    },
+    "./install-ownership.js": "./dist/installOwnership.js"
   },
   "files": [
     "dist",
     "README.md",
     "foreman.yaml",
-    "LICENSE"
+    "LICENSE",
+    "scripts/copy-graph-assets.mjs",
+    "scripts/graph-evaluation.mjs",
+    "scripts/graph-benchmark.mjs"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.build.json",
+    "build": "tsc -p tsconfig.build.json && node scripts/copy-graph-assets.mjs",
     "dev": "tsx src/index.ts",
     "test": "tsx --test --test-concurrency=1 test/*.test.ts test/*.test.mjs",
     "typecheck": "tsc --noEmit",
     "test:live-providers": "RAFI_LIVE_PROVIDER_SESSIONS=1 tsx --test test/providerSmoke.test.ts",
-    "prepublishOnly": "tsc -p ../spec/tsconfig.build.json && tsc -p tsconfig.build.json && pnpm --filter ai-foreman test && tsc --noEmit"
+    "prepublishOnly": "tsc -p ../spec/tsconfig.build.json && pnpm build && pnpm --filter ai-foreman test && tsc --noEmit"
   },
   "keywords": [
     "claude",
@@ -222,16 +298,17 @@
     "@anthropic-ai/claude-agent-sdk": "^0.3.148"
   },
   "dependencies": {
-    "special-agents": "0.8.12",
-    "rafi-spec": "0.8.13",
     "@clack/prompts": "^1.4.0",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",
     "better-sqlite3": "^12.10.0",
     "commander": "^12.1.0",
+    "ignore": "7.0.5",
     "luxon": "^3.7.2",
     "parse5": "7.2.1",
     "pdfjs-dist": "4.10.38",
+    "rafi-spec": "0.8.14",
+    "special-agents": "0.8.13",
     "yaml": "^2.5.1"
   },
   "devDependencies": {

package/scripts/copy-graph-assets.mjs

@@ -0,0 +1,3 @@
+import { copyFileSync, mkdirSync } from "node:fs";
+mkdirSync(new URL("../dist/graph/", import.meta.url), { recursive: true });
+copyFileSync(new URL("../src/graph/bridge.py", import.meta.url), new URL("../dist/graph/bridge.py", import.meta.url));

package/scripts/graph-benchmark.mjs

@@ -0,0 +1,20 @@
+/** Offline deterministic bridge measurement. No agents, network or corpus scan. */
+import { cpus, platform, release } from "node:os";
+import { performance } from "node:perf_hooks";
+import { resolveGraphPython, runBridge } from "../dist/graph/bridge.js";
+const python=await resolveGraphPython();
+const resourceControls=(await runBridge(python,{action:"probe"})).resourceControls;
+const samples=[];
+for(const count of [10000,100000]){
+  const graph={directed:true,multigraph:false,graph:{schema_version:1},nodes:Array.from({length:count},(_,i)=>({id:`n${i}`,label:`component${i}`,source_file:"fixture.py"})),links:Array.from({length:count-1},(_,i)=>({source:`n${i}`,target:`n${i+1}`,relation:"calls",confidence:"EXTRACTED",source_file:"fixture.py"}))};
+  const durations=[];let returned=0;
+  for(let sample=0;sample<5;sample++){
+    const start=performance.now();
+    const result=await runBridge(python,{action:"read",graph,operation:{operation:"neighbors",seeds:["n5000"],direction:"outgoing",depth:2}},{timeoutMs:15000});
+    durations.push(performance.now()-start);returned=result.nodes.length;
+    if(!result.nodes.some(n=>n.id==="n5002")||result.nodes.some(n=>n.id==="n4999"))throw new Error("Directed neighborhood correctness failed");
+  }
+  durations.sort((a,b)=>a-b);
+  samples.push({nodes:count,edges:count-1,samples:durations.length,medianMs:durations[2],largestObservedMs:durations[4],returnedNodes:returned,mode:"cold one-shot subprocess",warmLatency:"not measured",semanticCost:"not applicable"});
+}
+console.log(JSON.stringify({schema:1,measuredAt:new Date().toISOString(),resourceControls,environment:{platform:platform(),release:release(),cpu:cpus()[0]?.model,node:process.version},samples,limits:"Five samples establish a local fixture observation, not a production percentile or agent-quality evaluation."},null,2));

package/scripts/graph-evaluation.mjs

@@ -0,0 +1,85 @@
+/** Offline evaluation manifest and matched-result validator. Never invokes a provider. */
+import { createHash } from 'node:crypto';
+import { readFileSync, mkdirSync, writeFileSync } from 'node:fs';
+import { resolve, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');
+const categories = ['missed-caller', 'undeclared-dependency', 'cross-layer-regression', 'existing-invariant', 'captured-requirement'];
+export function evaluationManifest() {
+  const cases = categories.flatMap((category, index) => Array.from({ length: 4 }, (_, variant) => {
+    const name = `${category}-${variant + 1}`;
+    const files = {
+      'requirements.md': `# Approved requirement\nReject negative values; preserve zero; return the normalized integer to every caller.\n`,
+      'core.py': 'def normalize(value):\n    if value < 0:\n        raise ValueError("negative")\n    return int(value)\n',
+      'api.py': 'from core import normalize\ndef response(value):\n    return {"value": normalize(value)}\n',
+      'worker.py': 'from core import normalize\ndef process(value):\n    return normalize(value)\n',
+      'test_contract.py': 'from api import response\nfrom worker import process\ndef test_zero():\n    assert response(0) == {"value": 0}\n    assert process(0) == 0\n',
+      'plugin.py': 'def dynamic(module, value):\n    return getattr(module, "normalize")(value)\n',
+    };
+    const tasks = {
+      'missed-caller': 'Assess changing normalize to accept a required precision argument. Identify every affected caller, including uncertain dynamic dispatch.',
+      'undeclared-dependency': 'Plan a worker normalization change. Identify dependencies that must be reviewed before implementation.',
+      'cross-layer-regression': 'Change integer conversion behavior while keeping API and worker contracts consistent. Identify regression tests.',
+      'existing-invariant': 'Optimize normalization while preserving rejection of negative values and acceptance of zero.',
+      'captured-requirement': 'Trace the approved negative-value and zero requirements to implementation and tests; identify missing test coverage.',
+    };
+    // Variants alter a source fact, not only the case identifier.
+    files['variant.py'] = `from ${variant % 2 ? 'api import response' : 'worker import process'}\ndef scheduled_${variant}(value):\n    return ${variant % 2 ? 'response' : 'process'}(value)\n`;
+    return { id: name, category, files, baselineDigest: hash(files), task: tasks[category], relevantPaths: Object.keys(files),
+      sourceFacts: ['core.py rejects negative values', 'test_contract.py tests zero but omits negative-value coverage', 'plugin.py dispatch is dynamic and must be inspected in source'],
+      absentOrAmbiguousEdges: ['dynamic getattr dispatch is not assumed resolved', 'requirements-to-test semantic edges are navigation hints, not proof'],
+      mandatoryQa: ['inspect all affected callers', 'verify negative and zero behavior', 'run source tests', 'inspect dynamic dispatch'],
+      order: (index + variant) % 2 ? ['off', 'on'] : ['on', 'off'] };
+  }));
+  return { version: 1, kind: 'rafi_graph_evaluation', cases, negativeControl: { task: 'Correct a comment typo in core.py', expectedMaintenanceRuns: 0 },
+    controls: ['separate sessions and workspaces for each arm', 'identical approved scope, QA, depth, model, provider and baseline per pair', 'record cold and warm costs separately', 'retain blocked, cancelled, degraded and retried results', 'source inspection fallback is not graph retrieval'],
+    qualityTarget: { repeatedExplorationOrOmissionReduction: 0.2, coverageMustNotDecline: true },
+    limitation: 'Synthetic engineering fixtures validate machinery. Twenty matched fixture pairs do not establish representative production usefulness.' };
+}
+const metrics = ['usefulFindings', 'missedPaths', 'falseSuggestions', 'omissionRemediations', 'escapedDefects', 'repeatedExploration', 'preparationMs', 'buildMs', 'reviewMs', 'graphMs', 'extractionCostUsd', 'providerCostUsd', 'inputTokens', 'outputTokens'];
+export function reportResults(manifest, rows) {
+  if (!Array.isArray(rows)) throw new Error('Results must be an array');
+  const keys = new Set(), sessions = new Set();
+  for (const row of rows) {
+    const fixture = manifest.cases.find(c => c.id === row.caseId);
+    if (!fixture || !['on', 'off'].includes(row.arm) || !['cold', 'warm'].includes(row.cache)) throw new Error('Unknown case, arm or cache classification');
+    const key = `${row.caseId}:${row.arm}:${row.cache}`;
+    if (keys.has(key)) throw new Error(`Duplicate outcome ${key}`);
+    keys.add(key);
+    if (!row.sessionId || sessions.has(row.sessionId)) throw new Error('Each outcome needs an isolated session');
+    sessions.add(row.sessionId);
+    if (row.baselineDigest !== fixture.baselineDigest || row.qaDigest !== hash(fixture.mandatoryQa)) throw new Error('Baseline or mandatory QA differs from manifest');
+    if (!['completed', 'blocked', 'cancelled', 'degraded', 'failed'].includes(row.status)) throw new Error('Missing outcome classification');
+    for (const field of ['provider', 'model', 'depth', 'executionMode', 'semanticMode', 'difficulty']) if (typeof row[field] !== 'string' || !row[field]) throw new Error(`Missing ${field}`);
+    if (!Number.isInteger(row.retries) || row.retries < 0) throw new Error('Retain retry accounting');
+    if (typeof row.sourceVerified !== 'boolean' || typeof row.coverageVerified !== 'boolean') throw new Error('Explicit source and QA coverage review required');
+    for (const metric of metrics) if (row[metric] !== null && !(Number.isFinite(row[metric]) && row[metric] >= 0)) throw new Error(`${metric} must be a nonnegative number or null (unmeasured)`);
+  }
+  const pairs = [], unmatched = [];
+  for (const fixture of manifest.cases) for (const cache of ['cold', 'warm']) {
+    const arms = rows.filter(r => r.caseId === fixture.id && r.cache === cache);
+    if (arms.length !== 2) { unmatched.push({ caseId: fixture.id, cache, recordedArms: arms.map(r => r.arm) }); continue; }
+    const on = arms.find(r => r.arm === 'on'), off = arms.find(r => r.arm === 'off');
+    for (const field of ['provider', 'model', 'depth', 'executionMode', 'semanticMode', 'difficulty']) if (on[field] !== off[field]) throw new Error(`Unmatched ${field} in ${fixture.id}`);
+    pairs.push({ caseId: fixture.id, cache, statuses: { on: on.status, off: off.status }, coverageVerified: on.coverageVerified && off.coverageVerified,
+      differencesOnMinusOff: Object.fromEntries(metrics.map(m => [m, on[m] === null || off[m] === null ? null : on[m] - off[m]])) });
+  }
+  return { version: 1, recordedOutcomes: rows.length, matchedPairs: pairs.length, pairs, unmatched, outcomes: rows,
+    qualityConclusion: 'No automatic benefit claim. Review source-backed findings and equal QA coverage, include all statuses and unknown costs, and disclose sample uncertainty.',
+    liveEvaluation: rows.length ? 'User-supplied observations; harness does not execute or certify providers' : 'Deferred by user; no billed runs performed' };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
+  const [command, destination] = process.argv.slice(2);
+  const manifest = evaluationManifest();
+  if (command === 'prepare' && destination) {
+    mkdirSync(destination, { recursive: true });
+    for (const fixture of manifest.cases) {
+      const root = join(destination, fixture.id); mkdirSync(root, { recursive: true });
+      for (const [path, content] of Object.entries(fixture.files)) writeFileSync(join(root, path), content, { flag: 'wx' });
+    }
+    writeFileSync(join(destination, 'manifest.json'), JSON.stringify(manifest, null, 2), { flag: 'wx' });
+    console.log(`Prepared ${manifest.cases.length} offline cases in ${destination}. No providers invoked.`);
+  } else if (command === 'report') {
+    console.log(JSON.stringify(reportResults(manifest, destination ? JSON.parse(readFileSync(destination, 'utf8')) : []), null, 2));
+  } else throw new Error('Usage: node scripts/graph-evaluation.mjs prepare <new-directory> | report [results.json]');
+}