@tangle-network/agent-eval 0.144.5 → 0.144.7

package/dist/verdict-DExhxfgR.d.ts

@@ -0,0 +1,201 @@
+//#region src/verification-strategy.d.ts
+/**
+ * Verification-strategy family — the shared vocabulary for "what certified
+ * this result", covering tasks with an answer key and tasks without one.
+ *
+ * Charter Wave 5 (docs/charter.md): an unsolved problem has no held-out
+ * suite by definition, so the held-out suite must be ONE member of a
+ * strategy family, not the family itself. This module carries the taxonomy
+ * and the port shape only. A checker is an injected executable boundary
+ * that returns a typed outcome, so a consumer binds its own kernel (a Lean
+ * toolchain, a metamorphic harness, a replication runner) and this package
+ * never chooses one. Instruments, never methods.
+ *
+ * Full family doc, per-member failure modes in prose, and the BCWW pilot
+ * worked example: docs/verification-strategies.md.
+ *
+ * Leaf module: no imports from this package. `src/verdict.ts` and
+ * `src/rl/verifiable-reward.ts` both build on these types.
+ */
+/**
+ * One member of the verification-strategy family.
+ *
+ * Every member has a documented failure mode — the specific way it can
+ * certify a wrong result. A consumer that reads a certification must weigh
+ * the member's failure mode, not treat "certified" as one bit.
+ *
+ * - `'compile'` — typecheck / build / lint passed. Deterministic.
+ *   Failure mode: code that compiles is not code that is correct; the
+ *   weakest deterministic member.
+ * - `'test'` — unit / integration / held-out suite pass-rate. Deterministic.
+ *   Failure mode: assumes an answer key exists; certifies nothing outside
+ *   the suite's coverage, and a stubbed integration reports green.
+ * - `'schema'` — structured output validates. Deterministic.
+ *   Failure mode: shape is not meaning; a well-formed wrong answer passes.
+ * - `'sandbox'` — sandbox execution exit code. Deterministic.
+ *   Failure mode: an exit code compresses the whole run to one bit; a
+ *   faked success exits 0.
+ * - `'judge'` — LLM judge score. Probabilistic.
+ *   Failure mode: drifts across model versions and is Goodhart-gameable by
+ *   the graded policy.
+ * - `'composite'` — weighted blend across members. Determinism inherited
+ *   from its members.
+ *   Failure mode: scalar collapse — the blend hides which member carried
+ *   the score and which failed.
+ * - `'proof-kernel'` — a proof assistant's kernel accepted a formal proof.
+ *   Deterministic.
+ *   Failure mode: the formalization gap. The kernel certifies the formal
+ *   statement, never that the formal statement matches the informal claim;
+ *   a proved theorem about the wrong statement is a wrong result with a
+ *   valid certificate. Discharge it with the blind statement-equivalence
+ *   protocol (`src/equivalence-check.ts`).
+ * - `'invariant'` — invariant / metamorphic properties held on the result.
+ *   Deterministic.
+ *   Failure mode: weak invariants pass everything. An invariant set earns
+ *   weight only through seeded-bug calibration — a mutation the set
+ *   provably rejects; an uncalibrated set is a rubber stamp.
+ * - `'replication'` — an independent re-execution reproduced the result
+ *   from pinned inputs. Deterministic given the pins.
+ *   Failure mode: replication re-runs the method, so it catches
+ *   nondeterminism and environment drift, never a methodological error the
+ *   method itself carries.
+ * - `'agreement'` — independently-derived results agree. Probabilistic.
+ *   Failure mode: the shared blind spot. Derivers that share training
+ *   corpora, priors, or the same misread of the source agree for the same
+ *   wrong reason; the blindness provenance of each arm is what the
+ *   certificate rests on.
+ */
+type VerificationStrategySource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite' | 'proof-kernel' | 'invariant' | 'replication' | 'agreement';
+/**
+ * What a strategy member can honestly claim about its outcomes.
+ *
+ * `determinism: 'inherited'` exists only for `'composite'`: a blend takes
+ * the class of its members. The reward axis stays binary
+ * (`VerifiableReward.determinism`); this registry describes the family,
+ * it never scores a run.
+ */
+interface VerificationStrategyProfile {
+  /** Determinism class a well-formed checker of this member can claim. */
+  determinism: 'deterministic' | 'probabilistic' | 'inherited';
+  /** The documented way this member certifies a wrong result. */
+  failureMode: string;
+}
+/**
+ * The family registry. `Record` over the union keeps it exhaustive: adding
+ * a member to `VerificationStrategySource` without a profile here fails to
+ * compile. The failure mode travels with the taxonomy so a reader of a
+ * certification can surface it without this package's docs at hand.
+ */
+declare const VERIFICATION_STRATEGIES: Record<VerificationStrategySource, VerificationStrategyProfile>;
+/** Every family member, derived from the registry so it cannot drift. */
+declare const VERIFICATION_STRATEGY_SOURCES: readonly VerificationStrategySource[];
+/**
+ * Exact identity of the thing that checked. "lean4 4.33.0 with Mathlib
+ * db584cd6d46c" and "lean4, some version" are different certificates: the
+ * first is re-runnable, the second is a story.
+ */
+interface CheckerIdentity {
+  /** Toolchain or model name, e.g. `'lean4'` or a judge model id. */
+  name: string;
+  /** Exact version or snapshot, e.g. `'4.33.0'` or a model snapshot id. */
+  version: string;
+  /**
+   * Content pins the version alone does not capture, e.g.
+   * `{ mathlib: 'db584cd6d46c' }`. Keyed by dependency name.
+   */
+  pins?: Record<string, string>;
+}
+/**
+ * Typed outcome of an external checker call. Callers MUST inspect
+ * `succeeded` before touching `value`; a failed check keeps its full error
+ * text instead of collapsing to null.
+ */
+type CheckerOutcome<Result> = {
+  succeeded: true;
+  value: Result;
+} | {
+  succeeded: false;
+  error: string;
+};
+/**
+ * The port shape every strategy binds through: an injected executable
+ * boundary. This package defines the port and refuses to ship any
+ * implementation behind it — the consumer binds its own kernel, invariant
+ * harness, replication runner, or judge, and the binding carries its own
+ * identity and determinism claim.
+ *
+ * A checker that ran but could not decide returns
+ * `{ succeeded: false, error }` with the reason; `succeeded: true` is
+ * reserved for a discharged check.
+ */
+interface StrategyChecker<Input, Result> {
+  /** Which family member this checker discharges obligations for. */
+  strategy: VerificationStrategySource;
+  identity: CheckerIdentity;
+  /** Determinism class this checker claims for its outcomes. */
+  determinism: 'deterministic' | 'probabilistic';
+  check(input: Input): Promise<CheckerOutcome<Result>>;
+}
+//#endregion
+//#region src/verdict.d.ts
+/**
+ * What certified a verdict — the epistemics a bare `valid` + `score` pair
+ * cannot carry. A kernel-checked proof and an LLM judge can produce the
+ * same `{ valid: true, score: 1 }`; this record is what tells them apart.
+ *
+ * Every field is required on purpose. A certification without a checker
+ * identity or an evidence digest is exactly the unverifiable claim this
+ * record exists to kill.
+ */
+interface VerdictCertification {
+  /**
+   * Which verification-strategy member produced the certificate. Each
+   * member's failure mode is documented on the family type and in
+   * `VERIFICATION_STRATEGIES` (src/verification-strategy.ts).
+   */
+  strategy: VerificationStrategySource;
+  /**
+   * Exact checker identity, e.g. `{ name: 'lean4', version: '4.33.0',
+   * pins: { mathlib: 'db584cd6d46c' } }` for a kernel, or a judge model
+   * id + snapshot for a judge.
+   */
+  checker: CheckerIdentity;
+  /**
+   * Every step the certificate rests on that the checker did NOT verify,
+   * e.g. 'diagonal embedding argued in prose' or 'agreement arms share
+   * training-corpus blind spots'. An empty array is the producer's
+   * explicit, auditable claim that there are none — it is not a default.
+   */
+  assumptions: string[];
+  /** Digest (e.g. sha256) of the evidence artifact the checker consumed or produced. */
+  evidenceDigest: string;
+}
+/**
+ * Minimal verdict shape — `valid` + `score` are required; `scores` +
+ * `notes` are optional surface. Validators that need richer shapes
+ * parameterise `Validator<Output, MyVerdict>` with their own type.
+ *
+ * Need structured extras? Extend DefaultVerdict with typed fields — never
+ * serialize extras into `notes`.
+ */
+interface DefaultVerdict {
+  /** Whether the output meets the validator's pass criteria. */
+  valid: boolean;
+  /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
+  score: number;
+  /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
+  scores?: Record<string, number>;
+  /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
+  notes?: string;
+  /**
+   * What certified this verdict, when anything did. Absent = uncertified:
+   * scored, but no strategy vouches for it. A consumer that cares reads
+   * `certification.strategy` to tell a kernel-checked verdict from a
+   * judge-scored one; a consumer that does not read it sees the exact
+   * verdict it always saw.
+   */
+  certification?: VerdictCertification;
+}
+//#endregion
+export { StrategyChecker as a, VerificationStrategyProfile as c, CheckerOutcome as i, VerificationStrategySource as l, VerdictCertification as n, VERIFICATION_STRATEGIES as o, CheckerIdentity as r, VERIFICATION_STRATEGY_SOURCES as s, DefaultVerdict as t };
+//# sourceMappingURL=verdict-DExhxfgR.d.ts.map

package/dist/verdict-DExhxfgR.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict-DExhxfgR.d.ts","names":[],"sources":["../src/verification-strategy.ts","../src/verdict.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAmEY;;;;;;;;;UAoBK;;EAEf;;EAEA;;;;;;;;cASW,yBAAyB,OACpC,4BACA;;cAkDW,wCAEC;;;;;;UAOG;;EAEf;;EAEA;;;;;EAKA,OAAO;;;;;;;KAQG,eAAe;EACrB;EAAiB,OAAO;;EACxB;EAAkB;;;;;;;;;;;;;UAaP,gBAAgB,OAAO;;EAEtC,UAAU;EACV,UAAU;;EAEV;EACA,MAAM,OAAO,QAAQ,QAAQ,eAAe;;;;;;;;;;;;;UC5K7B;;;;;;EAMf,UAAU;;;;;;EAMV,SAAS;;;;;;;EAOT;;EAEA;;;;;;;;;;UAWe;;EAEf;;EAEA;;EAEA,SAAS;;EAET;;;;;;;;EAQA,gBAAgB"}

package/dist/verdict-cache-BCcOh0kF.js

@@ -0,0 +1,159 @@
+import { appendFileSync, existsSync, readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+//#region src/verdict-cache.ts
+/**
+* Content-addressed judge-verdict caching.
+*
+* LAW: cache JUDGE VERDICTS only — judging the same artifact with the same
+* judge+rubric is pure. NEVER cache agent rollouts. (A router that cached
+* identical fanout prompts silently destroyed best-of-N diversity; rollout
+* caching reintroduces that failure class. Judging has no diversity to
+* destroy — same artifact + same rubric ⇒ same verdict is the desired
+* property, not a bug.)
+*
+* The cache key is a sha-256 over the canonical JSON of everything that can
+* change a verdict: the artifact content, the scenario id, the judge name,
+* the full dimension list (key + description — the description IS the rubric
+* text shown to the judge), and a caller-supplied `judgeVersion`.
+* `judgeVersion` is REQUIRED: a judge whose prompt/model/ensemble changes
+* without a version bump would otherwise silently serve stale verdicts.
+*
+* Strict canonicalization (`canonicalJson`) throws on undefined / function /
+* symbol / non-finite numbers — an artifact that cannot be unambiguously
+* serialized cannot be content-addressed, and coercing it would let two
+* different artifacts collide on one key.
+*/
+function canonicalizeAt(value, path) {
+	if (value === null) return "null";
+	switch (typeof value) {
+		case "boolean": return value ? "true" : "false";
+		case "number":
+			if (!Number.isFinite(value)) throw new Error(`canonicalJson: non-finite number (${value}) at ${path} — ambiguity is an error, not a coercion`);
+			return JSON.stringify(value);
+		case "string": return JSON.stringify(value);
+		case "undefined":
+		case "function":
+		case "symbol": throw new Error(`canonicalJson: ${typeof value} at ${path} — ambiguity is an error, not a coercion`);
+		case "bigint": throw new Error(`canonicalJson: bigint at ${path} — not representable in JSON`);
+		case "object": break;
+	}
+	const obj = value;
+	if (typeof obj.toJSON === "function") return canonicalizeAt(obj.toJSON(), path);
+	if (Array.isArray(obj)) return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
+	if (obj instanceof Map || obj instanceof Set) throw new Error(`canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} — would serialize as '{}'; convert to a plain object/array first`);
+	return `{${Object.keys(obj).sort().map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`).join(",")}}`;
+}
+/**
+* Stable JSON stringify: object keys sorted recursively, so two semantically
+* equal values produce byte-identical output regardless of key insertion
+* order. Throws on undefined / function / symbol / NaN / ±Infinity / bigint /
+* Map / Set — anything JSON.stringify would coerce or drop silently.
+*
+* Distinct from `pre-registration.ts`'s `canonicalize`/`hashJson`, which are
+* permissive (coercion allowed) and async (web-crypto). Use THIS pair when a
+* hash collision or silent coercion would corrupt a cache key or attestation.
+*/
+function canonicalJson(value) {
+	return canonicalizeAt(value, "$");
+}
+/** Hex sha-256 over `canonicalJson(value)`. The content address used by the
+*  verdict cache and report attestation. */
+function contentHash(value) {
+	return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+/** Process-local Map-backed store. */
+function inMemoryVerdictCache() {
+	const entries = /* @__PURE__ */ new Map();
+	return {
+		get: (key) => entries.get(key),
+		set: (key, score) => {
+			entries.set(key, score);
+		}
+	};
+}
+function parseCacheLine(line, path, lineNo) {
+	let parsed;
+	try {
+		parsed = JSON.parse(line);
+	} catch (err) {
+		throw new Error(`fileVerdictCache: corrupt JSONL at ${path}:${lineNo} — ${err instanceof Error ? err.message : String(err)}`);
+	}
+	const rec = parsed;
+	if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") throw new Error(`fileVerdictCache: invalid record shape at ${path}:${lineNo} — expected {key, score:{dimensions, composite, notes}}`);
+	return rec;
+}
+/**
+* JSONL-file-backed store: the full file is loaded into an in-memory index at
+* construction; every `set` appends one line synchronously (durable before
+* the verdict is returned). A corrupt or malformed line throws at load with
+* file:line — a skipped line would silently re-judge (cost) or, worse, mask
+* a half-written file that needs operator attention.
+*/
+function fileVerdictCache(path) {
+	const entries = /* @__PURE__ */ new Map();
+	if (existsSync(path)) {
+		const lines = readFileSync(path, "utf8").split("\n");
+		for (let i = 0; i < lines.length; i++) {
+			const line = lines[i];
+			if (line === void 0 || line.trim() === "") continue;
+			const rec = parseCacheLine(line, path, i + 1);
+			entries.set(rec.key, rec.score);
+		}
+	}
+	return {
+		get: (key) => entries.get(key),
+		set: (key, score) => {
+			appendFileSync(path, `${JSON.stringify({
+				key,
+				score
+			})}\n`, "utf8");
+			entries.set(key, score);
+		}
+	};
+}
+/**
+* Wrap a `JudgeConfig` so repeat judgments of the same artifact are served
+* from the store instead of re-invoking `score()`. The wrapper is generic
+* over the judge's own type parameters and preserves `appliesTo` — it is a
+* drop-in replacement anywhere a `JudgeConfig` is accepted.
+*
+* A judge that throws is NOT cached: the error propagates and the next
+* attempt re-judges (caching a failure would pin a transient outage forever).
+*/
+function cachedJudge(judge, store, options) {
+	if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
+	const stats = {
+		hits: 0,
+		misses: 0
+	};
+	const wrapped = {
+		name: judge.name,
+		dimensions: judge.dimensions,
+		judgeVersion: options.judgeVersion,
+		async score(input) {
+			const key = contentHash({
+				artifact: canonicalJson(input.artifact),
+				scenarioId: input.scenario.id,
+				judgeName: judge.name,
+				dimensions: judge.dimensions,
+				judgeVersion: options.judgeVersion
+			});
+			const cached = await store.get(key);
+			if (cached !== void 0) {
+				stats.hits += 1;
+				return cached;
+			}
+			const score = await judge.score(input);
+			await store.set(key, score);
+			stats.misses += 1;
+			return score;
+		},
+		stats: () => ({ ...stats })
+	};
+	if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
+	return wrapped;
+}
+//#endregion
+export { inMemoryVerdictCache as a, fileVerdictCache as i, canonicalJson as n, contentHash as r, cachedJudge as t };
+
+//# sourceMappingURL=verdict-cache-BCcOh0kF.js.map

package/dist/verdict-cache-BCcOh0kF.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verdict-cache-BCcOh0kF.js","names":[],"sources":["../src/verdict-cache.ts"],"sourcesContent":["/**\n * Content-addressed judge-verdict caching.\n *\n * LAW: cache JUDGE VERDICTS only — judging the same artifact with the same\n * judge+rubric is pure. NEVER cache agent rollouts. (A router that cached\n * identical fanout prompts silently destroyed best-of-N diversity; rollout\n * caching reintroduces that failure class. Judging has no diversity to\n * destroy — same artifact + same rubric ⇒ same verdict is the desired\n * property, not a bug.)\n *\n * The cache key is a sha-256 over the canonical JSON of everything that can\n * change a verdict: the artifact content, the scenario id, the judge name,\n * the full dimension list (key + description — the description IS the rubric\n * text shown to the judge), and a caller-supplied `judgeVersion`.\n * `judgeVersion` is REQUIRED: a judge whose prompt/model/ensemble changes\n * without a version bump would otherwise silently serve stale verdicts.\n *\n * Strict canonicalization (`canonicalJson`) throws on undefined / function /\n * symbol / non-finite numbers — an artifact that cannot be unambiguously\n * serialized cannot be content-addressed, and coercing it would let two\n * different artifacts collide on one key.\n */\n\nimport { createHash } from 'node:crypto'\nimport { appendFileSync, existsSync, readFileSync } from 'node:fs'\nimport type { JudgeConfig, JudgeScore, Scenario } from './campaign/types'\n\n// ── canonical JSON + content hash ─────────────────────────────────────────\n\nfunction canonicalizeAt(value: unknown, path: string): string {\n  if (value === null) return 'null'\n  switch (typeof value) {\n    case 'boolean':\n      return value ? 'true' : 'false'\n    case 'number':\n      if (!Number.isFinite(value)) {\n        throw new Error(\n          `canonicalJson: non-finite number (${value}) at ${path} — ambiguity is an error, not a coercion`,\n        )\n      }\n      return JSON.stringify(value)\n    case 'string':\n      return JSON.stringify(value)\n    case 'undefined':\n    case 'function':\n    case 'symbol':\n      throw new Error(\n        `canonicalJson: ${typeof value} at ${path} — ambiguity is an error, not a coercion`,\n      )\n    case 'bigint':\n      throw new Error(`canonicalJson: bigint at ${path} — not representable in JSON`)\n    case 'object':\n      break\n  }\n  const obj = value as Record<string, unknown>\n  // Honor toJSON (Date → ISO string) before structural checks — without it a\n  // Date would canonicalize to '{}' and every timestamp would collide.\n  if (typeof obj.toJSON === 'function') {\n    return canonicalizeAt((obj as { toJSON(): unknown }).toJSON(), path)\n  }\n  if (Array.isArray(obj)) {\n    return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(',')}]`\n  }\n  if (obj instanceof Map || obj instanceof Set) {\n    throw new Error(\n      `canonicalJson: ${obj instanceof Map ? 'Map' : 'Set'} at ${path} — would serialize as '{}'; convert to a plain object/array first`,\n    )\n  }\n  const keys = Object.keys(obj).sort()\n  const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`)\n  return `{${parts.join(',')}}`\n}\n\n/**\n * Stable JSON stringify: object keys sorted recursively, so two semantically\n * equal values produce byte-identical output regardless of key insertion\n * order. Throws on undefined / function / symbol / NaN / ±Infinity / bigint /\n * Map / Set — anything JSON.stringify would coerce or drop silently.\n *\n * Distinct from `pre-registration.ts`'s `canonicalize`/`hashJson`, which are\n * permissive (coercion allowed) and async (web-crypto). Use THIS pair when a\n * hash collision or silent coercion would corrupt a cache key or attestation.\n */\nexport function canonicalJson(value: unknown): string {\n  return canonicalizeAt(value, '$')\n}\n\n/** Hex sha-256 over `canonicalJson(value)`. The content address used by the\n *  verdict cache and report attestation. */\nexport function contentHash(value: unknown): string {\n  return createHash('sha256').update(canonicalJson(value)).digest('hex')\n}\n\n// ── store contract ─────────────────────────────────────────────────────────\n\n/** Pluggable verdict store. Sync or async on both legs — `cachedJudge`\n *  awaits the results either way. */\nexport interface VerdictCacheStore {\n  get(key: string): Promise<JudgeScore | undefined> | JudgeScore | undefined\n  set(key: string, score: JudgeScore): Promise<void> | void\n}\n\n/** Process-local Map-backed store. */\nexport function inMemoryVerdictCache(): VerdictCacheStore {\n  const entries = new Map<string, JudgeScore>()\n  return {\n    get: (key) => entries.get(key),\n    set: (key, score) => {\n      entries.set(key, score)\n    },\n  }\n}\n\ninterface VerdictCacheLine {\n  key: string\n  score: JudgeScore\n}\n\nfunction parseCacheLine(line: string, path: string, lineNo: number): VerdictCacheLine {\n  let parsed: unknown\n  try {\n    parsed = JSON.parse(line)\n  } catch (err) {\n    throw new Error(\n      `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} — ${err instanceof Error ? err.message : String(err)}`,\n    )\n  }\n  const rec = parsed as Partial<VerdictCacheLine>\n  if (\n    typeof rec !== 'object' ||\n    rec === null ||\n    typeof rec.key !== 'string' ||\n    typeof rec.score !== 'object' ||\n    rec.score === null ||\n    typeof rec.score.composite !== 'number' ||\n    typeof rec.score.dimensions !== 'object'\n  ) {\n    throw new Error(\n      `fileVerdictCache: invalid record shape at ${path}:${lineNo} — expected {key, score:{dimensions, composite, notes}}`,\n    )\n  }\n  return rec as VerdictCacheLine\n}\n\n/**\n * JSONL-file-backed store: the full file is loaded into an in-memory index at\n * construction; every `set` appends one line synchronously (durable before\n * the verdict is returned). A corrupt or malformed line throws at load with\n * file:line — a skipped line would silently re-judge (cost) or, worse, mask\n * a half-written file that needs operator attention.\n */\nexport function fileVerdictCache(path: string): VerdictCacheStore {\n  const entries = new Map<string, JudgeScore>()\n  if (existsSync(path)) {\n    const lines = readFileSync(path, 'utf8').split('\\n')\n    for (let i = 0; i < lines.length; i++) {\n      const line = lines[i]\n      if (line === undefined || line.trim() === '') continue\n      const rec = parseCacheLine(line, path, i + 1)\n      entries.set(rec.key, rec.score)\n    }\n  }\n  return {\n    get: (key) => entries.get(key),\n    set: (key, score) => {\n      appendFileSync(path, `${JSON.stringify({ key, score })}\\n`, 'utf8')\n      entries.set(key, score)\n    },\n  }\n}\n\n// ── cached judge wrapper ───────────────────────────────────────────────────\n\nexport interface VerdictCacheStats {\n  hits: number\n  misses: number\n}\n\nexport interface CachedJudgeOptions {\n  /** REQUIRED — part of the cache key. Bump on any change to the judge's\n   *  prompt, model, ensemble, or scoring logic; silent judge upgrades must\n   *  never serve stale verdicts. */\n  judgeVersion: string\n}\n\n/** The wrapped judge: same `JudgeConfig` seam, plus hit/miss observability. */\nexport type CachedJudge<TArtifact, TScenario extends Scenario = Scenario> = JudgeConfig<\n  TArtifact,\n  TScenario\n> & {\n  stats(): VerdictCacheStats\n}\n\n/**\n * Wrap a `JudgeConfig` so repeat judgments of the same artifact are served\n * from the store instead of re-invoking `score()`. The wrapper is generic\n * over the judge's own type parameters and preserves `appliesTo` — it is a\n * drop-in replacement anywhere a `JudgeConfig` is accepted.\n *\n * A judge that throws is NOT cached: the error propagates and the next\n * attempt re-judges (caching a failure would pin a transient outage forever).\n */\nexport function cachedJudge<TArtifact, TScenario extends Scenario = Scenario>(\n  judge: JudgeConfig<TArtifact, TScenario>,\n  store: VerdictCacheStore,\n  options: CachedJudgeOptions,\n): CachedJudge<TArtifact, TScenario> {\n  if (typeof options.judgeVersion !== 'string' || options.judgeVersion.trim() === '') {\n    throw new Error('cachedJudge: judgeVersion is required and must be a non-empty string')\n  }\n  const stats: VerdictCacheStats = { hits: 0, misses: 0 }\n  const wrapped: CachedJudge<TArtifact, TScenario> = {\n    name: judge.name,\n    dimensions: judge.dimensions,\n    judgeVersion: options.judgeVersion,\n    async score(input) {\n      const key = contentHash({\n        artifact: canonicalJson(input.artifact),\n        scenarioId: input.scenario.id,\n        judgeName: judge.name,\n        dimensions: judge.dimensions,\n        judgeVersion: options.judgeVersion,\n      })\n      const cached = await store.get(key)\n      if (cached !== undefined) {\n        stats.hits += 1\n        return cached\n      }\n      const score = await judge.score(input)\n      await store.set(key, score)\n      stats.misses += 1\n      return score\n    },\n    stats: () => ({ ...stats }),\n  }\n  if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo\n  return wrapped\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA6BA,SAAS,eAAe,OAAgB,MAAsB;CAC5D,IAAI,UAAU,MAAM,OAAO;CAC3B,QAAQ,OAAO,OAAf;EACE,KAAK,WACH,OAAO,QAAQ,SAAS;EAC1B,KAAK;GACH,IAAI,CAAC,OAAO,SAAS,KAAK,GACxB,MAAM,IAAI,MACR,qCAAqC,MAAM,OAAO,KAAK,yCACzD;GAEF,OAAO,KAAK,UAAU,KAAK;EAC7B,KAAK,UACH,OAAO,KAAK,UAAU,KAAK;EAC7B,KAAK;EACL,KAAK;EACL,KAAK,UACH,MAAM,IAAI,MACR,kBAAkB,OAAO,MAAM,MAAM,KAAK,yCAC5C;EACF,KAAK,UACH,MAAM,IAAI,MAAM,4BAA4B,KAAK,6BAA6B;EAChF,KAAK,UACH;CACJ;CACA,MAAM,MAAM;CAGZ,IAAI,OAAO,IAAI,WAAW,YACxB,OAAO,eAAgB,IAA8B,OAAO,GAAG,IAAI;CAErE,IAAI,MAAM,QAAQ,GAAG,GACnB,OAAO,IAAI,IAAI,KAAK,MAAM,MAAM,eAAe,MAAM,GAAG,KAAK,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;CAEnF,IAAI,eAAe,OAAO,eAAe,KACvC,MAAM,IAAI,MACR,kBAAkB,eAAe,MAAM,QAAQ,MAAM,MAAM,KAAK,kEAClE;CAIF,OAAO,IAFM,OAAO,KAAK,GAAG,CAAC,CAAC,KACb,CAAC,CAAC,KAAK,MAAM,GAAG,KAAK,UAAU,CAAC,EAAE,GAAG,eAAe,IAAI,IAAI,GAAG,KAAK,GAAG,GAAG,GAC5E,CAAC,CAAC,KAAK,GAAG,EAAE;AAC7B;;;;;;;;;;;AAYA,SAAgB,cAAc,OAAwB;CACpD,OAAO,eAAe,OAAO,GAAG;AAClC;;;AAIA,SAAgB,YAAY,OAAwB;CAClD,OAAO,WAAW,QAAQ,CAAC,CAAC,OAAO,cAAc,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK;AACvE;;AAYA,SAAgB,uBAA0C;CACxD,MAAM,0BAAU,IAAI,IAAwB;CAC5C,OAAO;EACL,MAAM,QAAQ,QAAQ,IAAI,GAAG;EAC7B,MAAM,KAAK,UAAU;GACnB,QAAQ,IAAI,KAAK,KAAK;EACxB;CACF;AACF;AAOA,SAAS,eAAe,MAAc,MAAc,QAAkC;CACpF,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,IAAI;CAC1B,SAAS,KAAK;EACZ,MAAM,IAAI,MACR,sCAAsC,KAAK,GAAG,OAAO,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GAC3G;CACF;CACA,MAAM,MAAM;CACZ,IACE,OAAO,QAAQ,YACf,QAAQ,QACR,OAAO,IAAI,QAAQ,YACnB,OAAO,IAAI,UAAU,YACrB,IAAI,UAAU,QACd,OAAO,IAAI,MAAM,cAAc,YAC/B,OAAO,IAAI,MAAM,eAAe,UAEhC,MAAM,IAAI,MACR,6CAA6C,KAAK,GAAG,OAAO,wDAC9D;CAEF,OAAO;AACT;;;;;;;;AASA,SAAgB,iBAAiB,MAAiC;CAChE,MAAM,0BAAU,IAAI,IAAwB;CAC5C,IAAI,WAAW,IAAI,GAAG;EACpB,MAAM,QAAQ,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,IAAI;EACnD,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;GACrC,MAAM,OAAO,MAAM;GACnB,IAAI,SAAS,KAAA,KAAa,KAAK,KAAK,MAAM,IAAI;GAC9C,MAAM,MAAM,eAAe,MAAM,MAAM,IAAI,CAAC;GAC5C,QAAQ,IAAI,IAAI,KAAK,IAAI,KAAK;EAChC;CACF;CACA,OAAO;EACL,MAAM,QAAQ,QAAQ,IAAI,GAAG;EAC7B,MAAM,KAAK,UAAU;GACnB,eAAe,MAAM,GAAG,KAAK,UAAU;IAAE;IAAK;GAAM,CAAC,EAAE,KAAK,MAAM;GAClE,QAAQ,IAAI,KAAK,KAAK;EACxB;CACF;AACF;;;;;;;;;;AAiCA,SAAgB,YACd,OACA,OACA,SACmC;CACnC,IAAI,OAAO,QAAQ,iBAAiB,YAAY,QAAQ,aAAa,KAAK,MAAM,IAC9E,MAAM,IAAI,MAAM,sEAAsE;CAExF,MAAM,QAA2B;EAAE,MAAM;EAAG,QAAQ;CAAE;CACtD,MAAM,UAA6C;EACjD,MAAM,MAAM;EACZ,YAAY,MAAM;EAClB,cAAc,QAAQ;EACtB,MAAM,MAAM,OAAO;GACjB,MAAM,MAAM,YAAY;IACtB,UAAU,cAAc,MAAM,QAAQ;IACtC,YAAY,MAAM,SAAS;IAC3B,WAAW,MAAM;IACjB,YAAY,MAAM;IAClB,cAAc,QAAQ;GACxB,CAAC;GACD,MAAM,SAAS,MAAM,MAAM,IAAI,GAAG;GAClC,IAAI,WAAW,KAAA,GAAW;IACxB,MAAM,QAAQ;IACd,OAAO;GACT;GACA,MAAM,QAAQ,MAAM,MAAM,MAAM,KAAK;GACrC,MAAM,MAAM,IAAI,KAAK,KAAK;GAC1B,MAAM,UAAU;GAChB,OAAO;EACT;EACA,cAAc,EAAE,GAAG,MAAM;CAC3B;CACA,IAAI,MAAM,WAAW,QAAQ,YAAY,MAAM;CAC/C,OAAO;AACT"}

package/dist/wire/index.d.ts

@@ -1,7 +1,7 @@
-import { c as CostLedgerHandle } from "../cost-ledger-FuQvHxPm.js";
-import { Z as LlmRouteRequirements, q as LlmClientOptions } from "../types-BjMFz88h.js";
+import { c as CostLedgerHandle } from "../cost-ledger-Bv_e8XHY.js";
+import { Z as LlmRouteRequirements, q as LlmClientOptions } from "../types-D216SgwM.js";
 import { s as TraceStore } from "../store-CT9YIIve.js";
-import { _ as FeedbackTrajectoryStore } from "../feedback-trajectory-WK7x4mhy.js";
+import { _ as FeedbackTrajectoryStore } from "../feedback-trajectory-Rh280oXo.js";
 import { z } from "zod";
 import { ServerType } from "@hono/node-server";
 import { Hono } from "hono";

package/dist/wire/index.js

@@ -1,2 +1,2 @@
-import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-Df00sdwz.js";
+import { A as TraceEventSchema, C as HealthResponseSchema, D as RubricDimensionSchema, E as ListRubricsResponseSchema, F as hashRubric, M as TracesIngestResponseSchema, N as VersionResponseSchema, O as RubricInfoSchema, P as WIRE_VERSION, S as FeedbackTrajectorySchema, T as JudgeResultSchema, _ as ErrorResponseSchema, a as runRpcBatch, b as FeedbackIngestResponseSchema, c as WireError, d as handleListRubrics, f as handleTracesIngest, g as listBuiltinRubrics, h as getBuiltinRubric, i as dispatchRpc, j as TracesIngestRequestSchema, k as RubricSchema, l as handleFeedbackIngest, m as BUILTIN_RUBRICS, n as startServer, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi, t as createApp, u as handleJudge, v as FailureModeSchema, w as JudgeRequestSchema, x as FeedbackLabelSchema, y as FeedbackAttemptSchema } from "../server-iu0ede49.js";
 export { BUILTIN_RUBRICS, ErrorResponseSchema, FailureModeSchema, FeedbackAttemptSchema, FeedbackIngestResponseSchema, FeedbackLabelSchema, FeedbackTrajectorySchema, HealthResponseSchema, JudgeRequestSchema, JudgeResultSchema, ListRubricsResponseSchema, RubricDimensionSchema, RubricInfoSchema, RubricSchema, TraceEventSchema, TracesIngestRequestSchema, TracesIngestResponseSchema, VersionResponseSchema, WIRE_VERSION, WireError, buildOpenApi, createApp, dispatchRpc, getBuiltinRubric, handleFeedbackIngest, handleJudge, handleListRubrics, handleTracesIngest, handleVersion, hashRubric, listBuiltinRubrics, runRpcBatch, runRpcOnce, startServer, startServerAsync };

package/docs/building-doctrine.md

@@ -8,6 +8,21 @@ Every hard-coded model id or endpoint default is verifiable against the live rou
 
 Enforced by: `preflightModels` (membership + optional probe) and `assertModelsServed` (gate that names every unreachable id with status + detail).
 
+## 1a. Reachable is not the same as identified
+
+A 200 proves something answered, never that the requested model answered.
+A routing gateway can accept one id and reply from another, and the swap is silent: same status, same shape, only the response's `model` field betrays it.
+Every claim that names a model — a leaderboard row, a per-model cost, a cross-family judge panel, a non-self-judging exclusion — must be built from the id the provider echoed, not the id we sent.
+Requested is intent; served is evidence.
+
+Two failure shapes, and only one is a defect.
+Snapshot resolution (`gpt-4o-mini` → `gpt-4o-mini-2024-07-18`) pins a floating alias to a reproducible build and is the behaviour we want.
+Substitution (`gpt-4.1-mini` → `gemini-2.5-flash-lite`) mislabels every number the call produces.
+A response that echoes no model id at all is unproven, which fails closed rather than defaulting to agreement.
+
+Enforced by: `assertServedModel` / `assertServedModels` per call, `assertCrossFamilyServed` for panel diversity computed over the ids that answered, `LlmClientOptions.assertServedModel` to enforce it at the transport, and `assertModelsServed` (probe mode), which now fails a substituted id exactly as it fails a dead one.
+`assertCrossFamily` reads requested ids and therefore proves configuration only — reach for the served-side check wherever the diversity claim is load-bearing.
+
 ## 2. Probe the platform before peeling client layers
 
 When a request fails, one direct call against the live endpoint bisects platform-versus-client before any code-level debugging begins. A 401 from the router on a `model_not_found` is the platform telling you the default is dead; a connection refused is the platform being unreachable. Establish which side is at fault with a probe first, then debug only the side that is actually broken.

package/docs/charter.md

@@ -0,0 +1,112 @@
+# Charter: what agent-eval is for
+
+This document states what this package is, derived from the four end-states the stack must reach.
+It was written on 2026-08-10 from a measured inventory of this repo, agent-runtime, discovery, discovery-lab, braid, traces, and supervisor-lab.
+Every claim about current code cites the module that carries it.
+When behavior moves, move this document in the same change.
+
+## One sentence
+
+agent-eval is the honesty layer of the agent stack: instruments that make it structurally hard to fool ourselves at machine speed, and that never choose a research method.
+
+The anchor is discovery's covenant (discovery `docs/01-vision.md`): shared code enforces evidence integrity and immutable observations; it never chooses roles, methods, or winners.
+Everything above this package — runtime, discovery, braid, verticals — gets its freedom because honesty is enforced below.
+
+## The four end-states, and what each demands from this package
+
+1. **Build complex software rapidly, end to end, without issues.**
+   The gate is not generation.
+   The gate is knowing what is true about the work while it runs.
+   Measured on our own corpus: 87% of failed long runs end on a clean exit with the agent claiming success.
+   Demand: executable verification wired into the runtime loop, not beside it.
+
+2. **Do novel research: physics, quantum computing, math, unsolved problems.**
+   An unsolved problem has no held-out test suite by definition.
+   Every grader this package shipped before 2026-08 assumes an answer key.
+   Demand: verification strategies that certify without one — proof kernels, invariant checks, independent derivation agreement, replication — plus the statistics of a careful experimentalist.
+   This demand is measured, not speculative: discovery-lab holds 99 pursuit directories and 71 blind-graded oracle files on frontier problems, all verified today by hand-rolled run tools outside this package.
+
+3. **Build self-improving agents easily, and explain them easily.**
+   Improvement requires an ungameable signal; the grader, not the edit, decides improve-versus-game.
+   Explanation requires a portable artifact: the receipt that proves B beat A, verifiable by a third party who does not trust us.
+
+4. **Build new interfaces (braid-class): detached sessions, forking, analysts on tap.**
+   An interface can only expose what the layer below makes addressable.
+   Demand: sessions, traces, experiments, and verdicts as durable, forkable, queryable objects.
+
+## What already exists (the 2026-08-10 inventory, corrected)
+
+The fragmentation story is smaller than it feels.
+The audit refuted "built four times": traces imports this package's whole analyst suite; agent-runtime's live detectors import the detection kernel verbatim; supervisor-lab's judges are AgentProfiles dispatched through this package's judge primitives.
+
+- **Ask any question over any trace: exists.**
+  `TraceAnalysisEngine` (`src/analyst/engine.ts`) takes a free-text question.
+  A new custom question costs zero library files: `defineTraceAnalyst` + `runTraceAnalyst` on the `./analyst` subpath, with the DSPy RLM engine, seven byte-budgeted trace tools, and a metered model proxy behind it.
+- **Default failure analysts: exist.**
+  `buildDefaultAnalystRegistry` ships failure-mode, intent-divergence, knowledge-gap, knowledge-poisoning, improvement, control-integrity, and skill-usage kinds, engine-agnostic and versionable.
+- **Statistics: most of an A-plus toolkit, publicly exported.**
+  Paired bootstrap, clustered paired binary, exact and score risk differences, McNemar with power and required-n, MDE, multiplicity (Holm, Benjamini-Hochberg), e-process sequential gates, corpus inter-rater agreement, pre-registration manifests with content hashes.
+- **Executable process verification: proven this week.**
+  The trace-repair grader scores a proposed fix by executing it and running the task's own held-out suite from outside the container.
+  Oracle-fix separates from inert-probe (+0.353 vs 0.000 on milestone 1) with the floor pinned at zero.
+- **Integrity instruments hardened by this week's burns:**
+  served-model assertions (a gateway can answer one id with another model), oracle determinism certification (a wall-clock grader flipped 8 of 16 units on identical bytes), control-policy declaration (a zero-step control screened two milestones and could never fire), and equal-terms refusal between comparison arms (`repairArmAsymmetries`).
+
+## What is missing (the honest, short list)
+
+1. **Cluster-aware power with design-time refusal.**
+   "Four task clusters cannot certify any effect size, including 1.0" was learned by running the experiment.
+   A `clusteredPower` simulator must refuse the design before a dollar is spent.
+2. **Pre-registration as code, bound everywhere.**
+   The manifest binds exactly one statistic family today.
+   The registered decision rule must be the object the runner executes; drift between registered and ran must be unrepresentable.
+3. **A general prime query surface.**
+   The prime engine is benchmark-bound; there is no `runPrimeAnalyst` symmetric to `runTraceAnalyst`.
+4. **The funnel as a first-class object.**
+   Denominator chains are assembled by hand every run.
+5. **The unified analyst definition.**
+   One declarative unit — AgentProfile + evidence projection + reply contract + budget declaration — compiled to any engine, guarded by a byte-identity kill test against the bespoke arms.
+6. **Verification without an answer key.**
+   A strategy family where the held-out suite is one member: proof kernels, invariant and metamorphic checks, independent derivation agreement, replication.
+   The type layer now exists (`docs/verification-strategies.md`): the reward-source union is open, verdicts carry certifications, and the blind statement-equivalence protocol grades the formalization gap.
+   What remains is execution: a real kernel checker bound through the port, and the certified records flowing from live runs.
+7. **Session forking as one primitive.**
+   Both halves exist unjoined: agent-runtime's `SandboxLineage.fork` (live checkpoint) and this package's trajectory replay (recorded prefix).
+   Braid branches are metadata pointers; the provider session is always new.
+8. **The improvement receipt.**
+   Digests, attestations, and sealed manifests exist; the single portable, third-party-verifiable file does not.
+   Five gaming attacks on the receipt are named; each refusal must live inside the receipt.
+
+## Build order
+
+Wave 1 — in flight now.
+The analyst definition contract with its byte-identity CI kill test; prime as a HarnessType in agent-interface; the three-arm review fixes.
+
+Wave 2 — the experiment subpath. **Shipped: [docs/experiment.md](./experiment.md).**
+`./experiment`: compose the exported statistics into `defineExperiment` / `sealExperiment` with cluster-aware power refusal, pre-registration as executable decision rules, the funnel object, and matched-budget verification.
+Kill test first: re-derive this week's three hand-written PREREG.md files as decision-rule objects; any rule that needs an opaque escape hatch kills or extends the design.
+The kill test extended the design — ten node families beyond the seed AST, no opaque node — and the three preregistrations are the subpath's acceptance suite.
+
+Wave 3 — runtime wiring (blocked until agent-runtime's supervision merge resolves).
+The executable checker bound as a validator at the live-sandbox seam; a stop policy that consumes executable verdicts; then the supervisor budget-allocation experiment at equal compute.
+The published negative result to beat: StateSeal, −3.0pp, CI [−8.5, 1.1], n=540.
+Our measured headroom: blind continuation rescues 4.7% of rollouts; the done-signal has 62.5% precision.
+
+Wave 4 — the joins.
+`fork(session, step, modification)` joining lineage-fork and trajectory replay, consumed by braid; the improvement receipt v1 serializing evidence vector, pre-registration hash, grader calibration, and refusal outcomes into one attested file.
+
+Wave 5 — science.
+The verification-strategy family, proof kernels first.
+Its pilot is live: two independent Lean formalizations of the BCWW (4.6) inequality — one from the paper, one from the campaign's artifacts — with a kernel-checked equivalence verdict and the campaign's counterexample checked against both.
+A statement mismatch is a successful outcome; it is the formalization gap made visible.
+The contract layer is shipped: the family and checker port (`src/verification-strategy.ts`), verdict certifications (`src/verdict.ts`), and the blind two-arm equivalence protocol (`src/equivalence-check.ts`) — see `docs/verification-strategies.md` for the family, each member's failure mode, and the pilot as the worked example.
+
+## Standing principles (each earned by a measured burn)
+
+- **Instruments, never methods.** This package refuses to choose roles, prompts, models, or winners; it makes whatever runs honest.
+- **No upward dependencies.** Consumers import this package; never the reverse.
+- **A check that cannot run must never render as green.** (505 of 5,459 automated reviews published verdicts no evidence supported.)
+- **Certification is task-scoped.** A prompt certified on one task carries nothing onto another; re-authoring voids it.
+- **Access to the world beats loop sophistication.** Measured twice in one week: the analyst that could execute beat the one that could only read; framing carried more than the agent loop.
+- **Controls must be able to fire.** A screening control that cannot in principle produce the outcome it screens for is uncalibrated, not conservative.
+- **The refusal lives inside the artifact.** An adequacy check, an equal-terms check, a determinism check that runs beside the result can be skipped; one that the artifact carries cannot.