@prismatic-io/lux 0.0.1

package/src/orchestrator/compare.ts

@@ -0,0 +1,358 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  emptyRunUsage,
+  type GradingReport,
+  GradingReportSchema,
+  harnessUsage,
+  type PersistedCase,
+  PersistedCaseSchema,
+  type RunMetadata,
+  RunMetadataSchema,
+  type RunUsage,
+  stableJson,
+  totalTokens,
+} from "../core/index.js";
+import { stableDriverIdentity } from "./comparison-identity.js";
+import { safeReadJson, walkFiles } from "./fs-read.js";
+
+export type ComparisonInput = {
+  pathA: string;
+  pathB: string;
+};
+
+export type RunBundle = {
+  path: string;
+  caseJson: PersistedCase | null;
+  runJson: RunMetadata | null;
+  grading: GradingReport | null;
+  /** Relative artifact path → sha256 of its bytes. */
+  fileTree: Map<string, string>;
+};
+
+export type GradingDelta = {
+  id: string;
+  type: string;
+  name: string;
+  before: { passed: boolean; score: number } | null;
+  after: { passed: boolean; score: number } | null;
+  status: "added" | "removed" | "improved" | "regressed" | "unchanged";
+};
+
+export type RunComparison = {
+  pathA: string;
+  pathB: string;
+  caseMismatch: { field: string; a: unknown; b: unknown }[];
+  runDelta: {
+    exitReason: { a: string; b: string };
+    durationMsA: number | null;
+    durationMsB: number | null;
+    interruptCounts: {
+      a: { ask: number; approve: number; custom: number };
+      b: { ask: number; approve: number; custom: number };
+    };
+    usage: { a: RunUsage; b: RunUsage };
+  } | null;
+  grading: {
+    casePassedA: boolean | null;
+    casePassedB: boolean | null;
+    caseScoreA: number | null;
+    caseScoreB: number | null;
+    deltas: GradingDelta[];
+    improvements: number;
+    regressions: number;
+  };
+  artifactsDelta: {
+    added: string[];
+    removed: string[];
+    changed: string[];
+    unchanged: string[];
+  };
+};
+
+// Content hashes, not sizes: a same-size edit must still read as changed.
+const hashFileTree = async (root: string): Promise<Map<string, string>> => {
+  const out = new Map<string, string>();
+  for (const rel of await walkFiles(root)) {
+    out.set(
+      rel,
+      createHash("sha256")
+        .update(await readFile(join(root, rel)))
+        .digest("hex"),
+    );
+  }
+  return out;
+};
+
+export const loadRunBundle = async (path: string): Promise<RunBundle> => ({
+  path,
+  caseJson: await safeReadJson(join(path, "case.json"), PersistedCaseSchema),
+  runJson: await safeReadJson(join(path, "run.json"), RunMetadataSchema),
+  grading: await safeReadJson(join(path, "grading.json"), GradingReportSchema),
+  fileTree: await hashFileTree(join(path, "artifacts")),
+});
+
+const durationFromMeta = (m: RunMetadata | null): number | null => {
+  if (!m) return null;
+  const a = Date.parse(m.startedAt);
+  const b = Date.parse(m.endedAt);
+  if (Number.isNaN(a) || Number.isNaN(b)) return null;
+  return b - a;
+};
+
+const computeFileDelta = (
+  a: Map<string, string>,
+  b: Map<string, string>,
+): RunComparison["artifactsDelta"] => {
+  const added: string[] = [];
+  const removed: string[] = [];
+  const changed: string[] = [];
+  const unchanged: string[] = [];
+  for (const [path, hashB] of b) {
+    if (!a.has(path)) {
+      added.push(path);
+    } else if (a.get(path) !== hashB) {
+      changed.push(path);
+    } else {
+      unchanged.push(path);
+    }
+  }
+  for (const path of a.keys()) {
+    if (!b.has(path)) removed.push(path);
+  }
+  return {
+    added: added.sort(),
+    removed: removed.sort(),
+    changed: changed.sort(),
+    unchanged: unchanged.sort(),
+  };
+};
+
+// Duplicate assertion names would collapse into one map entry and silently
+// drop deltas; later occurrences get an index suffix so each keeps its row.
+const assertionsByIdentity = (
+  report: GradingReport | null,
+): Map<string, GradingReport["assertions"][number]> => {
+  const occurrences = new Map<string, number>();
+  const out = new Map<string, GradingReport["assertions"][number]>();
+  for (const x of report?.assertions ?? []) {
+    if (x.id) {
+      out.set(x.id, x);
+      continue;
+    }
+    const n = (occurrences.get(x.name) ?? 0) + 1;
+    occurrences.set(x.name, n);
+    out.set(n === 1 ? x.name : `${x.name} (#${n})`, x);
+  }
+  return out;
+};
+
+const computeGradingDeltas = (
+  a: GradingReport | null,
+  b: GradingReport | null,
+): { deltas: GradingDelta[]; improvements: number; regressions: number } => {
+  const aByName = assertionsByIdentity(a);
+  const bByName = assertionsByIdentity(b);
+
+  const all = new Set([...aByName.keys(), ...bByName.keys()]);
+  const deltas: GradingDelta[] = [];
+  let improvements = 0;
+  let regressions = 0;
+  for (const name of [...all].sort()) {
+    const ax = aByName.get(name);
+    const bx = bByName.get(name);
+    let status: GradingDelta["status"];
+    if (!ax && bx) status = "added";
+    else if (ax && !bx) status = "removed";
+    else if (ax && bx) {
+      if (ax.passed === bx.passed && ax.score === bx.score) status = "unchanged";
+      // A pass/fail flip dominates any score movement.
+      else if (ax.passed !== bx.passed) status = bx.passed ? "improved" : "regressed";
+      else status = ax.score < bx.score ? "improved" : "regressed";
+    } else {
+      status = "unchanged";
+    }
+    if (status === "improved") improvements++;
+    if (status === "regressed") regressions++;
+    deltas.push({
+      id: name,
+      type: ax?.type ?? bx?.type ?? "unknown",
+      name: ax?.id || bx?.id ? (ax?.name ?? bx?.name ?? name) : name,
+      before: ax ? { passed: ax.passed, score: ax.score } : null,
+      after: bx ? { passed: bx.passed, score: bx.score } : null,
+      status,
+    });
+  }
+  return { deltas, improvements, regressions };
+};
+
+export const compareRuns = async (input: ComparisonInput): Promise<RunComparison> => {
+  const [a, b] = await Promise.all([loadRunBundle(input.pathA), loadRunBundle(input.pathB)]);
+
+  const caseMismatch: { field: string; a: unknown; b: unknown }[] = [];
+  for (const field of ["id", "prompt", "persona"] as const) {
+    const av = a.caseJson?.[field];
+    const bv = b.caseJson?.[field];
+    if (av !== bv) caseMismatch.push({ field, a: av, b: bv });
+  }
+  for (const field of ["fixtures", "assertions"] as const) {
+    const av = a.caseJson?.[field];
+    const bv = b.caseJson?.[field];
+    if (stableJson(av) !== stableJson(bv)) caseMismatch.push({ field, a: av, b: bv });
+  }
+  const fixturesHashA = a.runJson?.provenance?.fixturesHash;
+  const fixturesHashB = b.runJson?.provenance?.fixturesHash;
+  if (fixturesHashA !== fixturesHashB) {
+    caseMismatch.push({ field: "fixturesHash", a: fixturesHashA, b: fixturesHashB });
+  }
+  const assertionsHashA = a.runJson?.provenance?.assertionsHash;
+  const assertionsHashB = b.runJson?.provenance?.assertionsHash;
+  if (assertionsHashA !== assertionsHashB) {
+    caseMismatch.push({ field: "assertionsHash", a: assertionsHashA, b: assertionsHashB });
+  }
+  for (const field of [
+    "caseHash",
+    "driverConfigHash",
+    "answererConfigHash",
+    "evaluatorConfigHash",
+  ] as const) {
+    const av = a.runJson?.provenance?.[field];
+    const bv = b.runJson?.provenance?.[field];
+    if (av !== bv) caseMismatch.push({ field, a: av, b: bv });
+  }
+  const runtimeA = a.runJson?.provenance?.runtime;
+  const runtimeB = b.runJson?.provenance?.runtime;
+  if (stableJson(runtimeA) !== stableJson(runtimeB)) {
+    caseMismatch.push({ field: "runtime", a: runtimeA, b: runtimeB });
+  }
+  const sourceA = a.runJson?.provenance?.source;
+  const sourceB = b.runJson?.provenance?.source;
+  const sourceHashUnavailable =
+    (sourceA?.git?.dirty === true && !sourceA.git.workingTreeHash) ||
+    (sourceB?.git?.dirty === true && !sourceB.git.workingTreeHash);
+  if (sourceHashUnavailable) {
+    caseMismatch.push({
+      field: "sourceWorkingTreeHash",
+      a: sourceA?.git?.workingTreeHash,
+      b: sourceB?.git?.workingTreeHash,
+    });
+  } else if (stableJson(sourceA) !== stableJson(sourceB)) {
+    caseMismatch.push({ field: "source", a: sourceA, b: sourceB });
+  }
+  const driverA = a.runJson ? stableDriverIdentity(a.runJson.driver) : undefined;
+  const driverB = b.runJson ? stableDriverIdentity(b.runJson.driver) : undefined;
+  if (stableJson(driverA) !== stableJson(driverB)) {
+    caseMismatch.push({ field: "driver", a: driverA, b: driverB });
+  }
+  const answererA = a.runJson?.answerer;
+  const answererB = b.runJson?.answerer;
+  if (stableJson(answererA) !== stableJson(answererB)) {
+    caseMismatch.push({ field: "answerer", a: answererA, b: answererB });
+  }
+
+  const runDelta =
+    a.runJson || b.runJson
+      ? {
+          exitReason: {
+            a: a.runJson?.exitReason ?? "unknown",
+            b: b.runJson?.exitReason ?? "unknown",
+          },
+          durationMsA: durationFromMeta(a.runJson),
+          durationMsB: durationFromMeta(b.runJson),
+          interruptCounts: {
+            a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
+            b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
+          },
+          usage: {
+            a: a.runJson?.usage ?? emptyRunUsage(),
+            b: b.runJson?.usage ?? emptyRunUsage(),
+          },
+        }
+      : null;
+
+  const grading = computeGradingDeltas(a.grading, b.grading);
+
+  return {
+    pathA: input.pathA,
+    pathB: input.pathB,
+    caseMismatch,
+    runDelta,
+    grading: {
+      casePassedA: a.grading?.casePassed ?? null,
+      casePassedB: b.grading?.casePassed ?? null,
+      caseScoreA: a.grading?.caseScore ?? null,
+      caseScoreB: b.grading?.caseScore ?? null,
+      ...grading,
+    },
+    artifactsDelta: computeFileDelta(a.fileTree, b.fileTree),
+  };
+};
+
+// The CI gate. Per-assertion counting alone misses whole-run regressions: a
+// candidate that crashed before grading has every assertion "removed" (0
+// regressed), and one downgraded by `incomplete` keeps all assertions passing
+// — so a case that went from passed to anything-but-passed must also gate.
+export const hasRegression = (cmp: RunComparison): boolean =>
+  cmp.caseMismatch.length > 0 ||
+  cmp.grading.regressions > 0 ||
+  cmp.grading.deltas.some((delta) => delta.status === "removed") ||
+  (cmp.grading.casePassedA === true && cmp.grading.casePassedB !== true);
+
+const formatMismatchValue = (value: unknown): string => {
+  const rendered = JSON.stringify(value) ?? String(value);
+  return rendered.length > 500 ? `${rendered.slice(0, 499)}…` : rendered;
+};
+
+export const formatComparisonText = (cmp: RunComparison): string => {
+  const lines: string[] = [];
+  lines.push(`A: ${cmp.pathA}`);
+  lines.push(`B: ${cmp.pathB}`);
+  lines.push("");
+  if (cmp.caseMismatch.length > 0) {
+    lines.push("Case mismatch:");
+    for (const mismatch of cmp.caseMismatch) {
+      lines.push(
+        `  ${mismatch.field}: A=${formatMismatchValue(mismatch.a)} | B=${formatMismatchValue(mismatch.b)}`,
+      );
+    }
+    lines.push("");
+  }
+  if (cmp.runDelta) {
+    lines.push("Run delta:");
+    lines.push(`  exit:        ${cmp.runDelta.exitReason.a} → ${cmp.runDelta.exitReason.b}`);
+    lines.push(
+      `  duration ms: ${cmp.runDelta.durationMsA ?? "?"} → ${cmp.runDelta.durationMsB ?? "?"}`,
+    );
+    const ai = cmp.runDelta.interruptCounts.a;
+    const bi = cmp.runDelta.interruptCounts.b;
+    lines.push(
+      `  interrupts:  ask=${ai.ask}/approve=${ai.approve}/custom=${ai.custom} → ` +
+        `ask=${bi.ask}/approve=${bi.approve}/custom=${bi.custom}`,
+    );
+    const au = cmp.runDelta.usage.a;
+    const bu = cmp.runDelta.usage.b;
+    lines.push(`  agent tokens: ${totalTokens(au.agent)} → ${totalTokens(bu.agent)}`);
+    lines.push(
+      `  harness tokens: ${totalTokens(harnessUsage(au))} → ${totalTokens(harnessUsage(bu))}`,
+    );
+    lines.push("");
+  }
+  lines.push("Grading:");
+  lines.push(
+    `  passed: ${String(cmp.grading.casePassedA)} → ${String(cmp.grading.casePassedB)}; ` +
+      `score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}`,
+  );
+  lines.push(`  ${cmp.grading.improvements} improved, ${cmp.grading.regressions} regressed`);
+  for (const d of cmp.grading.deltas) {
+    if (d.status === "unchanged") continue;
+    const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓" }[d.status];
+    lines.push(`    ${arrow} ${d.name}`);
+  }
+  lines.push("");
+  lines.push("Artifacts:");
+  lines.push(
+    `  +${cmp.artifactsDelta.added.length} -${cmp.artifactsDelta.removed.length} ~${cmp.artifactsDelta.changed.length}`,
+  );
+  return lines.join("\n");
+};

package/src/orchestrator/comparison-identity.ts

@@ -0,0 +1,45 @@
+import type { RunMetadata } from "../core/index.js";
+import { valueHash } from "../core/index.js";
+
+const stableDriverIdentity = (driver: RunMetadata["driver"]): unknown => {
+  const { detail: _detail, ...identity } = driver;
+  const detail = driver.detail
+    ? Object.fromEntries(
+        Object.entries(driver.detail).filter(([key]) => key !== "sessionId" && key !== "threadId"),
+      )
+    : undefined;
+  return {
+    ...identity,
+    ...(detail && Object.keys(detail).length > 0 ? { detail } : {}),
+  };
+};
+
+const sourceIdentity = (metadata: RunMetadata): unknown => {
+  const source = metadata.provenance?.source;
+  if (!source?.git?.dirty || source.git.workingTreeHash) return source;
+  // Old or oversized dirty runs have no content identity. Keep them from
+  // becoming false repetitions merely because commit + `dirty: true` match.
+  return { ...source, dirtySnapshotUnavailableForRun: metadata.startedAt };
+};
+
+/**
+ * Identity that must match before suite samples can be treated as repeated
+ * measurements. Subject/candidate hashes are intentionally excluded: those
+ * are the independent variable a comparison is meant to evaluate.
+ */
+export const runComparisonIdentity = (metadata: RunMetadata): string =>
+  valueHash({
+    source: sourceIdentity(metadata),
+    runtime: metadata.provenance?.runtime,
+    caseHash: metadata.provenance?.caseHash,
+    assertionsHash: metadata.provenance?.assertionsHash,
+    fixturesHash: metadata.provenance?.fixturesHash,
+    profileId: metadata.provenance?.profileId,
+    driverConfigHash: metadata.provenance?.driverConfigHash,
+    answererConfigHash: metadata.provenance?.answererConfigHash,
+    evaluatorConfigHash: metadata.provenance?.evaluatorConfigHash,
+    driver: stableDriverIdentity(metadata.driver),
+    answerer: metadata.answerer,
+  });
+
+export { stableDriverIdentity };

package/src/orchestrator/config.ts

@@ -0,0 +1,205 @@
+import { existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { z } from "zod";
+import { claudeCodeAnswerer } from "../answerers/claude-code/index.js";
+import { personaAnswerer } from "../answerers/persona/index.js";
+import { scriptedAnswerer } from "../answerers/scripted/index.js";
+import { terminalAnswerer } from "../answerers/terminal/index.js";
+import { allCoreAssertions } from "../assertions/core/index.js";
+import { allMcpAssertions } from "../assertions/mcp/index.js";
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+import {
+  type AnswererPlugin,
+  AnswererRefSchema,
+  type Assertion,
+  type AssertionPlugin,
+  assertionPluginIssue,
+  type DriverPlugin,
+  DriverRefSchema,
+  formatZodIssues,
+  HarnessModelConfigSchema,
+  type LifecycleHooksPlugin,
+  lifecycleHooksPluginIssue,
+  Registry,
+} from "../core/index.js";
+import { claudeCodeDriver } from "../drivers/claude-code/index.js";
+import { codexDriver } from "../drivers/codex/index.js";
+import { mcpProbeDriver } from "../drivers/mcp/index.js";
+import { subprocessDriver } from "../drivers/subprocess/index.js";
+
+const pluginLike = z.looseObject({
+  name: z.string(),
+  create: z.custom<unknown>((v) => typeof v === "function"),
+});
+
+export const ExperimentProfileSchema = z
+  .object({
+    driver: DriverRefSchema.optional(),
+    answerer: AnswererRefSchema.optional(),
+    harness: HarnessModelConfigSchema.optional(),
+  })
+  .strict();
+export type ExperimentProfile = z.infer<typeof ExperimentProfileSchema>;
+
+// `z.custom` validates in place; `z.object` would rebuild a class plugin and
+// strip its prototype.
+const asPlugin = <T>(shape: z.ZodType, label: string) =>
+  z.custom<T>((v) => shape.safeParse(v).success, { message: `expected ${label}` });
+
+const AssertionPluginSchema = z.custom<AssertionPlugin<Assertion>>().superRefine((value, ctx) => {
+  if (!value || typeof value !== "object") {
+    ctx.addIssue({ code: "custom", message: "assertion plugin must be an object" });
+    return;
+  }
+  const issue = assertionPluginIssue(value);
+  if (issue) ctx.addIssue({ code: "custom", message: issue });
+});
+
+const LifecycleHooksPluginSchema = z.custom<LifecycleHooksPlugin>().superRefine((value, ctx) => {
+  if (!value || typeof value !== "object") {
+    ctx.addIssue({
+      code: "custom",
+      message: "lifecycle hooks plugin must be an object",
+    });
+    return;
+  }
+  const issue = lifecycleHooksPluginIssue(value);
+  if (issue) ctx.addIssue({ code: "custom", message: issue });
+});
+
+const LuxConfigSchema = z
+  .object({
+    casesRoot: z.string().optional(),
+    fixturesRoot: z.string().optional(),
+    runsRoot: z.string().default(".lux-runs"),
+    /** Root bound into case values created with `subjectPath()`. */
+    subjectRoot: z.string().optional(),
+    defaultAnswerer: AnswererRefSchema.default("persona"),
+    defaultDriver: DriverRefSchema.optional(),
+    /** Shared provider/model defaults for persona answering and rubric judging. */
+    harness: HarnessModelConfigSchema.optional(),
+    concurrency: z.number().int().min(1).default(1),
+    profiles: z.record(z.string(), ExperimentProfileSchema).optional(),
+    drivers: z
+      .array(asPlugin<DriverPlugin<never>>(pluginLike, "a driver plugin: { name, create }"))
+      .optional(),
+    answerers: z
+      .array(asPlugin<AnswererPlugin<never>>(pluginLike, "an answerer plugin: { name, create }"))
+      .optional(),
+    assertions: z.array(AssertionPluginSchema).optional(),
+    lifecycleFixtures: z
+      .array(LifecycleHooksPluginSchema)
+      .superRefine((fixtures, ctx) => {
+        const names = new Set<string>();
+        for (const [index, fixture] of fixtures.entries()) {
+          if (names.has(fixture.name)) {
+            ctx.addIssue({
+              code: "custom",
+              message: `duplicate lifecycle fixture plugin '${fixture.name}'`,
+              path: [index, "name"],
+            });
+          }
+          names.add(fixture.name);
+        }
+      })
+      .optional(),
+  })
+  .strict();
+
+/**
+ * Author-facing config before schema defaults are applied.
+ *
+ * Keep this as the schema input: using `z.infer`/`z.output` here makes
+ * defaulted fields look required in otherwise-valid `lux.config.ts` files.
+ */
+export type LuxConfig = z.input<typeof LuxConfigSchema>;
+type ParsedLuxConfig = z.output<typeof LuxConfigSchema>;
+
+export type ResolvedLuxConfig = ParsedLuxConfig & {
+  configPath: string | null;
+  rootDir: string;
+};
+
+const CONFIG_FILENAMES = ["lux.config.ts", "lux.config.js", "lux.config.mjs"];
+
+export const findConfigFile = (cwd: string): string | null => {
+  for (const name of CONFIG_FILENAMES) {
+    const path = join(cwd, name);
+    if (existsSync(path)) return path;
+  }
+  return null;
+};
+
+export const loadLuxConfig = async (cwd: string = process.cwd()): Promise<ResolvedLuxConfig> => {
+  const root = resolve(cwd);
+  const configPath = findConfigFile(root);
+  if (!configPath) {
+    return {
+      configPath: null,
+      rootDir: root,
+      runsRoot: join(root, ".lux-runs"),
+      defaultAnswerer: "persona",
+      concurrency: 1,
+    };
+  }
+  const mod = (await import(pathToFileURL(configPath).href)) as { default?: unknown };
+  if (mod.default === undefined) {
+    throw new Error(
+      `Lux config at ${configPath} has no default export. Use \`export default { ... }\`.`,
+    );
+  }
+  const parsed = LuxConfigSchema.safeParse(mod.default);
+  if (!parsed.success) {
+    throw new Error(`Invalid Lux config at ${configPath}: ${formatZodIssues(parsed.error)}`);
+  }
+  const validated = parsed.data;
+  return {
+    configPath,
+    rootDir: root,
+    runsRoot: resolve(root, validated.runsRoot),
+    ...(validated.subjectRoot ? { subjectRoot: resolve(root, validated.subjectRoot) } : {}),
+    ...(validated.casesRoot ? { casesRoot: resolve(root, validated.casesRoot) } : {}),
+    ...(validated.fixturesRoot ? { fixturesRoot: resolve(root, validated.fixturesRoot) } : {}),
+    defaultAnswerer: validated.defaultAnswerer,
+    ...(validated.defaultDriver ? { defaultDriver: validated.defaultDriver } : {}),
+    ...(validated.harness ? { harness: validated.harness } : {}),
+    concurrency: validated.concurrency,
+    ...(validated.profiles ? { profiles: validated.profiles } : {}),
+    ...(validated.drivers ? { drivers: validated.drivers } : {}),
+    ...(validated.answerers ? { answerers: validated.answerers } : {}),
+    ...(validated.assertions ? { assertions: validated.assertions } : {}),
+    ...(validated.lifecycleFixtures ? { lifecycleFixtures: validated.lifecycleFixtures } : {}),
+  };
+};
+
+const builtinDrivers: DriverPlugin<never>[] = [
+  claudeCodeDriver,
+  codexDriver,
+  subprocessDriver,
+  mcpProbeDriver,
+];
+const builtinAnswerers: AnswererPlugin<never>[] = [
+  personaAnswerer,
+  claudeCodeAnswerer,
+  scriptedAnswerer,
+  terminalAnswerer,
+];
+export const buildRegistryFromConfig = (config: ResolvedLuxConfig): Registry => {
+  const registry = new Registry();
+  for (const d of config.drivers ?? []) registry.registerDriver(d);
+  for (const a of config.answerers ?? []) registry.registerAnswerer(a);
+  for (const a of config.assertions ?? []) registry.registerAssertion(a);
+  // Config plugins claim their names first so a config can shadow a bundled
+  // plugin; built-ins fill whatever remains.
+  for (const d of builtinDrivers) if (!registry.getDriver(d.name)) registry.registerDriver(d);
+  for (const a of builtinAnswerers) if (!registry.getAnswerer(a.name)) registry.registerAnswerer(a);
+  const builtinAssertions = [
+    ...allCoreAssertions,
+    createRubricAssertion(config.harness),
+    ...allMcpAssertions,
+  ];
+  for (const a of builtinAssertions)
+    if (!registry.getAssertion(a.type)) registry.registerAssertion(a);
+  return registry;
+};

package/src/orchestrator/content-bound-json.ts

@@ -0,0 +1,71 @@
+import { randomUUID } from "node:crypto";
+import { link, mkdir, rm, unlink, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { z } from "zod";
+import { valueHash } from "../core/index.js";
+import { safeReadJson } from "./fs-read.js";
+
+const markerSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    evaluationHash: z.string().min(1),
+    contentHash: z.exactOptional(z.string().min(1)),
+  })
+  .strict();
+
+const markerPath = (path: string): string => `${path}.identity.json`;
+
+const publishExclusiveJson = async (path: string, value: unknown): Promise<void> => {
+  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
+  try {
+    await link(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+};
+
+export const retireContentBoundJson = async (path: string): Promise<void> => {
+  for (const target of [path, markerPath(path)]) {
+    await unlink(target).catch((error) => {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    });
+  }
+};
+
+export const readContentBoundJson = async <Schema extends z.ZodType>(
+  path: string,
+  schema: Schema,
+  evaluationHash: string,
+): Promise<z.infer<Schema> | null> => {
+  const marker = await safeReadJson(markerPath(path), markerSchema);
+  if (!marker?.contentHash || marker.evaluationHash !== evaluationHash) {
+    await retireContentBoundJson(path);
+    return null;
+  }
+  const value = await safeReadJson(path, schema);
+  if (!value) {
+    await retireContentBoundJson(path);
+    return null;
+  }
+  if (marker.contentHash !== valueHash(value)) {
+    throw new Error(`cached JSON failed its content check: ${path}`);
+  }
+  return value;
+};
+
+export const writeContentBoundJson = async (
+  path: string,
+  value: unknown,
+  evaluationHash: string,
+  beforeBinding?: () => Promise<void>,
+): Promise<void> => {
+  await mkdir(dirname(path), { recursive: true });
+  await publishExclusiveJson(path, value);
+  await beforeBinding?.();
+  await publishExclusiveJson(markerPath(path), {
+    schemaVersion: 1,
+    evaluationHash,
+    contentHash: valueHash(value),
+  });
+};