@prismatic-io/lux 0.0.1

package/src/orchestrator/experiment-report.ts

@@ -0,0 +1,788 @@
+import { basename } from "node:path";
+import { z } from "zod";
+import { valueHash } from "../core/index.js";
+import { deriveExperimentSelection, rankCandidates } from "./experiment.js";
+import {
+  type ExperimentRun,
+  ExperimentRunSchema,
+  experimentActiveElapsedMs,
+  loadExperimentRun,
+  verifyExperimentRunIntegrity,
+} from "./experiment-runs.js";
+
+const ExperimentReportSplitSchema = z.enum(["train", "validation", "test"]);
+const OptionalMetricSchema = z.number().finite().nullable();
+
+export const ExperimentDecisionReportSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    experimentDir: z.string(),
+    experimentId: z.string(),
+    campaignId: z.string(),
+    mode: z.enum(["experiment", "optimize"]),
+    evaluationKind: z.enum(["agent", "grader"]),
+    status: z.enum(["running", "passed", "failed", "aborted"]),
+    outcome: z.enum(["promote", "retain", "reject", "error", "incomplete"]),
+    error: z.string().nullable(),
+    candidateAndAccountingVerified: z.boolean(),
+    championCandidateId: z.string().nullable(),
+    ranking: z.array(z.string()),
+    eligibleCandidateIds: z.array(z.string()),
+    source: z
+      .object({
+        baselineBytes: z.number().int().nonnegative().nullable(),
+        championBytes: z.number().int().nonnegative().nullable(),
+        deltaBytes: z.number().int().nullable(),
+      })
+      .strict(),
+    gates: z.array(
+      z
+        .object({
+          metric: z.string(),
+          profile: z.string().nullable(),
+          split: ExperimentReportSplitSchema,
+          passed: z.boolean(),
+          values: z.array(z.number().finite()),
+          interval: z.string().nullable(),
+          note: z.string().nullable(),
+        })
+        .strict(),
+    ),
+    evidence: z.array(
+      z
+        .object({
+          candidateId: z.string(),
+          profileId: z.string(),
+          split: ExperimentReportSplitSchema,
+          repetitions: z.number().int().positive(),
+          caseIds: z.array(z.string()),
+          passRate: z.number().finite(),
+          passRateSamples: z.number().int().nonnegative().nullable(),
+          subjectInputTokens: OptionalMetricSchema,
+          subjectOutputTokens: OptionalMetricSchema,
+          subjectTotalTokens: OptionalMetricSchema,
+          subjectTokenSamples: z.number().int().nonnegative().nullable(),
+          evaluationTokens: z.number().nonnegative().nullable(),
+          meanQuestions: OptionalMetricSchema,
+          questionSamples: z.number().int().nonnegative().nullable(),
+          meanToolCalls: OptionalMetricSchema,
+          toolSamples: z.number().int().nonnegative().nullable(),
+          reportedCostUsd: z.number().nonnegative().nullable(),
+          unpricedModelCalls: z.number().int().nonnegative().nullable(),
+        })
+        .strict(),
+    ),
+    deltas: z.array(
+      z
+        .object({
+          profileId: z.string(),
+          split: ExperimentReportSplitSchema,
+          passRate: OptionalMetricSchema,
+          subjectTotalTokens: OptionalMetricSchema,
+          meanQuestions: OptionalMetricSchema,
+          meanToolCalls: OptionalMetricSchema,
+          reportedCostUsd: OptionalMetricSchema,
+        })
+        .strict(),
+    ),
+    spend: z
+      .object({
+        candidates: z.number().int().nonnegative(),
+        metricCalls: z.number().int().nonnegative(),
+        judgeCalls: z.number().int().nonnegative(),
+        proposalCalls: z.number().int().nonnegative(),
+        totalTokens: z.number().nonnegative(),
+        costUsd: z.number().nonnegative(),
+        unpricedModelCalls: z.number().int().nonnegative().nullable(),
+        activeTimeMs: z.number().nonnegative(),
+      })
+      .strict(),
+    budget: z
+      .object({
+        maxCandidates: z.number().int().positive(),
+        maxMetricCalls: z.number().int().positive(),
+        maxJudgeCalls: z.number().int().positive(),
+        maxProposalCalls: z.number().int().positive(),
+        maxTokens: z.number().int().positive(),
+        maxCostUsd: z.number().positive(),
+        maxUnpricedModelCalls: z.number().int().nonnegative(),
+        maxWallTimeMs: z.number().int().positive(),
+      })
+      .strict(),
+  })
+  .strict();
+export type ExperimentDecisionReport = z.infer<typeof ExperimentDecisionReportSchema>;
+
+export const VerifiedExperimentDecisionSnapshotSchema = z
+  .object({
+    experiment: ExperimentRunSchema,
+    report: ExperimentDecisionReportSchema,
+  })
+  .strict();
+export type VerifiedExperimentDecisionSnapshot = z.infer<
+  typeof VerifiedExperimentDecisionSnapshotSchema
+>;
+
+const concise = (value: number): string =>
+  Number.isInteger(value) ? String(value) : value.toFixed(4).replace(/0+$/, "").replace(/\.$/, "");
+
+const markdown = (value: string): string =>
+  [...value]
+    .map((character) => {
+      const code = character.codePointAt(0) ?? 0;
+      return code <= 0x1f || (code >= 0x7f && code <= 0x9f) ? " " : character;
+    })
+    .join("")
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll("`", "&#96;")
+    .replaceAll("!", "&#33;")
+    .replaceAll("[", "&#91;")
+    .replaceAll("]", "&#93;")
+    .replaceAll("(", "&#40;")
+    .replaceAll(")", "&#41;")
+    .replaceAll("\\", "&#92;")
+    .replaceAll("|", "\\|")
+    .replace(/\s*\n\s*/g, " ");
+
+const intervalText = (
+  interval:
+    | NonNullable<NonNullable<ExperimentRun["selection"]>["constraints"][number]["interval"]>
+    | undefined,
+): string | null => {
+  if (!interval) return null;
+  const resamples = interval.resamples > 0 ? `, resamples=${interval.resamples}` : "";
+  return `${interval.method} ${concise(interval.level * 100)}% ${concise(interval.estimate)} [${concise(interval.lower)}, ${concise(interval.upper)}], n=${interval.n}${resamples}`;
+};
+
+const reportOutcome = (
+  experiment: ExperimentRun,
+  championId: string | null,
+): ExperimentDecisionReport["outcome"] => {
+  if (experiment.status === "passed") return championId === "baseline" ? "retain" : "promote";
+  if (experiment.status === "failed" && experiment.errorMessage) return "error";
+  if (experiment.status === "failed" && experiment.selection) return "reject";
+  if (experiment.status === "failed") return "error";
+  return "incomplete";
+};
+
+type ExperimentSelection = NonNullable<ExperimentRun["selection"]>;
+type ExperimentCandidate = ExperimentRun["candidates"][number];
+type ExperimentSplit = ExperimentRun["evaluations"][number]["split"];
+
+const isCleanExperimentDecision = (experiment: ExperimentRun): boolean =>
+  experiment.status === "passed" || (experiment.status === "failed" && !experiment.errorMessage);
+
+const verifyIncompleteDecisionHasNoChampion = (experiment: ExperimentRun): void => {
+  if (experiment.championCandidateId || experiment.selection?.championCandidateId) {
+    throw new Error(`experiment status '${experiment.status}' cannot have a champion`);
+  }
+};
+
+const requireCompletedSelection = (experiment: ExperimentRun): ExperimentSelection => {
+  const selection = experiment.selection;
+  if (!selection) {
+    throw new Error(`completed experiment status '${experiment.status}' has no selection`);
+  }
+  if (!experiment.endedAt) {
+    throw new Error(
+      `completed experiment status '${experiment.status}' has no completion timestamp`,
+    );
+  }
+  if (
+    experiment.status === "passed" &&
+    (!experiment.championCandidateId || !selection.championCandidateId)
+  ) {
+    throw new Error("passed experiment has no complete champion selection");
+  }
+  return selection;
+};
+
+const requireCanonicalCandidatePopulation = (experiment: ExperimentRun): ExperimentCandidate => {
+  const candidateIds = experiment.candidates.map((candidate) => candidate.id);
+  const candidateSet = new Set(candidateIds);
+  if (candidateSet.size !== candidateIds.length) {
+    throw new Error("experiment candidate IDs must be unique");
+  }
+  const baseline = experiment.candidates.find((candidate) => candidate.id === "baseline");
+  if (!baseline) throw new Error("experiment selection has no baseline candidate");
+  return baseline;
+};
+
+const requireCandidateRecords = (
+  experiment: ExperimentRun,
+  candidateIds: readonly string[],
+  kind: "promotion finalist" | "decision candidate",
+): ExperimentCandidate[] =>
+  candidateIds.map((candidateId) => {
+    const candidate = experiment.candidates.find((entry) => entry.id === candidateId);
+    if (!candidate) throw new Error(`${kind} '${candidateId}' has no candidate record`);
+    return candidate;
+  });
+
+const requireCompletedEvidence = (
+  experiment: ExperimentRun,
+  candidateId: string,
+  profileId: string,
+  split: ExperimentSplit,
+  repetitions: number,
+  errorMessage: string,
+): void => {
+  const evaluation = experiment.evaluations.find(
+    (entry) =>
+      entry.candidateId === candidateId &&
+      entry.profileId === profileId &&
+      entry.split === split &&
+      entry.repetitions === repetitions,
+  );
+  if (!evaluation || Object.keys(evaluation.casePassRates).length === 0) {
+    throw new Error(errorMessage);
+  }
+};
+
+const verifyPromotionFinalistEvidence = (
+  experiment: ExperimentRun,
+  finalists: readonly ExperimentCandidate[],
+): void => {
+  const searchSplits: ExperimentSplit[] = [
+    ...(experiment.campaignSnapshot.splits.train ? (["train"] as const) : []),
+    "validation",
+  ];
+  for (const finalist of finalists) {
+    for (const profileId of experiment.campaignSnapshot.profiles) {
+      for (const split of searchSplits) {
+        requireCompletedEvidence(
+          experiment,
+          finalist.id,
+          profileId,
+          split,
+          experiment.campaignSnapshot.repetitions.search,
+          `optimize experiment finalist '${finalist.id}' lacks search ${split} evidence for profile '${profileId}'`,
+        );
+      }
+      requireCompletedEvidence(
+        experiment,
+        finalist.id,
+        profileId,
+        "validation",
+        experiment.campaignSnapshot.repetitions.promotion,
+        `optimize experiment finalist '${finalist.id}' lacks promotion evidence for profile '${profileId}'`,
+      );
+    }
+  }
+};
+
+const requirePromotionFinalists = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  baseline: ExperimentCandidate,
+): ExperimentCandidate[] => {
+  const optimizer = experiment.campaignSnapshot.optimizer;
+  if (!optimizer) throw new Error("optimize experiment has no optimizer snapshot");
+  const finalistIds = selection.promotionFinalistCandidateIds;
+  if (!finalistIds) {
+    throw new Error("optimize experiment selection has no promotion finalist checkpoint");
+  }
+  const expectedFinalists = 1 + Math.min(optimizer.promotionTopK, experiment.candidates.length - 1);
+  if (
+    finalistIds[0] !== baseline.id ||
+    finalistIds.length !== expectedFinalists ||
+    new Set(finalistIds).size !== finalistIds.length
+  ) {
+    throw new Error("optimize experiment has an invalid promotion finalist population");
+  }
+  return requireCandidateRecords(experiment, finalistIds, "promotion finalist");
+};
+
+const expectedOptimizationDecisionCandidates = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  baseline: ExperimentCandidate,
+): ExperimentCandidate[] => {
+  const finalists = requirePromotionFinalists(experiment, selection, baseline);
+  verifyPromotionFinalistEvidence(experiment, finalists);
+  const promoted =
+    rankCandidates(experiment.campaignSnapshot, finalists, experiment, [
+      "train",
+      "validation",
+    ])[0] ?? baseline;
+  return promoted.id === baseline.id ? [baseline] : [baseline, promoted];
+};
+
+const expectedDecisionCandidates = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  baseline: ExperimentCandidate,
+): ExperimentCandidate[] => {
+  if (experiment.mode === "optimize") {
+    return expectedOptimizationDecisionCandidates(experiment, selection, baseline);
+  }
+  if (selection.promotionFinalistCandidateIds) {
+    throw new Error("static experiment selection cannot have promotion finalists");
+  }
+  return experiment.candidates;
+};
+
+const requireDecisionCandidates = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  expectedCandidates: readonly ExperimentCandidate[],
+): { candidates: ExperimentCandidate[]; candidateIds: string[] } => {
+  const expectedDecisionCandidateIds = expectedCandidates.map((candidate) => candidate.id);
+  const decisionCandidateIds = selection.decisionCandidateIds ?? selection.rankedCandidateIds;
+  if (valueHash(decisionCandidateIds) !== valueHash(expectedDecisionCandidateIds)) {
+    throw new Error(
+      "experiment selection does not use the canonical decision candidate population",
+    );
+  }
+  const candidateSet = new Set(experiment.candidates.map((candidate) => candidate.id));
+  const decisionCandidateSet = new Set(decisionCandidateIds);
+  if (
+    decisionCandidateIds.length === 0 ||
+    decisionCandidateSet.size !== decisionCandidateIds.length ||
+    decisionCandidateIds.some((candidateId) => !candidateSet.has(candidateId))
+  ) {
+    throw new Error("experiment selection has an invalid decision candidate population");
+  }
+  return {
+    candidates: requireCandidateRecords(experiment, decisionCandidateIds, "decision candidate"),
+    candidateIds: decisionCandidateIds,
+  };
+};
+
+const verifyDecisionCandidateEvidence = (
+  experiment: ExperimentRun,
+  decisionCandidates: readonly ExperimentCandidate[],
+): void => {
+  const requiredSplits: ExperimentSplit[] = [
+    ...(experiment.campaignSnapshot.splits.train ? ["train" as const] : []),
+    "validation",
+    ...(experiment.campaignSnapshot.splits.test ? ["test" as const] : []),
+  ];
+  for (const candidate of decisionCandidates) {
+    for (const profileId of experiment.campaignSnapshot.profiles) {
+      for (const split of requiredSplits) {
+        const repetitions =
+          experiment.mode === "optimize" && split === "train"
+            ? experiment.campaignSnapshot.repetitions.search
+            : experiment.campaignSnapshot.repetitions.promotion;
+        requireCompletedEvidence(
+          experiment,
+          candidate.id,
+          profileId,
+          split,
+          repetitions,
+          `experiment selection is missing completed ${split} evidence for candidate '${candidate.id}' profile '${profileId}'`,
+        );
+      }
+    }
+  }
+};
+
+const deriveCanonicalSelection = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  decisionCandidates: readonly ExperimentCandidate[],
+): ExperimentSelection =>
+  deriveExperimentSelection(experiment.campaignSnapshot, decisionCandidates, experiment, {
+    decidedAt: selection.decidedAt,
+    ...(selection.promotionFinalistCandidateIds
+      ? { promotionFinalistCandidateIds: selection.promotionFinalistCandidateIds }
+      : {}),
+  });
+
+const verifyRecomputedSelection = (
+  experiment: ExperimentRun,
+  selection: ExperimentSelection,
+  decisionCandidateIds: readonly string[],
+  derived: ExperimentSelection,
+): void => {
+  const normalizedSelection = { ...selection, decisionCandidateIds };
+  if (valueHash(derived) !== valueHash(normalizedSelection)) {
+    throw new Error("experiment selection does not match recomputed evaluation evidence");
+  }
+  const derivedChampionId = derived.championCandidateId;
+  if (experiment.status === "passed") {
+    if (!experiment.endedAt) {
+      throw new Error("passed experiment has no completion timestamp");
+    }
+    if (
+      experiment.championCandidateId !== derivedChampionId ||
+      selection.championCandidateId !== derivedChampionId
+    ) {
+      throw new Error("experiment manifest and recomputed selection disagree on the champion");
+    }
+    return;
+  }
+  if (experiment.championCandidateId || selection.championCandidateId || derivedChampionId) {
+    throw new Error("failed experiment cannot have an eligible champion");
+  }
+};
+
+export const verifyExperimentDecisionIntegrity = (experiment: ExperimentRun): void => {
+  if (experiment.status === "passed" && experiment.errorMessage) {
+    throw new Error("passed experiment cannot retain an execution error");
+  }
+  if (!isCleanExperimentDecision(experiment)) {
+    verifyIncompleteDecisionHasNoChampion(experiment);
+    return;
+  }
+  const selection = requireCompletedSelection(experiment);
+  const baseline = requireCanonicalCandidatePopulation(experiment);
+  const expectedCandidates = expectedDecisionCandidates(experiment, selection, baseline);
+  const decision = requireDecisionCandidates(experiment, selection, expectedCandidates);
+  verifyDecisionCandidateEvidence(experiment, decision.candidates);
+  const derived = deriveCanonicalSelection(experiment, selection, decision.candidates);
+  verifyRecomputedSelection(experiment, selection, decision.candidateIds, derived);
+};
+
+const matchedMetricDelta = (
+  before: number | null,
+  after: number | null,
+  beforeSamples: number | null,
+  afterSamples: number | null,
+  expectedSamples: number,
+): number | null => {
+  if (before === null || after === null) return null;
+  if (beforeSamples === null || afterSamples === null) return null;
+  if (beforeSamples !== afterSamples) return null;
+  if (beforeSamples !== expectedSamples) return null;
+  return after - before;
+};
+
+const reportedCostDelta = (before: number | null, after: number | null): number | null => {
+  if (before === null || after === null) return null;
+  return after - before;
+};
+
+const projectExperimentDecisionReport = (
+  experimentDir: string,
+  experiment: ExperimentRun,
+  candidateAndAccountingVerified: boolean,
+): ExperimentDecisionReport => {
+  verifyExperimentDecisionIntegrity(experiment);
+  const championId = experiment.championCandidateId ?? null;
+  let decisionSelection: ExperimentRun["selection"];
+  if (experiment.status === "passed") {
+    decisionSelection = experiment.selection;
+  } else if (experiment.status === "failed" && !experiment.errorMessage) {
+    decisionSelection = experiment.selection;
+  }
+  const targetId = championId ?? decisionSelection?.rankedCandidateIds[0] ?? "baseline";
+  const baseline = experiment.candidates.find((candidate) => candidate.id === "baseline");
+  const champion = championId
+    ? experiment.candidates.find((candidate) => candidate.id === championId)
+    : undefined;
+  if (experiment.status === "passed" && !baseline) {
+    throw new Error("passed experiment has no baseline candidate record");
+  }
+  if (championId && !champion) {
+    throw new Error(`experiment champion '${championId}' has no candidate record`);
+  }
+  if (experiment.status === "passed" && (!experiment.selection || !championId)) {
+    throw new Error("passed experiment has no complete champion selection");
+  }
+  if (experiment.status !== "passed" && championId) {
+    throw new Error(`experiment status '${experiment.status}' cannot have a champion`);
+  }
+  if (experiment.selection && (experiment.selection.championCandidateId ?? null) !== championId) {
+    throw new Error("experiment manifest and selection disagree on the champion");
+  }
+  const usage = experiment.budgetUsage.usage;
+  const subjectMetricPrefix = experiment.evaluationKind === "agent" ? "agent" : "harness";
+  const relevantCandidates = new Set(["baseline", ...(championId ? [championId] : [targetId])]);
+  const evidenceByScope = new Map<string, ExperimentRun["evaluations"][number]>();
+  for (const evaluation of experiment.evaluations) {
+    if (!relevantCandidates.has(evaluation.candidateId)) continue;
+    const key = `${evaluation.candidateId}:${evaluation.profileId}:${evaluation.split}`;
+    const current = evidenceByScope.get(key);
+    if (!current || evaluation.repetitions > current.repetitions) {
+      evidenceByScope.set(key, evaluation);
+    }
+  }
+  const evidence = [...evidenceByScope.values()]
+    .sort((left, right) =>
+      `${left.candidateId}:${left.profileId}:${left.split}`.localeCompare(
+        `${right.candidateId}:${right.profileId}:${right.split}`,
+      ),
+    )
+    .map((evaluation) => {
+      const evaluationUsage = evaluation.usage;
+      const sampleCount = (metric: string): number | null =>
+        evaluation.samples?.[metric]?.length ?? null;
+      return {
+        candidateId: evaluation.candidateId,
+        profileId: evaluation.profileId,
+        split: evaluation.split,
+        repetitions: evaluation.repetitions,
+        caseIds: Object.keys(evaluation.casePassRates).sort(),
+        passRate: evaluation.passRate,
+        passRateSamples: sampleCount("quality.passRate"),
+        subjectInputTokens: evaluation.metrics[`${subjectMetricPrefix}.input`] ?? null,
+        subjectOutputTokens: evaluation.metrics[`${subjectMetricPrefix}.output`] ?? null,
+        subjectTotalTokens: evaluation.metrics[`${subjectMetricPrefix}.total`] ?? null,
+        subjectTokenSamples: sampleCount(`${subjectMetricPrefix}.total`),
+        evaluationTokens: evaluationUsage
+          ? evaluationUsage.input +
+            evaluationUsage.output +
+            evaluationUsage.cacheRead +
+            evaluationUsage.cacheCreation
+          : null,
+        meanQuestions: evaluation.metrics["interrupts.ask"] ?? null,
+        questionSamples: sampleCount("interrupts.ask"),
+        meanToolCalls: evaluation.metrics.toolCalls ?? null,
+        toolSamples: sampleCount("toolCalls"),
+        reportedCostUsd: evaluation.costUsd ?? null,
+        unpricedModelCalls: evaluation.unpricedModelCalls ?? null,
+      };
+    });
+  const deltas = evidence
+    .filter((entry) => championId !== null && entry.candidateId === championId)
+    .flatMap((after) => {
+      const before = evidence.find(
+        (entry) =>
+          entry.candidateId === "baseline" &&
+          entry.profileId === after.profileId &&
+          entry.split === after.split,
+      );
+      if (!before) return [];
+      const scopesMatch =
+        before.repetitions === after.repetitions &&
+        before.caseIds.join("\n") === after.caseIds.join("\n");
+      if (!scopesMatch) return [];
+      return [
+        {
+          profileId: after.profileId,
+          split: after.split,
+          passRate: matchedMetricDelta(
+            before.passRate,
+            after.passRate,
+            before.passRateSamples,
+            after.passRateSamples,
+            after.caseIds.length,
+          ),
+          subjectTotalTokens: matchedMetricDelta(
+            before.subjectTotalTokens,
+            after.subjectTotalTokens,
+            before.subjectTokenSamples,
+            after.subjectTokenSamples,
+            after.caseIds.length,
+          ),
+          meanQuestions: matchedMetricDelta(
+            before.meanQuestions,
+            after.meanQuestions,
+            before.questionSamples,
+            after.questionSamples,
+            after.caseIds.length,
+          ),
+          meanToolCalls: matchedMetricDelta(
+            before.meanToolCalls,
+            after.meanToolCalls,
+            before.toolSamples,
+            after.toolSamples,
+            after.caseIds.length,
+          ),
+          reportedCostUsd: reportedCostDelta(before.reportedCostUsd, after.reportedCostUsd),
+        },
+      ];
+    });
+  return ExperimentDecisionReportSchema.parse({
+    schemaVersion: 1,
+    experimentDir,
+    experimentId: experiment.experimentId,
+    campaignId: experiment.campaignId,
+    mode: experiment.mode,
+    evaluationKind: experiment.evaluationKind,
+    status: experiment.status,
+    error: experiment.errorMessage ?? null,
+    candidateAndAccountingVerified,
+    outcome: reportOutcome(experiment, championId),
+    championCandidateId: championId,
+    ranking: decisionSelection?.rankedCandidateIds ?? [],
+    eligibleCandidateIds: decisionSelection?.eligibleCandidateIds ?? [],
+    source: {
+      baselineBytes: baseline?.sourceBytes ?? null,
+      championBytes: champion?.sourceBytes ?? null,
+      deltaBytes: baseline && champion ? champion.sourceBytes - baseline.sourceBytes : null,
+    },
+    gates: (decisionSelection?.constraints ?? [])
+      .filter((constraint) => constraint.candidateId === targetId)
+      .map((constraint) => ({
+        metric: constraint.metric,
+        profile: constraint.profile ?? null,
+        split: constraint.split,
+        passed: constraint.passed,
+        values: constraint.values,
+        interval: intervalText(constraint.interval),
+        note: constraint.note ?? null,
+      })),
+    evidence,
+    deltas,
+    spend: {
+      candidates: experiment.candidates.length,
+      metricCalls: experiment.budgetUsage.metricCalls,
+      judgeCalls: experiment.budgetUsage.judgeCalls,
+      proposalCalls: experiment.budgetUsage.proposalCalls,
+      totalTokens: usage.input + usage.output + usage.cacheRead + usage.cacheCreation,
+      costUsd: experiment.budgetUsage.costUsd,
+      unpricedModelCalls: experiment.budgetUsage.unpricedModelCalls ?? null,
+      activeTimeMs: experimentActiveElapsedMs(experiment),
+    },
+    budget: {
+      maxCandidates: experiment.campaignSnapshot.budget.maxCandidates,
+      maxMetricCalls: experiment.campaignSnapshot.budget.maxMetricCalls,
+      maxJudgeCalls: experiment.campaignSnapshot.budget.maxJudgeCalls,
+      maxProposalCalls: experiment.campaignSnapshot.budget.maxProposalCalls,
+      maxTokens: experiment.campaignSnapshot.budget.maxTokens,
+      maxCostUsd: experiment.campaignSnapshot.budget.maxCostUsd,
+      maxUnpricedModelCalls: experiment.campaignSnapshot.budget.maxUnpricedModelCalls,
+      maxWallTimeMs: experiment.campaignSnapshot.budget.maxWallTimeMs,
+    },
+  });
+};
+
+export const createExperimentDecisionReport = (
+  experimentDir: string,
+  experiment: ExperimentRun,
+): ExperimentDecisionReport => projectExperimentDecisionReport(experimentDir, experiment, false);
+
+export const loadVerifiedExperimentDecisionSnapshot = async (
+  experimentDir: string,
+): Promise<VerifiedExperimentDecisionSnapshot> => {
+  const experiment = await loadExperimentRun(experimentDir);
+  if (!experiment) {
+    throw new Error(`experiment manifest not found or invalid at ${experimentDir}`);
+  }
+  await verifyExperimentRunIntegrity(experimentDir, experiment);
+  return VerifiedExperimentDecisionSnapshotSchema.parse({
+    experiment,
+    report: projectExperimentDecisionReport(experimentDir, experiment, true),
+  });
+};
+
+export const loadExperimentDecisionReport = async (
+  experimentDir: string,
+): Promise<ExperimentDecisionReport> =>
+  (await loadVerifiedExperimentDecisionSnapshot(experimentDir)).report;
+
+const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`;
+const shellPath = (value: string): string => {
+  if (
+    [...value].some((character) => {
+      const code = character.codePointAt(0) ?? 0;
+      return code <= 0x1f || (code >= 0x7f && code <= 0x9f);
+    })
+  ) {
+    throw new Error("experiment path contains control characters and cannot be rendered safely");
+  }
+  return shellQuote(value);
+};
+const signed = (value: number, suffix = ""): string =>
+  `${value > 0 ? "+" : ""}${concise(value)}${suffix}`;
+
+const decisionText = (report: ExperimentDecisionReport): string => {
+  switch (report.outcome) {
+    case "promote":
+      return `Promote \`${markdown(report.championCandidateId ?? "")}\`.`;
+    case "retain":
+      return "Retain the baseline; no authored candidate earned promotion.";
+    case "reject":
+      return "Do not promote; the experiment failed its authored gates.";
+    case "error":
+      return "No promotion decision; experiment execution failed before gate selection.";
+    case "incomplete":
+      return "No promotion decision is available; the experiment is incomplete.";
+  }
+};
+
+const optionalNumber = (value: number | null): string => (value === null ? "—" : concise(value));
+
+const sourceBytesText = (report: ExperimentDecisionReport): string => {
+  const baseline = report.source.baselineBytes ?? "—";
+  const champion = report.source.championBytes ?? "—";
+  if (report.source.deltaBytes === null) return `${baseline} → ${champion}`;
+  const sign = report.source.deltaBytes >= 0 ? "+" : "";
+  return `${baseline} → ${champion} (${sign}${report.source.deltaBytes})`;
+};
+
+export const formatExperimentDecisionMarkdown = (report: ExperimentDecisionReport): string => {
+  const decision = decisionText(report);
+  const lines = [
+    `# Lux experiment: ${markdown(report.campaignId)}`,
+    "",
+    `> **Decision:** ${decision}`,
+    "",
+    `- Experiment: \`${markdown(report.experimentId)}\``,
+    `- Mode: \`${report.mode}\``,
+    `- Evaluation: \`${report.evaluationKind}\``,
+    `- Status: **${report.status.toUpperCase()}**`,
+    `- Candidate/accounting integrity: **${report.candidateAndAccountingVerified ? "VERIFIED" : "NOT VERIFIED"}**`,
+    `- Champion: ${report.championCandidateId ? `\`${markdown(report.championCandidateId)}\`` : "—"}`,
+    `- Ranking: ${report.ranking.map((id) => `\`${markdown(id)}\``).join(" → ") || "—"}`,
+    ...(report.error ? [`- Error: ${markdown(report.error)}`] : []),
+    "",
+    "## Promotion gates",
+    "",
+  ];
+  if (report.gates.length === 0) {
+    lines.push("_No completed promotion gates._");
+  } else {
+    lines.push(
+      "| Result | Metric | Profile | Split | Values | Confidence | Note |",
+      "| --- | --- | --- | --- | ---: | --- | --- |",
+      ...report.gates.map(
+        (gate) =>
+          `| ${gate.passed ? "PASS" : "FAIL"} | ${markdown(gate.metric)} | ${markdown(gate.profile ?? "all")} | ${gate.split} | ${gate.values.map(concise).join(", ") || "—"} | ${markdown(gate.interval ?? "—")} | ${markdown(gate.note ?? "—")} |`,
+      ),
+    );
+  }
+  lines.push("", "## Baseline and decision evidence", "");
+  if (report.evidence.length === 0) {
+    lines.push("_No completed evaluation evidence._");
+  } else {
+    lines.push(
+      "| Candidate | Profile | Split | Reps | Cases | Pass rate | Pass n | Mean subject input | Mean subject output | Mean subject total | Subject n | Eval tokens | Mean questions | Question n | Mean tools | Tool n | Reported cost | Unpriced |",
+      "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+      ...report.evidence.map(
+        (entry) =>
+          `| ${markdown(entry.candidateId)} | ${markdown(entry.profileId)} | ${entry.split} | ${entry.repetitions} | ${entry.caseIds.length} | ${concise(entry.passRate)} | ${optionalNumber(entry.passRateSamples)} | ${optionalNumber(entry.subjectInputTokens)} | ${optionalNumber(entry.subjectOutputTokens)} | ${optionalNumber(entry.subjectTotalTokens)} | ${optionalNumber(entry.subjectTokenSamples)} | ${optionalNumber(entry.evaluationTokens)} | ${optionalNumber(entry.meanQuestions)} | ${optionalNumber(entry.questionSamples)} | ${optionalNumber(entry.meanToolCalls)} | ${optionalNumber(entry.toolSamples)} | ${entry.reportedCostUsd === null ? "—" : `$${entry.reportedCostUsd.toFixed(4)}`} | ${entry.unpricedModelCalls ?? "—"} |`,
+      ),
+    );
+  }
+  lines.push("", "## Change from baseline", "");
+  if (report.deltas.length === 0) {
+    lines.push("_No matched baseline/champion evidence._");
+  } else {
+    lines.push(
+      "| Profile | Split | Pass rate | Mean subject total | Mean questions | Mean tools | Reported cost |",
+      "| --- | --- | ---: | ---: | ---: | ---: | ---: |",
+      ...report.deltas.map(
+        (delta) =>
+          `| ${markdown(delta.profileId)} | ${delta.split} | ${delta.passRate === null ? "—" : signed(delta.passRate)} | ${delta.subjectTotalTokens === null ? "—" : signed(delta.subjectTotalTokens)} | ${delta.meanQuestions === null ? "—" : signed(delta.meanQuestions)} | ${delta.meanToolCalls === null ? "—" : signed(delta.meanToolCalls)} | ${delta.reportedCostUsd === null ? "—" : signed(delta.reportedCostUsd, " USD")} |`,
+      ),
+    );
+  }
+  lines.push(
+    "",
+    "## Source and spend",
+    "",
+    `- Source bytes: ${sourceBytesText(report)}`,
+    `- Candidates: ${report.spend.candidates}/${report.budget.maxCandidates}`,
+    `- Metric calls: ${report.spend.metricCalls}/${report.budget.maxMetricCalls}`,
+    `- Judge calls: ${report.spend.judgeCalls}/${report.budget.maxJudgeCalls}`,
+    `- Proposal calls: ${report.spend.proposalCalls}/${report.budget.maxProposalCalls}`,
+    `- Tokens: ${report.spend.totalTokens}/${report.budget.maxTokens}`,
+    `- Reported cost: $${report.spend.costUsd.toFixed(4)}/$${report.budget.maxCostUsd.toFixed(2)}`,
+    `- Unpriced model calls: ${report.spend.unpricedModelCalls ?? "unknown"}/${report.budget.maxUnpricedModelCalls}`,
+    `- Active wall time: ${report.spend.activeTimeMs}ms/${report.budget.maxWallTimeMs}ms`,
+    "",
+    "## Reproduce",
+    "",
+    `    lux view ${shellPath(report.experimentDir)}`,
+    ...(report.outcome === "promote" ? [`    lux apply ${shellPath(report.experimentDir)}`] : []),
+    "",
+    `Generated from the integrity-checked local experiment record \`${markdown(basename(report.experimentDir))}\`.`,
+  );
+  return lines.join("\n");
+};