@prismatic-io/lux 0.0.1

package/src/orchestrator/experiment-evaluation.ts

@@ -0,0 +1,749 @@
+import { randomUUID } from "node:crypto";
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { z } from "zod";
+import {
+  createRubricAssertion,
+  RubricModelAccountingDetailSchema,
+} from "../assertions/rubric/index.js";
+import {
+  addUsage,
+  type Candidate,
+  type CandidateEvaluation,
+  CandidateEvaluationSchema,
+  type ExperimentSelector,
+  emptyUsage,
+  TokenUsageSchema,
+  valueHash,
+} from "../core/index.js";
+import { materializeCandidate } from "./candidates.js";
+import { readContentBoundJson, writeContentBoundJson } from "./content-bound-json.js";
+import { type discoverCases, selectCases } from "./discover.js";
+import { ensureBudget, throwIfAborted } from "./experiment-budget.js";
+import type { CampaignExecutionOptions } from "./experiment-context.js";
+import { profileOrThrow } from "./experiment-context.js";
+import type { ExperimentEvaluation, OpenedExperimentRun } from "./experiment-runs.js";
+import { average } from "./experiment-selection.js";
+import { evaluateGraderCandidate } from "./grader-experiment.js";
+import { loadEvalCase } from "./loader.js";
+import { Orchestrator, type SuiteCaseResult, type SuiteItem } from "./orchestrator.js";
+import { loadSuiteSummary, type SuiteSummary } from "./suite-summary.js";
+
+type ExperimentSplit = "train" | "validation" | "test";
+
+const gradingFailureMessage = (results: readonly SuiteCaseResult[]): string | undefined => {
+  for (const result of results) {
+    if (!result.result) {
+      return result.errorMessage
+        ? `case '${result.caseId}' failed operationally: ${result.errorMessage}`
+        : `case '${result.caseId}' produced no durable run result`;
+    }
+    const operationalFailure = result.result.metadata.operationalFailure;
+    if (operationalFailure) {
+      return `case '${result.caseId}' failed during ${operationalFailure.phase}: ${operationalFailure.reason}`;
+    }
+    const report = result.result?.grading;
+    const failedAssertion = report.assertions.find((assertion) => {
+      if (assertion.errorMessage) return true;
+      const detail =
+        assertion.detail && typeof assertion.detail === "object"
+          ? (assertion.detail as Record<string, unknown>)
+          : undefined;
+      return typeof detail?.graderError === "string";
+    });
+    if (failedAssertion) {
+      return (
+        failedAssertion.errorMessage ??
+        failedAssertion.comment ??
+        `assertion '${failedAssertion.id ?? failedAssertion.name}' could not produce a judgment`
+      );
+    }
+    if (report.incomplete?.phase === "grading" || report.incomplete?.phase === "collection") {
+      return `${report.incomplete.phase} did not complete`;
+    }
+  }
+  return undefined;
+};
+
+const failedCases = (results: readonly SuiteCaseResult[]): CandidateEvaluation["failedCases"] =>
+  results.flatMap((result) => {
+    if (result.exitReason === "done" && result.casePassed === true) return [];
+    const feedback =
+      result.result?.grading.assertions
+        .filter((assertion) => !assertion.passed)
+        .map(
+          (assertion) =>
+            `${assertion.id ?? assertion.name}: ${assertion.comment ?? assertion.errorMessage ?? `score ${assertion.score}`}`,
+        ) ?? [];
+    if (feedback.length === 0) {
+      feedback.push(
+        result.errorMessage ??
+          (result.exitReason === "done" ? "case did not pass" : `run ended ${result.exitReason}`),
+      );
+    }
+    return [
+      {
+        caseId: result.caseId,
+        score: result.result?.grading.caseScore ?? null,
+        feedback,
+      },
+    ];
+  });
+
+const evaluationMetrics = (
+  summary: SuiteSummary,
+  candidate: Candidate,
+): Record<string, number> => ({
+  "quality.passRate": summary.totals.passRate,
+  "subject.bytes": candidate.sourceBytes,
+  ...Object.fromEntries(
+    Object.entries(summary.metrics).map(([name, metric]) => [name, metric.mean]),
+  ),
+});
+
+const summaryUsage = (summary: SuiteSummary) =>
+  summary.samples.reduce(
+    (usage, sample) =>
+      addUsage(usage, {
+        input: (sample.metrics["agent.input"] ?? 0) + (sample.metrics["harness.input"] ?? 0),
+        output: (sample.metrics["agent.output"] ?? 0) + (sample.metrics["harness.output"] ?? 0),
+        cacheRead:
+          (sample.metrics["agent.cacheRead"] ?? 0) + (sample.metrics["harness.cacheRead"] ?? 0),
+        cacheCreation:
+          (sample.metrics["agent.cacheCreation"] ?? 0) +
+          (sample.metrics["harness.cacheCreation"] ?? 0),
+      }),
+    emptyUsage(),
+  );
+
+const EvaluationAccountingSchema = z
+  .object({
+    metricCalls: z.number().int().nonnegative().optional(),
+    judgeCalls: z.number().int().nonnegative().optional(),
+    unpricedModelCalls: z.number().int().nonnegative().optional(),
+    usage: TokenUsageSchema.optional(),
+    costUsd: z.number().nonnegative().optional(),
+  })
+  .strict();
+export type EvaluationAccounting = z.infer<typeof EvaluationAccountingSchema>;
+
+const recordEvaluationUnitBudget = async (
+  experiment: OpenedExperimentRun,
+  unitId: string,
+  evaluation: EvaluationAccounting,
+): Promise<boolean> => experiment.recordAccounting(unitId, evaluation);
+
+const rubricModelAccounting = (assertion: {
+  type: string;
+  detail?: unknown;
+}): { attempts: number; costUsd: number; unpricedModelCalls: number } => {
+  if (assertion.type !== "rubric") {
+    return { attempts: 0, costUsd: 0, unpricedModelCalls: 0 };
+  }
+  const parsed = RubricModelAccountingDetailSchema.safeParse(assertion.detail);
+  if (!parsed.success) {
+    return { attempts: 0, costUsd: 0, unpricedModelCalls: 0 };
+  }
+  return {
+    attempts: parsed.data.attempts,
+    costUsd: parsed.data.costUsd,
+    unpricedModelCalls: parsed.data.unpricedAttempts,
+  };
+};
+
+const assertionBudgetAccounting = (assertion: {
+  type: string;
+  detail?: unknown;
+  usage?: z.infer<typeof TokenUsageSchema>;
+}): EvaluationAccounting => {
+  const model = rubricModelAccounting(assertion);
+  let unpricedModelCalls = 0;
+  if (model.unpricedModelCalls > 0) {
+    unpricedModelCalls = model.unpricedModelCalls;
+  } else if (model.attempts === 0 && assertion.usage) {
+    unpricedModelCalls = 1;
+  }
+  return {
+    metricCalls: 0,
+    // Judge calls are journaled at provider-attempt start so a hard kill
+    // cannot erase them. Assertion completion records only observed spend.
+    judgeCalls: 0,
+    unpricedModelCalls,
+    usage: assertion.usage ?? emptyUsage(),
+    costUsd: model.costUsd,
+  };
+};
+
+const evaluationSamples = (summary: SuiteSummary): Record<string, number[]> => {
+  const samples: Record<string, number[]> = {
+    "quality.passRate": summary.cases.map((sample) => sample.passRate),
+  };
+  const add = (name: string, value: number): void => {
+    const values = samples[name] ?? [];
+    values.push(value);
+    samples[name] = values;
+  };
+  for (const sample of summary.cases) {
+    for (const [name, value] of Object.entries(sample.metrics)) {
+      add(name, value.mean);
+    }
+  }
+  return samples;
+};
+
+const AgentEvaluationUnitSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    unitId: z.string(),
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    repetitions: z.number().int().positive(),
+    caseId: z.string(),
+    repetition: z.number().int().positive(),
+    evaluation: CandidateEvaluationSchema,
+    budgetAccounting: EvaluationAccountingSchema,
+  })
+  .strict();
+type AgentEvaluationUnit = z.infer<typeof AgentEvaluationUnitSchema>;
+
+const agentUnitIdentity = (input: {
+  experimentId: string;
+  candidateId: string;
+  candidateHash: string;
+  profileId: string;
+  split: ExperimentSplit;
+  repetitions: number;
+  caseId: string;
+  casePath: string;
+  repetition: number;
+}): string => `agent-${valueHash(input).slice(0, 32)}`;
+
+const agentUnitPath = (experiment: OpenedExperimentRun, unitId: string): string =>
+  join(experiment.path, "evaluation-units", `${unitId}.json`);
+
+const modelAccountingFromMetadata = (
+  metadata: NonNullable<SuiteCaseResult["result"]>["metadata"] | null,
+): { costUsd: number; unpricedModelCalls: number } => {
+  const accounting = metadata?.modelAccounting;
+  if (accounting) {
+    return {
+      costUsd: accounting.reportedCostUsd,
+      unpricedModelCalls: accounting.unpricedModelCalls,
+    };
+  }
+  if (!metadata) return { costUsd: 0, unpricedModelCalls: 0 };
+  const costUsd = metadata.cost?.usd;
+  return {
+    costUsd: costUsd ?? 0,
+    // Conservative compatibility for an older persisted run: a completed
+    // model-driver session without a reported price is explicitly unpriced.
+    unpricedModelCalls: costUsd === undefined ? 1 : 0,
+  };
+};
+
+const agentMetadataBudgetAccounting = (
+  metadata: NonNullable<SuiteCaseResult["result"]>["metadata"],
+): EvaluationAccounting => ({
+  metricCalls: 0,
+  judgeCalls: 0,
+  // Answerer calls settle independently at provider-call boundaries. This
+  // checkpoint covers only the subject agent, avoiding delayed limit checks
+  // and double-counting persona spend.
+  usage: metadata.usage.agent,
+  costUsd: metadata.cost?.usd ?? 0,
+  unpricedModelCalls:
+    (metadata.modelAccounting?.agentCalls ?? 1) > 0 && metadata.cost?.usd === undefined ? 1 : 0,
+});
+
+const agentEvaluationFromSuite = (input: {
+  candidate: Candidate;
+  profileId: string;
+  split: ExperimentSplit;
+  summary: SuiteSummary;
+  results: readonly SuiteCaseResult[];
+  suiteDir: string;
+}): CandidateEvaluation => {
+  const { candidate, profileId, split, summary, results, suiteDir } = input;
+  const modelAccounting = results.map((result) =>
+    modelAccountingFromMetadata(result.result?.metadata ?? null),
+  );
+  return {
+    profileId,
+    split,
+    suiteDir,
+    passRate: summary.totals.passRate,
+    casePassRates: Object.fromEntries(
+      summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
+    ),
+    metrics: evaluationMetrics(summary, candidate),
+    samples: evaluationSamples(summary),
+    metricCalls: summary.samples.length,
+    judgeCalls: results.reduce(
+      (total, result) =>
+        total +
+        (result.result?.grading.assertions.reduce(
+          (assertionTotal, assertion) => assertionTotal + rubricModelAccounting(assertion).attempts,
+          0,
+        ) ?? 0),
+      0,
+    ),
+    unpricedModelCalls:
+      modelAccounting.reduce((total, entry) => total + entry.unpricedModelCalls, 0) +
+      results.reduce(
+        (total, result) =>
+          total +
+          (result.result?.grading.assertions.reduce(
+            (assertionTotal, assertion) =>
+              assertionTotal + rubricModelAccounting(assertion).unpricedModelCalls,
+            0,
+          ) ?? 0),
+        0,
+      ),
+    usage: summaryUsage(summary),
+    costUsd:
+      modelAccounting.reduce((total, entry) => total + entry.costUsd, 0) +
+      results.reduce(
+        (total, result) =>
+          total +
+          (result.result?.grading.assertions.reduce(
+            (assertionTotal, assertion) =>
+              assertionTotal + rubricModelAccounting(assertion).costUsd,
+            0,
+          ) ?? 0),
+        0,
+      ),
+    failedCases: failedCases(results),
+  };
+};
+
+const aggregateAgentUnits = (
+  candidate: Candidate,
+  profileId: string,
+  split: ExperimentSplit,
+  repetitions: number,
+  units: readonly AgentEvaluationUnit[],
+): ExperimentEvaluation => {
+  const samplesByCase = new Map<string, Map<string, number[]>>();
+  const metricValues = new Map<string, number[]>();
+  const caseValues = new Map<string, number[]>();
+  for (const { caseId, evaluation } of units) {
+    for (const [name, values] of Object.entries(evaluation.samples ?? {})) {
+      const byCase = samplesByCase.get(name) ?? new Map<string, number[]>();
+      byCase.set(caseId, [...(byCase.get(caseId) ?? []), ...values]);
+      samplesByCase.set(name, byCase);
+    }
+    for (const [name, value] of Object.entries(evaluation.metrics)) {
+      const values = metricValues.get(name) ?? [];
+      values.push(value);
+      metricValues.set(name, values);
+    }
+    for (const [evaluatedCaseId, value] of Object.entries(evaluation.casePassRates)) {
+      const values = caseValues.get(evaluatedCaseId) ?? [];
+      values.push(value);
+      caseValues.set(evaluatedCaseId, values);
+    }
+  }
+  const samples = Object.fromEntries(
+    [...samplesByCase].map(([name, byCase]) => [
+      name,
+      [...byCase.entries()]
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([, values]) => average(values)),
+    ]),
+  );
+  return {
+    candidateId: candidate.id,
+    profileId,
+    split,
+    repetitions,
+    passRate: average(samples["quality.passRate"] ?? units.map((unit) => unit.evaluation.passRate)),
+    casePassRates: Object.fromEntries(
+      [...caseValues].map(([caseId, values]) => [caseId, average(values)]),
+    ),
+    metrics: Object.fromEntries(
+      [...metricValues].map(([name, values]) => [
+        name,
+        name === "subject.bytes" ? candidate.sourceBytes : average(values),
+      ]),
+    ),
+    samples,
+    metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
+    judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
+    unpricedModelCalls: units.reduce(
+      (total, unit) => total + (unit.evaluation.unpricedModelCalls ?? 0),
+      0,
+    ),
+    usage: units.reduce(
+      (usage, unit) => addUsage(usage, unit.evaluation.usage ?? emptyUsage()),
+      emptyUsage(),
+    ),
+    costUsd: units.reduce((total, unit) => total + (unit.evaluation.costUsd ?? 0), 0),
+    failedCases: units.flatMap((unit) => unit.evaluation.failedCases),
+  };
+};
+
+export const invokeWithVerifiedExecutionIdentity = async <T>(
+  verifyExecutionIdentity: () => Promise<void>,
+  invocation: () => Promise<T>,
+): Promise<T> => {
+  await verifyExecutionIdentity();
+  try {
+    return await invocation();
+  } finally {
+    await verifyExecutionIdentity();
+  }
+};
+
+const verifiedCachedEvaluation = (
+  cached: ExperimentEvaluation,
+  recomputed: ExperimentEvaluation,
+): ExperimentEvaluation => {
+  if (valueHash(cached) !== valueHash(recomputed)) {
+    throw new Error("cached aggregate evaluation does not match its verified evidence");
+  }
+  return cached;
+};
+
+export const evaluateAgentCandidate = async (options: {
+  execution: CampaignExecutionOptions;
+  experiment: OpenedExperimentRun;
+  candidate: Candidate;
+  allCases: Awaited<ReturnType<typeof discoverCases>>["cases"];
+  profileId: string;
+  split: ExperimentSplit;
+  selector: ExperimentSelector;
+  repetitions: number;
+  verifyExecutionIdentity: () => Promise<void>;
+}): Promise<ExperimentEvaluation> => {
+  const { execution, experiment, candidate, profileId, split, repetitions } = options;
+  const { verifyExecutionIdentity } = options;
+  const cached = experiment.manifest.evaluations.find(
+    (evaluation) =>
+      evaluation.candidateId === candidate.id &&
+      evaluation.profileId === profileId &&
+      evaluation.split === split &&
+      evaluation.repetitions === repetitions,
+  );
+  const evaluationHash = experiment.manifest.evaluationHash;
+  if (!evaluationHash) throw new Error("agent experiment has no evaluation identity");
+  const selectedCases = selectCases(options.allCases, options.selector);
+  if (selectedCases.length === 0) {
+    throw new Error(
+      `experiment '${execution.loaded.campaign.id}' ${split} selector matched no cases`,
+    );
+  }
+  const profile = profileOrThrow(execution.config, profileId);
+  const harness = profile.harness ?? execution.config.harness;
+  const defaultDriver = profile.driver ?? execution.config.defaultDriver;
+  const registry = execution.registry.withAssertion(createRubricAssertion(harness));
+  const orchestrator = new Orchestrator({
+    registry,
+    defaultAnswerer: profile.answerer ?? execution.config.defaultAnswerer,
+    ...(defaultDriver ? { defaultDriver } : {}),
+    ...(harness ? { harness } : {}),
+    lifecycleFixtures: execution.config.lifecycleFixtures,
+  });
+  const runsRoot = execution.runsRoot ?? execution.config.runsRoot;
+  const units: AgentEvaluationUnit[] = [];
+  for (const evalCase of selectedCases) {
+    for (let repetition = 1; repetition <= repetitions; repetition++) {
+      const unitId = agentUnitIdentity({
+        experimentId: experiment.manifest.experimentId,
+        candidateId: candidate.id,
+        candidateHash: candidate.candidateHash,
+        profileId,
+        split,
+        repetitions,
+        caseId: evalCase.id,
+        casePath: evalCase.path,
+        repetition,
+      });
+      const path = agentUnitPath(experiment, unitId);
+      let unit = await readContentBoundJson(path, AgentEvaluationUnitSchema, evaluationHash);
+      if (unit) {
+        if (
+          unit.unitId !== unitId ||
+          unit.candidateId !== candidate.id ||
+          unit.profileId !== profileId ||
+          unit.split !== split ||
+          unit.repetitions !== repetitions ||
+          unit.caseId !== evalCase.id ||
+          unit.repetition !== repetition
+        ) {
+          throw new Error(`agent evaluation unit failed its identity check: ${path}`);
+        }
+      } else {
+        let unitEvaluation: CandidateEvaluation;
+        const budgetAccounting: EvaluationAccounting = {};
+        const unitRoot = await materializeCandidate(
+          execution.loaded.subjectRoot,
+          execution.loaded.campaign.subject,
+          candidate,
+        );
+        try {
+          ensureBudget(execution.loaded.campaign, experiment, { metricCalls: 1 });
+          const authored = await loadEvalCase(evalCase.path);
+          const rubricAssertions = (authored.assertions ?? []).filter(
+            (assertion) => assertion.type === "rubric",
+          ).length;
+          ensureBudget(execution.loaded.campaign, experiment, {
+            judgeCalls: rubricAssertions * 2,
+          });
+          const item: SuiteItem = {
+            path: evalCase.path,
+            id: repetitions === 1 ? evalCase.id : `${evalCase.id} #${repetition}`,
+            caseId: evalCase.id,
+          };
+          const assertionAttemptId = randomUUID();
+          const metricAttemptId = randomUUID();
+          let pendingModelSettlement: { id: string; accounting: EvaluationAccounting } | undefined;
+          const suite = await invokeWithVerifiedExecutionIdentity(
+            verifyExecutionIdentity,
+            async () => {
+              await recordEvaluationUnitBudget(
+                experiment,
+                `agent-metric-attempt-${metricAttemptId}`,
+                {
+                  metricCalls: 1,
+                },
+              );
+              await experiment.write();
+              ensureBudget(execution.loaded.campaign, experiment);
+              return orchestrator.runSuite([item], {
+                runsRoot,
+                fixturesRoot: execution.config.fixturesRoot,
+                subjectRoot: unitRoot,
+                subjectHash: candidate.candidateHash,
+                profileId,
+                experimentId: experiment.manifest.experimentId,
+                experimentUnitId: unitId,
+                experimentAttemptId: metricAttemptId,
+                variantId: candidate.id,
+                candidateId: candidate.id,
+                recordPreGradingAccounting: async (metadata) => {
+                  await verifyExecutionIdentity();
+                  pendingModelSettlement = {
+                    id: `agent-base-attempt-${metricAttemptId}`,
+                    accounting: agentMetadataBudgetAccounting(metadata),
+                  };
+                },
+                recordModelAttemptStart: async (kind) => {
+                  await verifyExecutionIdentity();
+                  ensureBudget(execution.loaded.campaign, experiment, {
+                    judgeCalls: kind === "judge" ? 1 : 0,
+                  });
+                  const attemptId = await experiment.startModelAttempt(kind);
+                  return attemptId;
+                },
+                recordModelAttemptComplete: async (attemptId, accounting) => {
+                  await verifyExecutionIdentity();
+                  await experiment.settleModelAttempts(
+                    `answerer-call:${attemptId}`,
+                    {
+                      usage: accounting.usage,
+                      costUsd: accounting.costUsd ?? 0,
+                      unpricedModelCalls: accounting.unpricedModelCalls,
+                    },
+                    [attemptId],
+                  );
+                  ensureBudget(execution.loaded.campaign, experiment);
+                },
+                recordModelAttemptsComplete: async (attemptIds) => {
+                  await verifyExecutionIdentity();
+                  const settlement = pendingModelSettlement;
+                  pendingModelSettlement = undefined;
+                  if (settlement) {
+                    await experiment.settleModelAttempts(
+                      settlement.id,
+                      settlement.accounting,
+                      attemptIds,
+                    );
+                  } else {
+                    await experiment.completeModelAttempts(attemptIds);
+                  }
+                  ensureBudget(execution.loaded.campaign, experiment);
+                },
+                recordGradingAssertion: async (assertion, index) => {
+                  await verifyExecutionIdentity();
+                  pendingModelSettlement = {
+                    id: `agent-judge-${unitId}-${assertionAttemptId}-${index}`,
+                    accounting: assertionBudgetAccounting(assertion),
+                  };
+                },
+                abortSignal: execution.abortSignal,
+                concurrency: 1,
+              });
+            },
+          );
+          const summary = await loadSuiteSummary(suite.suiteDir);
+          if (!summary) throw new Error(`suite summary was not written at ${suite.suiteDir}`);
+          const gradingError = gradingFailureMessage(suite.results);
+          if (
+            suite.aborted ||
+            summary.totals.pending !== 0 ||
+            summary.samples.length !== 1 ||
+            gradingError
+          ) {
+            await recordEvaluationUnitBudget(
+              experiment,
+              `agent-attempt-${suite.suiteDir}`,
+              budgetAccounting,
+            );
+            await experiment.write();
+            throwIfAborted(execution.abortSignal);
+            if (gradingError) {
+              throw new Error(
+                `agent evaluation unit '${unitId}' grading did not complete: ${gradingError}`,
+              );
+            }
+            throw new Error(`agent evaluation unit '${unitId}' did not complete`);
+          }
+          unitEvaluation = agentEvaluationFromSuite({
+            candidate,
+            profileId,
+            split,
+            summary,
+            results: suite.results,
+            suiteDir: suite.suiteDir,
+          });
+        } finally {
+          await rm(unitRoot, { recursive: true, force: true });
+        }
+        unit = {
+          schemaVersion: 1,
+          unitId,
+          candidateId: candidate.id,
+          profileId,
+          split,
+          repetitions,
+          caseId: evalCase.id,
+          repetition,
+          evaluation: unitEvaluation,
+          budgetAccounting,
+        };
+        await verifyExecutionIdentity();
+        await writeContentBoundJson(path, unit, evaluationHash, verifyExecutionIdentity);
+      }
+      const newlyRecorded = await recordEvaluationUnitBudget(
+        experiment,
+        unitId,
+        unit.budgetAccounting,
+      );
+      if (newlyRecorded) {
+        await execution.onEvaluationUnit?.({
+          unitId,
+          candidateId: candidate.id,
+          profileId,
+          split,
+        });
+        // Unit completion is the accounting checkpoint. Individual assertion
+        // entries remain durable in the append-only journal without making
+        // every judge call rewrite an ever-growing manifest.
+        await experiment.write();
+      }
+      ensureBudget(execution.loaded.campaign, experiment);
+      units.push(unit);
+      throwIfAborted(execution.abortSignal);
+    }
+  }
+  if (units.length !== selectedCases.length * repetitions) {
+    throw new Error(`agent evaluation did not produce every requested case repetition`);
+  }
+  const aggregate = aggregateAgentUnits(candidate, profileId, split, repetitions, units);
+  if (cached) return verifiedCachedEvaluation(cached, aggregate);
+  experiment.manifest.evaluations.push(aggregate);
+  await experiment.write();
+  return aggregate;
+};
+
+export const evaluateOfflineGraderCandidate = async (options: {
+  execution: CampaignExecutionOptions;
+  experiment: OpenedExperimentRun;
+  candidate: Candidate;
+  materializedRoot: string;
+  profileId: string;
+  split: ExperimentSplit;
+  selector: ExperimentSelector;
+  repetitions: number;
+  verifyExecutionIdentity: () => Promise<void>;
+}): Promise<ExperimentEvaluation> => {
+  const cached = options.experiment.manifest.evaluations.find(
+    (cachedEvaluation) =>
+      cachedEvaluation.candidateId === options.candidate.id &&
+      cachedEvaluation.profileId === options.profileId &&
+      cachedEvaluation.split === options.split &&
+      cachedEvaluation.repetitions === options.repetitions,
+  );
+  let pendingModelSettlement: { id: string; accounting: EvaluationAccounting } | undefined;
+  const graded = await invokeWithVerifiedExecutionIdentity(options.verifyExecutionIdentity, () =>
+    evaluateGraderCandidate({
+      ...options,
+      preflightUnit: (maximumJudgeCalls) =>
+        ensureBudget(options.execution.loaded.campaign, options.experiment, {
+          metricCalls: 1,
+          judgeCalls: maximumJudgeCalls,
+        }),
+      recordUnit: async (unitId, accounting) => {
+        await options.verifyExecutionIdentity();
+        const newlyRecorded = await recordEvaluationUnitBudget(
+          options.experiment,
+          unitId,
+          accounting,
+        );
+        if (newlyRecorded) {
+          await options.execution.onEvaluationUnit?.({
+            unitId,
+            candidateId: options.candidate.id,
+            profileId: options.profileId,
+            split: options.split,
+          });
+          await options.experiment.write();
+        }
+        ensureBudget(options.execution.loaded.campaign, options.experiment);
+      },
+      recordAttempt: async (attemptId, accounting) => {
+        await options.verifyExecutionIdentity();
+        await recordEvaluationUnitBudget(options.experiment, attemptId, accounting);
+        await options.experiment.write();
+        ensureBudget(options.execution.loaded.campaign, options.experiment);
+      },
+      recordAssertion: async (assertionId, accounting) => {
+        await options.verifyExecutionIdentity();
+        pendingModelSettlement = { id: assertionId, accounting };
+      },
+      recordModelAttemptStart: async (kind) => {
+        await options.verifyExecutionIdentity();
+        ensureBudget(options.execution.loaded.campaign, options.experiment, {
+          judgeCalls: kind === "judge" ? 1 : 0,
+        });
+        const attemptId = await options.experiment.startModelAttempt(kind);
+        return attemptId;
+      },
+      completeModelAttempts: async (attemptIds) => {
+        await options.verifyExecutionIdentity();
+        const settlement = pendingModelSettlement;
+        pendingModelSettlement = undefined;
+        if (settlement) {
+          await options.experiment.settleModelAttempts(
+            settlement.id,
+            settlement.accounting,
+            attemptIds,
+          );
+        } else {
+          await options.experiment.completeModelAttempts(attemptIds);
+        }
+        ensureBudget(options.execution.loaded.campaign, options.experiment);
+      },
+    }),
+  );
+  const evaluation: ExperimentEvaluation = {
+    candidateId: options.candidate.id,
+    repetitions: options.repetitions,
+    ...graded,
+  };
+  if (cached) return verifiedCachedEvaluation(cached, evaluation);
+  options.experiment.manifest.evaluations.push(evaluation);
+  await options.experiment.write();
+  return evaluation;
+};