@prismatic-io/lux 0.0.1

package/src/orchestrator/grader-experiment.ts

@@ -0,0 +1,856 @@
+import { randomUUID } from "node:crypto";
+import { join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import {
+  createRubricAssertion,
+  RubricModelAccountingDetailSchema,
+} from "../assertions/rubric/index.js";
+import {
+  type Candidate,
+  type CandidateEvaluation,
+  type ExperimentSelector,
+  emptyUsage,
+  type GraderAnnotation,
+  type GradingReport,
+  GradingReportSchema,
+  sumUsage,
+  TokenUsageSchema,
+  totalTokens,
+  valueHash,
+} from "../core/index.js";
+import { selectGraderAnnotations } from "./annotation-loader.js";
+import { buildRegistryFromConfig } from "./config.js";
+import {
+  readContentBoundJson,
+  retireContentBoundJson,
+  writeContentBoundJson,
+} from "./content-bound-json.js";
+import type { CampaignExecutionOptions } from "./experiment.js";
+import type { OpenedExperimentRun } from "./experiment-runs.js";
+import { gradeRun } from "./grading.js";
+import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
+import { loadRubricDefinition } from "./rubric-loader.js";
+
+const GraderJudgmentSchema = z.object({
+  key: z.string(),
+  kind: z.enum(["case-pass", "case-score", "assertion-pass", "assertion-score"]),
+  expected: z.union([z.boolean(), z.number()]),
+  actual: z.union([z.boolean(), z.number()]).nullable(),
+  matched: z.boolean(),
+  absoluteError: z.exactOptional(z.number()),
+});
+export type GraderJudgment = z.infer<typeof GraderJudgmentSchema>;
+
+const GraderEvidenceItemSchema = z.object({
+  annotationId: z.string(),
+  repetition: z.number().int().positive(),
+  runDir: z.string(),
+  rubricPath: z.string(),
+  feedback: z.exactOptional(z.string()),
+  judgments: z.array(GraderJudgmentSchema),
+  report: GradingReportSchema,
+  durationMs: z.number().nonnegative(),
+  usage: TokenUsageSchema,
+  costUsd: z.number().nonnegative(),
+  unpricedModelCalls: z.number().int().nonnegative(),
+});
+type GraderEvidenceItem = z.infer<typeof GraderEvidenceItemSchema>;
+
+const GraderEvaluationUnitSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    unitId: z.string(),
+    candidateId: z.string(),
+    profileId: z.string(),
+    split: z.enum(["train", "validation", "test"]),
+    repetitions: z.number().int().positive(),
+    annotationsHash: z.string(),
+    annotationId: z.string(),
+    repetition: z.number().int().positive(),
+    item: GraderEvidenceItemSchema,
+  })
+  .strict();
+type GraderEvaluationUnit = z.infer<typeof GraderEvaluationUnitSchema>;
+
+export const GraderEvaluationEvidenceSchema = z.object({
+  schemaVersion: z.literal(1),
+  candidateId: z.string(),
+  profileId: z.string(),
+  split: z.enum(["train", "validation", "test"]),
+  repetitions: z.number().int().positive(),
+  annotationsHash: z.string(),
+  generatedAt: z.string(),
+  metrics: z.record(z.string(), z.number()),
+  samples: z.record(z.string(), z.array(z.number())),
+  items: z.array(GraderEvidenceItemSchema),
+});
+export type GraderEvaluationEvidence = z.infer<typeof GraderEvaluationEvidenceSchema>;
+
+const assertionOperationalError = (
+  entry: GradingReport["assertions"][number] | undefined,
+): boolean => {
+  if (!entry) return false;
+  if (entry.errorMessage) return true;
+  const detail = RubricModelAccountingDetailSchema.safeParse(entry.detail);
+  return detail.success && detail.data.graderError !== undefined;
+};
+
+const assertionModelAccounting = (
+  assertion: GradingReport["assertions"][number],
+): { attempts: number; costUsd: number; unpricedModelCalls: number } => {
+  const detail = RubricModelAccountingDetailSchema.safeParse(assertion.detail);
+  if (!detail.success) {
+    return { attempts: 0, costUsd: 0, unpricedModelCalls: 0 };
+  }
+  return {
+    attempts: detail.data.attempts,
+    costUsd: detail.data.costUsd,
+    unpricedModelCalls: detail.data.unpricedAttempts,
+  };
+};
+
+export const compareGraderAnnotation = (
+  annotation: GraderAnnotation,
+  report: GradingReport,
+  scoreTolerance: number,
+): GraderJudgment[] => {
+  const judgments: GraderJudgment[] = [];
+  const reportFailed = report.assertions.some(assertionOperationalError);
+  if (annotation.expected.casePassed !== undefined) {
+    const actual = reportFailed ? null : report.casePassed;
+    judgments.push({
+      key: "case",
+      kind: "case-pass",
+      expected: annotation.expected.casePassed,
+      actual,
+      matched: actual === annotation.expected.casePassed,
+    });
+  }
+  if (annotation.expected.caseScore !== undefined) {
+    const actual = reportFailed ? null : report.caseScore;
+    const absoluteError = actual === null ? 1 : Math.abs(actual - annotation.expected.caseScore);
+    judgments.push({
+      key: "case",
+      kind: "case-score",
+      expected: annotation.expected.caseScore,
+      actual,
+      matched: actual !== null && absoluteError <= scoreTolerance,
+      absoluteError,
+    });
+  }
+  const entries = new Map(
+    report.assertions.flatMap((entry) => (entry.id ? [[entry.id, entry] as const] : [])),
+  );
+  for (const [assertionId, expected] of Object.entries(annotation.expected.assertions)) {
+    const entry = entries.get(assertionId);
+    const actual = assertionOperationalError(entry) ? undefined : entry;
+    judgments.push({
+      key: assertionId,
+      kind: "assertion-pass",
+      expected: expected.passed,
+      actual: actual?.passed ?? null,
+      matched: actual?.passed === expected.passed,
+    });
+    if (expected.score !== undefined) {
+      const absoluteError = actual ? Math.abs(actual.score - expected.score) : 1;
+      judgments.push({
+        key: assertionId,
+        kind: "assertion-score",
+        expected: expected.score,
+        actual: actual?.score ?? null,
+        matched: actual !== undefined && absoluteError <= scoreTolerance,
+        absoluteError,
+      });
+    }
+  }
+  return judgments;
+};
+
+const ratio = (numerator: number, denominator: number, empty: number): number =>
+  denominator === 0 ? empty : numerator / denominator;
+
+const addSample = (samples: Record<string, number[]>, name: string, value: number): void => {
+  if (!Number.isFinite(value)) return;
+  const values = samples[name] ?? [];
+  values.push(value);
+  samples[name] = values;
+};
+
+const mean = (values: readonly number[], empty = 0): number =>
+  values.length === 0 ? empty : values.reduce((sum, value) => sum + value, 0) / values.length;
+
+type GraderEvidenceCounts = {
+  truePositive: number;
+  falsePositive: number;
+  falseNegative: number;
+  positiveLabels: number;
+  negativeLabels: number;
+  abstentions: number;
+  passJudgments: number;
+  scoreJudgments: number;
+};
+
+const collectGraderEvidence = (
+  items: readonly GraderEvidenceItem[],
+): {
+  byAnnotation: Map<string, z.infer<typeof GraderJudgmentSchema>[]>;
+  itemsByAnnotation: Map<string, GraderEvidenceItem[]>;
+  counts: GraderEvidenceCounts;
+} => {
+  const byAnnotation = new Map<string, z.infer<typeof GraderJudgmentSchema>[]>();
+  const itemsByAnnotation = new Map<string, GraderEvidenceItem[]>();
+  const counts: GraderEvidenceCounts = {
+    truePositive: 0,
+    falsePositive: 0,
+    falseNegative: 0,
+    positiveLabels: 0,
+    negativeLabels: 0,
+    abstentions: 0,
+    passJudgments: 0,
+    scoreJudgments: 0,
+  };
+  for (const item of items) {
+    const annotationJudgments = byAnnotation.get(item.annotationId) ?? [];
+    annotationJudgments.push(...item.judgments);
+    byAnnotation.set(item.annotationId, annotationJudgments);
+    const annotationItems = itemsByAnnotation.get(item.annotationId) ?? [];
+    annotationItems.push(item);
+    itemsByAnnotation.set(item.annotationId, annotationItems);
+    for (const judgment of item.judgments) {
+      if (judgment.absoluteError !== undefined) counts.scoreJudgments++;
+      if (!judgment.kind.endsWith("-pass") || typeof judgment.expected !== "boolean") continue;
+      counts.passJudgments++;
+      if (judgment.expected) counts.positiveLabels++;
+      else counts.negativeLabels++;
+      if (judgment.actual === null) counts.abstentions++;
+      else if (judgment.expected && judgment.actual) counts.truePositive++;
+      else if (!judgment.expected && judgment.actual) counts.falsePositive++;
+      else if (judgment.expected && !judgment.actual) counts.falseNegative++;
+    }
+  }
+  return { byAnnotation, itemsByAnnotation, counts };
+};
+
+export const summarizeGraderEvidence = (
+  items: readonly z.infer<typeof GraderEvidenceItemSchema>[],
+  candidate: Candidate,
+): { metrics: Record<string, number>; samples: Record<string, number[]> } => {
+  const samples: Record<string, number[]> = {};
+  const { byAnnotation, itemsByAnnotation, counts } = collectGraderEvidence(items);
+  let caseJudgmentCount = 0;
+  let assertionJudgmentCount = 0;
+  for (const [annotationId, judgments] of byAnnotation) {
+    const annotationItems = itemsByAnnotation.get(annotationId) ?? [];
+    addSample(samples, "phase.gradingMs", mean(annotationItems.map((item) => item.durationMs)));
+    addSample(samples, "harness.input", mean(annotationItems.map((item) => item.usage.input)));
+    addSample(samples, "harness.output", mean(annotationItems.map((item) => item.usage.output)));
+    addSample(
+      samples,
+      "harness.cacheRead",
+      mean(annotationItems.map((item) => item.usage.cacheRead)),
+    );
+    addSample(
+      samples,
+      "harness.cacheCreation",
+      mean(annotationItems.map((item) => item.usage.cacheCreation)),
+    );
+    addSample(
+      samples,
+      "harness.total",
+      mean(annotationItems.map((item) => totalTokens(item.usage))),
+    );
+    const allAgreement = mean(judgments.map((judgment) => (judgment.matched ? 1 : 0)));
+    addSample(samples, "quality.passRate", allAgreement);
+    addSample(samples, "grader.agreement", allAgreement);
+    const caseJudgments = judgments.filter((judgment) => judgment.kind.startsWith("case"));
+    if (caseJudgments.length > 0) {
+      caseJudgmentCount += caseJudgments.length;
+      addSample(
+        samples,
+        "grader.caseAgreement",
+        mean(caseJudgments.map((judgment) => (judgment.matched ? 1 : 0))),
+      );
+    }
+    const assertionJudgments = judgments.filter((judgment) =>
+      judgment.kind.startsWith("assertion"),
+    );
+    if (assertionJudgments.length > 0) {
+      assertionJudgmentCount += assertionJudgments.length;
+      addSample(
+        samples,
+        "grader.assertionAgreement",
+        mean(assertionJudgments.map((judgment) => (judgment.matched ? 1 : 0))),
+      );
+    }
+    const passes = judgments.filter(
+      (judgment) => judgment.kind.endsWith("-pass") && typeof judgment.expected === "boolean",
+    );
+    const scoreErrors = judgments.flatMap((judgment) =>
+      judgment.absoluteError === undefined ? [] : [judgment.absoluteError],
+    );
+    if (scoreErrors.length > 0) {
+      addSample(samples, "grader.scoreAbsoluteError", mean(scoreErrors));
+      addSample(samples, "grader.scoreMae", mean(scoreErrors));
+    }
+    const positive = passes.filter((judgment) => judgment.expected === true);
+    const negative = passes.filter((judgment) => judgment.expected === false);
+    const coveredPositive = positive.filter((judgment) => judgment.actual !== null);
+    const coveredNegative = negative.filter((judgment) => judgment.actual !== null);
+    if (coveredPositive.length > 0) {
+      addSample(
+        samples,
+        "grader.macroFalseNegativeRate",
+        coveredPositive.filter((judgment) => judgment.actual === false).length /
+          coveredPositive.length,
+      );
+    }
+    if (coveredNegative.length > 0) {
+      addSample(
+        samples,
+        "grader.macroFalsePositiveRate",
+        coveredNegative.filter((judgment) => judgment.actual === true).length /
+          coveredNegative.length,
+      );
+    }
+    if (passes.length > 0) {
+      addSample(
+        samples,
+        "grader.coverage",
+        passes.filter((judgment) => judgment.actual !== null).length / passes.length,
+      );
+      addSample(
+        samples,
+        "grader.abstentionRate",
+        passes.filter((judgment) => judgment.actual === null).length / passes.length,
+      );
+    }
+  }
+  const precision = ratio(
+    counts.truePositive,
+    counts.truePositive + counts.falsePositive,
+    Number.NaN,
+  );
+  const recall = ratio(counts.truePositive, counts.truePositive + counts.falseNegative, Number.NaN);
+  const metrics: Record<string, number> = {
+    "quality.passRate": mean(samples["quality.passRate"] ?? []),
+    "grader.agreement": mean(samples["grader.agreement"] ?? []),
+    "grader.labels": byAnnotation.size,
+    "grader.judgments": items.reduce((total, item) => total + item.judgments.length, 0),
+    "grader.passJudgments": counts.passJudgments,
+    "grader.caseJudgments": caseJudgmentCount,
+    "grader.assertionJudgments": assertionJudgmentCount,
+    "grader.scoreJudgments": counts.scoreJudgments,
+    "grader.positiveLabels": counts.positiveLabels,
+    "grader.negativeLabels": counts.negativeLabels,
+    "grader.abstentions": counts.abstentions,
+    "subject.bytes": candidate.sourceBytes,
+  };
+  const addMetric = (name: string, value: number | undefined): void => {
+    if (value !== undefined && Number.isFinite(value)) metrics[name] = value;
+  };
+  addMetric(
+    "grader.caseAgreement",
+    samples["grader.caseAgreement"] ? mean(samples["grader.caseAgreement"]) : undefined,
+  );
+  addMetric(
+    "grader.assertionAgreement",
+    samples["grader.assertionAgreement"] ? mean(samples["grader.assertionAgreement"]) : undefined,
+  );
+  addMetric("grader.microPrecision", precision);
+  addMetric("grader.microRecall", recall);
+  let microF1: number | undefined;
+  if (Number.isFinite(precision) && Number.isFinite(recall)) {
+    microF1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+  }
+  addMetric("grader.microF1", microF1);
+  addMetric(
+    "grader.macroFalsePositiveRate",
+    samples["grader.macroFalsePositiveRate"]
+      ? mean(samples["grader.macroFalsePositiveRate"])
+      : undefined,
+  );
+  addMetric(
+    "grader.macroFalseNegativeRate",
+    samples["grader.macroFalseNegativeRate"]
+      ? mean(samples["grader.macroFalseNegativeRate"])
+      : undefined,
+  );
+  addMetric(
+    "grader.coverage",
+    samples["grader.coverage"] ? mean(samples["grader.coverage"]) : undefined,
+  );
+  addMetric(
+    "grader.abstentionRate",
+    samples["grader.abstentionRate"] ? mean(samples["grader.abstentionRate"]) : undefined,
+  );
+  addMetric(
+    "grader.scoreMae",
+    samples["grader.scoreMae"] ? mean(samples["grader.scoreMae"]) : undefined,
+  );
+  for (const [name, values] of Object.entries(samples)) {
+    if (!(name in metrics)) metrics[name] = mean(values);
+  }
+  return { metrics, samples };
+};
+
+const graderEvidencePath = (
+  experiment: OpenedExperimentRun,
+  identityInput: Pick<
+    GraderEvaluationEvidence,
+    "candidateId" | "profileId" | "split" | "repetitions" | "annotationsHash"
+  >,
+): string => {
+  const directory = join(experiment.path, "grader-evaluations");
+  const identity = valueHash({
+    candidateId: identityInput.candidateId,
+    profileId: identityInput.profileId,
+    split: identityInput.split,
+    repetitions: identityInput.repetitions,
+    annotationsHash: identityInput.annotationsHash,
+  }).slice(0, 20);
+  return join(directory, `${identityInput.candidateId}-${identity}.json`);
+};
+
+const graderUnitId = (input: {
+  candidateId: string;
+  profileId: string;
+  split: "train" | "validation" | "test";
+  repetitions: number;
+  annotationsHash: string;
+  annotationId: string;
+  repetition: number;
+}): string => `grader-${valueHash(input).slice(0, 32)}`;
+
+const graderUnitPath = (experiment: OpenedExperimentRun, unitId: string): string =>
+  join(experiment.path, "evaluation-units", `${unitId}.json`);
+
+const itemAccounting = (): Pick<
+  CandidateEvaluation,
+  "metricCalls" | "judgeCalls" | "unpricedModelCalls" | "usage" | "costUsd"
+> => ({
+  metricCalls: 1,
+  judgeCalls: 0,
+  usage: emptyUsage(),
+  costUsd: 0,
+  unpricedModelCalls: 0,
+});
+
+const evaluationFromEvidence = (
+  experiment: OpenedExperimentRun,
+  evidencePath: string,
+  evidence: GraderEvaluationEvidence,
+): CandidateEvaluation & { evidencePath: string } => {
+  const byAnnotation = new Map<string, number[]>();
+  for (const item of evidence.items) {
+    const values = byAnnotation.get(item.annotationId) ?? [];
+    values.push(...item.judgments.map((judgment) => (judgment.matched ? 1 : 0)));
+    byAnnotation.set(item.annotationId, values);
+  }
+  const failedCases: CandidateEvaluation["failedCases"] = evidence.items.flatMap((item) => {
+    const mismatches = item.judgments.filter((judgment) => !judgment.matched);
+    if (mismatches.length === 0) return [];
+    const feedback = mismatches.map(
+      (judgment) =>
+        `${judgment.key} ${judgment.kind}: expected ${String(judgment.expected)}, got ${String(judgment.actual)}`,
+    );
+    if (item.feedback) feedback.unshift(`human feedback: ${item.feedback}`);
+    for (const judgment of mismatches) {
+      if (!judgment.kind.startsWith("assertion")) continue;
+      const comment = item.report.assertions.find((entry) => entry.id === judgment.key)?.comment;
+      if (comment) feedback.push(`grader comment for ${judgment.key}: ${comment}`);
+    }
+    return [
+      {
+        caseId: `${item.annotationId} #${item.repetition}`,
+        score: mean(item.judgments.map((judgment) => (judgment.matched ? 1 : 0))),
+        feedback,
+      },
+    ];
+  });
+  return {
+    profileId: evidence.profileId,
+    split: evidence.split,
+    evidencePath: relative(experiment.path, evidencePath),
+    passRate: evidence.metrics["quality.passRate"] ?? 0,
+    casePassRates: Object.fromEntries([...byAnnotation].map(([id, values]) => [id, mean(values)])),
+    metrics: evidence.metrics,
+    samples: evidence.samples,
+    metricCalls: evidence.items.length,
+    judgeCalls: evidence.items.reduce(
+      (total, item) =>
+        total +
+        item.report.assertions.reduce(
+          (assertionTotal, assertion) =>
+            assertionTotal + assertionModelAccounting(assertion).attempts,
+          0,
+        ),
+      0,
+    ),
+    usage: sumUsage(evidence.items.map((item) => item.usage)),
+    costUsd: evidence.items.reduce((total, item) => total + item.costUsd, 0),
+    unpricedModelCalls: evidence.items.reduce((total, item) => total + item.unpricedModelCalls, 0),
+    failedCases,
+  };
+};
+
+const candidateRubricPath = (materializedRoot: string, rubricPath: string): string => {
+  const root = resolve(materializedRoot);
+  const path = resolve(root, rubricPath);
+  if (path !== root && !path.startsWith(`${root}${sep}`)) {
+    throw new Error(`grader rubric path escapes candidate root: ${rubricPath}`);
+  }
+  return path;
+};
+
+const assertionIds = (rubric: Awaited<ReturnType<typeof loadRubricDefinition>>): string[] =>
+  rubric.assertions.map((assertion) => assertion.id ?? "").sort();
+
+const assertEvidenceIdentity = (
+  evidence: GraderEvaluationEvidence,
+  expected: {
+    candidateId: string;
+    profileId: string;
+    split: "train" | "validation" | "test";
+    repetitions: number;
+    annotationsHash: string;
+  },
+  path: string,
+): void => {
+  if (
+    evidence.candidateId !== expected.candidateId ||
+    evidence.profileId !== expected.profileId ||
+    evidence.split !== expected.split ||
+    evidence.repetitions !== expected.repetitions ||
+    evidence.annotationsHash !== expected.annotationsHash
+  ) {
+    throw new Error(`cached grader evidence failed its identity check: ${path}`);
+  }
+};
+
+const assertUnitIdentity = (
+  unit: GraderEvaluationUnit,
+  expected: Omit<GraderEvaluationUnit, "schemaVersion" | "item">,
+  path: string,
+): void => {
+  if (
+    unit.unitId !== expected.unitId ||
+    unit.candidateId !== expected.candidateId ||
+    unit.profileId !== expected.profileId ||
+    unit.split !== expected.split ||
+    unit.repetitions !== expected.repetitions ||
+    unit.annotationsHash !== expected.annotationsHash ||
+    unit.annotationId !== expected.annotationId ||
+    unit.repetition !== expected.repetition
+  ) {
+    throw new Error(`grader evaluation unit failed its identity check: ${path}`);
+  }
+};
+
+const throwIfExperimentAborted = (signal: AbortSignal | undefined): void => {
+  if (!signal?.aborted) return;
+  throw signal.reason instanceof Error ? signal.reason : new Error("experiment aborted");
+};
+
+const validateCandidateRubricIds = (
+  rubricPath: string,
+  baselineIds: readonly string[],
+  candidateIds: readonly string[],
+  allowedAdditions: ReadonlySet<string>,
+): void => {
+  const missingBaseline = baselineIds.filter((id) => !candidateIds.includes(id));
+  const unlabeledAdditions = candidateIds.filter(
+    (id) => !baselineIds.includes(id) && !allowedAdditions.has(id),
+  );
+  if (missingBaseline.length === 0 && unlabeledAdditions.length === 0) return;
+  throw new Error(
+    `rubric '${rubricPath}' must preserve baseline IDs and may add only explicitly allowed IDs` +
+      `${missingBaseline.length > 0 ? `; missing: ${missingBaseline.join(", ")}` : ""}` +
+      `${unlabeledAdditions.length > 0 ? `; unlabeled additions: ${unlabeledAdditions.join(", ")}` : ""}`,
+  );
+};
+
+export const evaluateGraderCandidate = async (options: {
+  execution: CampaignExecutionOptions;
+  experiment: OpenedExperimentRun;
+  candidate: Candidate;
+  materializedRoot: string;
+  profileId: string;
+  split: "train" | "validation" | "test";
+  selector: ExperimentSelector;
+  repetitions: number;
+  preflightUnit: (maximumJudgeCalls: number) => void;
+  recordUnit: (
+    unitId: string,
+    accounting: Pick<
+      CandidateEvaluation,
+      "metricCalls" | "judgeCalls" | "unpricedModelCalls" | "usage" | "costUsd"
+    >,
+  ) => Promise<void>;
+  recordAttempt: (
+    attemptId: string,
+    accounting: Pick<
+      CandidateEvaluation,
+      "metricCalls" | "judgeCalls" | "unpricedModelCalls" | "usage" | "costUsd"
+    >,
+  ) => Promise<void>;
+  recordAssertion: (
+    assertionId: string,
+    accounting: Pick<
+      CandidateEvaluation,
+      "metricCalls" | "judgeCalls" | "unpricedModelCalls" | "usage" | "costUsd"
+    >,
+  ) => Promise<void>;
+  recordModelAttemptStart: (kind: "judge") => string | Promise<string>;
+  completeModelAttempts: (attemptIds: readonly string[]) => void | Promise<void>;
+  verifyExecutionIdentity: () => Promise<void>;
+}): Promise<CandidateEvaluation & { evidencePath: string }> => {
+  const { execution, experiment, candidate, profileId, split, repetitions } = options;
+  const annotations = execution.loaded.graderAnnotations;
+  if (!annotations || execution.loaded.campaign.evaluation.kind !== "grader") {
+    throw new Error("grader experiment is missing its loaded annotation dataset");
+  }
+  const evaluationHash = experiment.manifest.evaluationHash;
+  if (!evaluationHash) throw new Error("grader experiment has no evaluation identity");
+  const selected = selectGraderAnnotations(annotations.annotations, options.selector);
+  if (selected.length === 0) {
+    throw new Error(
+      `experiment '${execution.loaded.campaign.id}' ${split} selector matched no labels`,
+    );
+  }
+  const profile = execution.config.profiles?.[profileId];
+  if (!profile) throw new Error(`experiment profile '${profileId}' is not configured`);
+  const expectedEvidencePath = graderEvidencePath(experiment, {
+    candidateId: candidate.id,
+    profileId,
+    split,
+    repetitions,
+    annotationsHash: annotations.hash,
+  });
+  let cachedEvidence = await readContentBoundJson(
+    expectedEvidencePath,
+    GraderEvaluationEvidenceSchema,
+    evaluationHash,
+  );
+  if (cachedEvidence) {
+    assertEvidenceIdentity(
+      cachedEvidence,
+      {
+        candidateId: candidate.id,
+        profileId,
+        split,
+        repetitions,
+        annotationsHash: annotations.hash,
+      },
+      expectedEvidencePath,
+    );
+  }
+  const harness = profile.harness ?? execution.config.harness;
+  const registry = buildRegistryFromConfig({
+    ...execution.config,
+    ...(harness ? { harness } : {}),
+  }).withAssertion(createRubricAssertion(harness));
+  const items: GraderEvidenceItem[] = [];
+  for (const annotation of selected) {
+    const rubric = await loadRubricDefinition(
+      candidateRubricPath(options.materializedRoot, annotation.rubricPath),
+    );
+    const baselineRubric = await loadRubricDefinition(
+      candidateRubricPath(execution.loaded.subjectRoot, annotation.rubricPath),
+    );
+    const baselineIds = assertionIds(baselineRubric);
+    const candidateIds = assertionIds(rubric);
+    const allowedAdditions = new Set(
+      execution.loaded.campaign.evaluation.allowedAssertionAdditions[annotation.rubricPath] ?? [],
+    );
+    validateCandidateRubricIds(annotation.rubricPath, baselineIds, candidateIds, allowedAdditions);
+    const persistedRun = await loadFrozenRegradableRun(annotation.runDir);
+    try {
+      if (rubric.id !== persistedRun.case.id) {
+        throw new Error(
+          `rubric '${annotation.rubricPath}' id '${rubric.id}' does not match persisted case '${persistedRun.case.id}'`,
+        );
+      }
+      for (let repetition = 1; repetition <= repetitions; repetition++) {
+        const unitId = graderUnitId({
+          candidateId: candidate.id,
+          profileId,
+          split,
+          repetitions,
+          annotationsHash: annotations.hash,
+          annotationId: annotation.id,
+          repetition,
+        });
+        const unitPath = graderUnitPath(experiment, unitId);
+        const existing = await readContentBoundJson(
+          unitPath,
+          GraderEvaluationUnitSchema,
+          evaluationHash,
+        );
+        if (existing) {
+          assertUnitIdentity(
+            existing,
+            {
+              unitId,
+              candidateId: candidate.id,
+              profileId,
+              split,
+              repetitions,
+              annotationsHash: annotations.hash,
+              annotationId: annotation.id,
+              repetition,
+            },
+            unitPath,
+          );
+          await options.recordUnit(unitId, { ...itemAccounting(), metricCalls: 0 });
+          throwIfExperimentAborted(execution.abortSignal);
+          items.push(existing.item);
+          continue;
+        }
+        if (cachedEvidence) {
+          await retireContentBoundJson(expectedEvidencePath);
+          cachedEvidence = null;
+        }
+        options.preflightUnit(rubric.assertions.length * 2);
+        const run = {
+          ...persistedRun,
+          case: { ...persistedRun.case, assertions: rubric.assertions },
+        };
+        const startedAt = Date.now();
+        const assertionAttemptId = randomUUID();
+        await options.recordAttempt(
+          `grader-metric-attempt-${unitId}-${assertionAttemptId}`,
+          itemAccounting(),
+        );
+        let report: GradingReport;
+        try {
+          report = await gradeRun(
+            run,
+            registry,
+            undefined,
+            execution.abortSignal,
+            async (assertion, index) => {
+              const accounting = assertionModelAccounting(assertion);
+              let unpricedModelCalls = accounting.unpricedModelCalls;
+              if (unpricedModelCalls === 0 && accounting.attempts === 0 && assertion.usage) {
+                unpricedModelCalls = 1;
+              }
+              await options.recordAssertion(
+                `grader-judge-${unitId}-${assertionAttemptId}-${index}`,
+                {
+                  metricCalls: 0,
+                  // Provider-attempt starts journal judgeCalls before spend.
+                  judgeCalls: 0,
+                  usage: assertion.usage ?? emptyUsage(),
+                  costUsd: accounting.costUsd,
+                  unpricedModelCalls,
+                },
+              );
+            },
+            {
+              start: options.recordModelAttemptStart,
+              complete: options.completeModelAttempts,
+            },
+          );
+          const failedAssertion = report.assertions.find(assertionOperationalError);
+          if (report.incomplete?.phase === "grading" || failedAssertion) {
+            throw new Error(
+              failedAssertion?.errorMessage ??
+                failedAssertion?.comment ??
+                "grader assertion could not produce a judgment",
+            );
+          }
+        } catch (error) {
+          await options.recordUnit(`grader-attempt-${unitId}-${assertionAttemptId}`, {
+            ...itemAccounting(),
+            metricCalls: 0,
+          });
+          throw error;
+        }
+        const usage = sumUsage(
+          report.assertions.flatMap((assertion) => (assertion.usage ? [assertion.usage] : [])),
+        );
+        const costUsd = report.assertions.reduce(
+          (total, assertion) => total + assertionModelAccounting(assertion).costUsd,
+          0,
+        );
+        const unpricedModelCalls = report.assertions.reduce(
+          (total, assertion) => total + assertionModelAccounting(assertion).unpricedModelCalls,
+          0,
+        );
+        const item: GraderEvidenceItem = {
+          annotationId: annotation.id,
+          repetition,
+          runDir: annotation.runDir,
+          rubricPath: annotation.rubricPath,
+          ...(annotation.feedback ? { feedback: annotation.feedback } : {}),
+          judgments: compareGraderAnnotation(
+            annotation,
+            report,
+            execution.loaded.campaign.evaluation.scoreTolerance,
+          ),
+          report,
+          durationMs: Date.now() - startedAt,
+          usage,
+          costUsd,
+          unpricedModelCalls,
+        };
+        if (execution.abortSignal?.aborted) {
+          await options.recordUnit(`grader-attempt-${unitId}-${startedAt}`, {
+            ...itemAccounting(),
+            metricCalls: 0,
+          });
+          throwIfExperimentAborted(execution.abortSignal);
+        }
+        const unit: GraderEvaluationUnit = {
+          schemaVersion: 1,
+          unitId,
+          candidateId: candidate.id,
+          profileId,
+          split,
+          repetitions,
+          annotationsHash: annotations.hash,
+          annotationId: annotation.id,
+          repetition,
+          item,
+        };
+        await options.verifyExecutionIdentity();
+        await writeContentBoundJson(unitPath, unit, evaluationHash, async () => {
+          await options.recordUnit(unitId, { ...itemAccounting(), metricCalls: 0 });
+          await options.verifyExecutionIdentity();
+        });
+        throwIfExperimentAborted(execution.abortSignal);
+        items.push(item);
+      }
+    } finally {
+      await releaseFrozenRegradableRun(persistedRun);
+    }
+  }
+  const { metrics, samples } = summarizeGraderEvidence(items, candidate);
+  if (cachedEvidence) {
+    const recomputed = { ...cachedEvidence, metrics, samples, items };
+    if (valueHash(cachedEvidence) !== valueHash(recomputed)) {
+      throw new Error("cached grader evidence does not match its verified evaluation units");
+    }
+    return evaluationFromEvidence(experiment, expectedEvidencePath, cachedEvidence);
+  }
+  const evidence: GraderEvaluationEvidence = {
+    schemaVersion: 1,
+    candidateId: candidate.id,
+    profileId,
+    split,
+    repetitions,
+    annotationsHash: annotations.hash,
+    generatedAt: new Date().toISOString(),
+    metrics,
+    samples,
+    items,
+  };
+  await options.verifyExecutionIdentity();
+  await writeContentBoundJson(
+    expectedEvidencePath,
+    evidence,
+    evaluationHash,
+    options.verifyExecutionIdentity,
+  );
+  return evaluationFromEvidence(experiment, expectedEvidencePath, evidence);
+};