@valerypopoff/rivet2-evaluations 2.3.0 → 2.5.0

package/dist/cjs/bundle.cjs

@@ -28,30 +28,40 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
-var index_exports = {};
-__export(index_exports, {
+var src_exports = {};
+__export(src_exports, {
   EvaluationGraphExecutionError: () => EvaluationGraphExecutionError,
   InMemoryEvaluationRunStore: () => InMemoryEvaluationRunStore,
   LEGACY_EVALUATOR_INPUT_IDS: () => LEGACY_EVALUATOR_INPUT_IDS,
+  applyEvaluationLibraryMutation: () => applyEvaluationLibraryMutation,
   areEvaluationDataTypesCompatible: () => areEvaluationDataTypesCompatible,
   assertEvaluationDatasetSnapshot: () => assertEvaluationDatasetSnapshot,
   assertEvaluationDatasetValuesMatchDeclaredTypes: () => assertEvaluationDatasetValuesMatchDeclaredTypes,
+  assertEvaluationRecordingArtifact: () => assertEvaluationRecordingArtifact,
   assertPortableJson: () => assertPortableJson,
   canonicalStringify: () => canonicalStringify,
   canonicalizePortableJson: () => canonicalizePortableJson,
   clonePortableJson: () => clonePortableJson,
+  createEmptyEvaluationLibrary: () => createEmptyEvaluationLibrary,
   createEmptyEvaluationProjectData: () => createEmptyEvaluationProjectData,
   createEmptyMetrics: () => createEmptyMetrics,
   createEvaluationBaselineSnapshot: () => createEvaluationBaselineSnapshot,
+  createEvaluationRunPlan: () => createEvaluationRunPlan,
+  createEvaluationRunShell: () => createEvaluationRunShell,
   deserializeEvaluationDatasetJson: () => deserializeEvaluationDatasetJson,
   deserializeEvaluationProjectData: () => deserializeEvaluationProjectData,
   deserializeEvaluationProjectDataFromString: () => deserializeEvaluationProjectDataFromString,
   deserializeEvaluationSuiteBundleJson: () => deserializeEvaluationSuiteBundleJson,
+  diffEvaluationLibraryMutation: () => diffEvaluationLibraryMutation,
   evaluateAssertion: () => evaluateAssertion,
   evaluateThresholdResults: () => evaluateThresholdResults,
+  evaluationLibraryValueEquals: () => evaluationLibraryValueEquals,
   finalizeEvaluationRecordingRetention: () => finalizeEvaluationRecordingRetention,
+  finalizeEvaluationRun: () => finalizeEvaluationRun,
   fingerprint: () => fingerprint,
   fingerprintEvaluationDataset: () => fingerprintEvaluationDataset,
+  getEvaluationLibraryResource: () => getEvaluationLibraryResource,
+  getEvaluationSuiteBundle: () => getEvaluationSuiteBundle,
   getEvaluationSuiteMode: () => getEvaluationSuiteMode,
   getEvaluationTopLevelOutputId: () => getEvaluationTopLevelOutputId,
   hasAuthoritativeEvaluationCriteria: () => hasAuthoritativeEvaluationCriteria,
@@ -61,11 +71,15 @@ __export(index_exports, {
   localizeEvaluationDataset: () => localizeEvaluationDataset,
   normalizeEvaluationAggregate: () => normalizeEvaluationAggregate,
   normalizeEvaluationBaselineSnapshot: () => normalizeEvaluationBaselineSnapshot,
+  normalizeEvaluationLibrary: () => normalizeEvaluationLibrary,
   normalizeEvaluationRun: () => normalizeEvaluationRun,
   normalizeEvaluationTrial: () => normalizeEvaluationTrial,
   portableEvaluationDataTypes: () => portableEvaluationDataTypes,
+  preserveEvaluationRunName: () => preserveEvaluationRunName,
+  reconcileEvaluationRunSnapshots: () => reconcileEvaluationRunSnapshots,
   runEvaluationCases: () => runEvaluationCases,
   runEvaluationSuite: () => runEvaluationSuite,
+  runEvaluationTrial: () => runEvaluationTrial,
   runEvaluationWorkPool: () => runEvaluationWorkPool,
   serializeEvaluationDatasetJson: () => serializeEvaluationDatasetJson,
   serializeEvaluationProjectData: () => serializeEvaluationProjectData,
@@ -79,7 +93,7 @@ __export(index_exports, {
   validateEvaluationDataset: () => validateEvaluationDataset,
   validateEvaluationDatasetForTransfer: () => validateEvaluationDatasetForTransfer
 });
-module.exports = __toCommonJS(index_exports);
+module.exports = __toCommonJS(src_exports);
 
 // src/types.ts
 var EvaluationGraphExecutionError = class extends Error {
@@ -178,13 +192,29 @@ function fingerprintEvaluationDataset(value) {
   return fingerprint({ datasetId: value.id, fields: value.fields, cases: value.cases });
 }
 function assertEvaluationDatasetSnapshot(snapshot) {
-  if (snapshot.dataset.projectId !== void 0 && snapshot.dataset.projectId !== snapshot.projectId) {
+  if (typeof snapshot.projectId !== "string" || snapshot.projectId.length === 0 || typeof snapshot.fingerprint !== "string" || snapshot.fingerprint.length === 0 || typeof snapshot.createdAt !== "string" || !Number.isFinite(Date.parse(snapshot.createdAt))) {
+    throw new Error("An evaluation dataset snapshot has invalid identity or creation time.");
+  }
+  if (snapshot.dataset.projectId !== snapshot.projectId) {
     throw new Error("An evaluation dataset snapshot must belong to its project.");
   }
   if (fingerprintEvaluationDataset(snapshot.dataset) !== snapshot.fingerprint) {
     throw new Error("An evaluation dataset snapshot fingerprint must match its dataset ID, fields, and cases.");
   }
 }
+function assertEvaluationRecordingArtifact(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error("An evaluation recording artifact must be an object.");
+  }
+  const artifact = value;
+  if (typeof artifact.projectId !== "string" || artifact.projectId.length === 0 || typeof artifact.runId !== "string" || artifact.runId.length === 0 || typeof artifact.trialId !== "string" || artifact.trialId.length === 0 || typeof artifact.serialized !== "string" || typeof artifact.createdAt !== "string" || !Number.isFinite(Date.parse(artifact.createdAt))) {
+    throw new Error("An evaluation recording artifact has invalid identity, payload, or creation time.");
+  }
+  const reference = artifact.reference;
+  if (typeof reference !== "object" || reference === null || typeof reference.id !== "string" || reference.id.length === 0 || !["temporary", "failure", "baseline", "retained"].includes(reference.retention ?? "") || reference.expiresAt !== void 0 && (typeof reference.expiresAt !== "string" || !Number.isFinite(Date.parse(reference.expiresAt)))) {
+    throw new Error("An evaluation recording artifact has an invalid retention reference.");
+  }
+}
 
 // src/assertions.ts
 function readQuotedPathKey(source, start) {
@@ -680,6 +710,12 @@ function isRunExecutionStatus(value) {
 function isTerminalRunExecutionStatus(status) {
   return status === "completed" || status === "canceled" || status === "error";
 }
+function reconcileRecordingReference(primary, secondary) {
+  if (!primary) return secondary;
+  if (!secondary) return primary;
+  const rank = (retention) => retention === "temporary" ? 0 : retention === "failure" ? 1 : 2;
+  return rank(secondary.retention) > rank(primary.retention) ? secondary : primary;
+}
 function shouldReplaceEvaluationRun(existing, incoming) {
   if (!existing) return true;
   const existingRevision = existing.revision ?? 0;
@@ -687,6 +723,64 @@ function shouldReplaceEvaluationRun(existing, incoming) {
   if (incomingRevision !== existingRevision) return incomingRevision > existingRevision;
   return !isTerminalRunExecutionStatus(existing.executionStatus) || isTerminalRunExecutionStatus(incoming.executionStatus);
 }
+function preserveEvaluationRunName(existing, incoming) {
+  return (existing == null ? void 0 : existing.name) !== void 0 && incoming.name === void 0 ? { ...incoming, name: existing.name } : incoming;
+}
+function reconcileEvaluationRunSnapshots(existing, incoming) {
+  if (!existing) return incoming;
+  const existingRevision = existing.revision ?? 0;
+  const incomingRevision = incoming.revision ?? 0;
+  if (incomingRevision < existingRevision) return existing;
+  if (incomingRevision > existingRevision) return preserveEvaluationRunName(existing, incoming);
+  if (!shouldReplaceEvaluationRun(existing, incoming)) return existing;
+  const existingTerminal = isTerminalRunExecutionStatus(existing.executionStatus);
+  const incomingTerminal = isTerminalRunExecutionStatus(incoming.executionStatus);
+  const existingTrials = existing.trials ?? [];
+  const incomingTrials = incoming.trials ?? [];
+  const incomingIsPrimary = incomingTerminal !== existingTerminal ? incomingTerminal : incomingTrials.length >= existingTrials.length;
+  const primary = preserveEvaluationRunName(existing, incomingIsPrimary ? incoming : existing);
+  const secondary = incomingIsPrimary ? existing : incoming;
+  const primaryTrials = primary.trials ?? [];
+  const secondaryTrialList = secondary.trials ?? [];
+  const name = existing.name ?? incoming.name;
+  const primaryTrialIds = new Set(primaryTrials.map((trial) => trial.id));
+  const secondaryTrials = new Map(secondaryTrialList.map((trial) => [trial.id, trial]));
+  return {
+    ...primary,
+    trials: [
+      ...primaryTrials.map((trial) => {
+        const other = secondaryTrials.get(trial.id);
+        if (!other) return trial;
+        const secondaryObservations = new Map(
+          (other.observations ?? []).map((observation) => [observation.id, observation])
+        );
+        const primaryObservationIds = new Set((trial.observations ?? []).map((observation) => observation.id));
+        const recording = reconcileRecordingReference(trial.recording, other.recording);
+        return {
+          ...trial,
+          ...recording === void 0 ? {} : { recording },
+          observations: [
+            ...(trial.observations ?? []).map((observation) => {
+              const otherObservation = secondaryObservations.get(observation.id);
+              const observationRecording = reconcileRecordingReference(
+                observation.recording,
+                otherObservation == null ? void 0 : otherObservation.recording
+              );
+              return observationRecording === observation.recording ? observation : {
+                ...observation,
+                ...observationRecording === void 0 ? {} : { recording: observationRecording }
+              };
+            }),
+            ...(other.observations ?? []).filter((observation) => !primaryObservationIds.has(observation.id))
+          ]
+        };
+      }),
+      ...secondaryTrialList.filter((trial) => !primaryTrialIds.has(trial.id))
+    ],
+    ...name === void 0 ? {} : { name },
+    warnings: [.../* @__PURE__ */ new Set([...primary.warnings ?? [], ...secondary.warnings ?? []])]
+  };
+}
 function isPurpose(value) {
   return value === "evaluation" || value === "execution-benchmark";
 }
@@ -797,6 +891,11 @@ function normalizeEvaluationAggregate(value, trials) {
   if (!isRecord(value)) throw new Error("Evaluation aggregate must be an object.");
   const normalized = structuredClone(value);
   if (!isNormalizedScore(normalized.meanScore)) delete normalized.meanScore;
+  if (!isNormalizedScore(normalized.medianScore)) delete normalized.medianScore;
+  if (!isNormalizedScore(normalized.p95Score)) delete normalized.p95Score;
+  if (typeof normalized.medianLatencyMs !== "number" || !Number.isFinite(normalized.medianLatencyMs) || normalized.medianLatencyMs < 0) {
+    delete normalized.medianLatencyMs;
+  }
   if (trials) {
     const completed = trials.filter((trial) => trial.executionStatus === "completed");
     const evaluated2 = completed.filter((trial) => trial.qualityStatus === "passed" || trial.qualityStatus === "failed");
@@ -899,6 +998,10 @@ function normalizeEvaluationRun(value) {
   } : value.provenance;
   const normalized = structuredClone(value);
   delete normalized.verdict;
+  const name = typeof value.name === "string" && value.name.trim().length > 0 ? value.name.trim() : void 0;
+  delete normalized.name;
+  const requestedTrialCount = typeof value.requestedTrialCount === "number" && Number.isSafeInteger(value.requestedTrialCount) && value.requestedTrialCount >= 0 ? value.requestedTrialCount : void 0;
+  delete normalized.requestedTrialCount;
   const aggregate2 = value.aggregate === void 0 ? void 0 : normalizeEvaluationAggregate(value.aggregate, trials);
   return {
     ...normalized,
@@ -918,6 +1021,8 @@ function normalizeEvaluationRun(value) {
     provenance,
     trials,
     thresholdResults: Array.isArray(value.thresholdResults) ? structuredClone(value.thresholdResults) : [],
+    ...name === void 0 ? {} : { name },
+    ...requestedTrialCount === void 0 ? {} : { requestedTrialCount },
     ...aggregate2 === void 0 ? {} : { aggregate: accountingStatus === "partial" ? withoutAuthoritativeCost(aggregate2) : aggregate2 }
   };
 }
@@ -1036,7 +1141,7 @@ async function runEvaluationWorkPool(options) {
       const work = options.work[index];
       const result = await options.execute(work, index);
       results[index] = result;
-      (_b = options.onSettled) == null ? void 0 : _b.call(options, result, index, results);
+      await ((_b = options.onSettled) == null ? void 0 : _b.call(options, result, index, results));
     }
   };
   await Promise.all(Array.from({ length: Math.min(options.concurrency, options.work.length) }, worker));
@@ -1316,9 +1421,7 @@ function validateSuite(project, suite, dataset, purpose) {
     for (const assertion of suite.assertions) {
       const outputId = getEvaluationTopLevelOutputId(assertion.outputPath);
       if (assertion.outputPath.trim() !== "$" && outputId === void 0) {
-        throw new Error(
-          `Assertion "${assertion.name}" output path must start with a named target Graph Output.`
-        );
+        throw new Error(`Assertion "${assertion.name}" output path must start with a named target Graph Output.`);
       }
       if (outputId !== void 0 && !graphOutputIds.includes(outputId)) {
         throw new Error(`Assertion "${assertion.name}" references missing target output "${outputId}".`);
@@ -1697,9 +1800,9 @@ async function runWithTimeout(run, timeoutMs, outer) {
     outer == null ? void 0 : outer.removeEventListener("abort", abort);
   }
 }
-async function runTrial(input) {
+async function runEvaluationTrial(input) {
   var _a, _b, _c, _d;
-  const { project, suite, dataset, purpose, testCase, caseIndex, trialIndex, runId, runGraph, signal } = input;
+  const { project, suite, dataset, purpose, testCase, caseIndex, trialIndex, runId, runGraph, signal, trialId } = input;
   const evaluationMode = getEvaluationSuiteMode(suite);
   const { inputs, expected } = resolveCaseInputs(suite, dataset, testCase, purpose);
   const seed = deriveSeed((_a = suite.configuration) == null ? void 0 : _a.seed, testCase.id, trialIndex);
@@ -1745,7 +1848,7 @@ async function runTrial(input) {
       const executionStatus2 = "canceled";
       const qualityStatus2 = "not-evaluated";
       return {
-        id: makeId("trial"),
+        id: trialId ?? makeId("trial"),
         caseId: testCase.id,
         caseName: testCase.name,
         caseIndex,
@@ -1873,7 +1976,7 @@ async function runTrial(input) {
         const executionStatus2 = "canceled";
         const qualityStatus2 = "not-evaluated";
         return {
-          id: makeId("trial"),
+          id: trialId ?? makeId("trial"),
           caseId: testCase.id,
           caseName: testCase.name,
           caseIndex,
@@ -1958,7 +2061,7 @@ async function runTrial(input) {
   }
   const totalMetrics = mergeMetrics(targetMetrics, evaluatorMetrics);
   return {
-    id: makeId("trial"),
+    id: trialId ?? makeId("trial"),
     caseId: testCase.id,
     caseName: testCase.name,
     caseIndex,
@@ -1984,6 +2087,13 @@ function percentile(values, percentage) {
   const ordered = values.slice().sort((left, right) => left - right);
   return ordered[Math.min(ordered.length - 1, Math.ceil(percentage / 100 * ordered.length) - 1)];
 }
+function median(values) {
+  if (values.length === 0) return 0;
+  const ordered = values.slice().sort((left, right) => left - right);
+  const upperIndex = Math.floor(ordered.length / 2);
+  if (ordered.length % 2 !== 0) return ordered[upperIndex];
+  return (ordered[upperIndex - 1] + ordered[upperIndex]) / 2;
+}
 function meanObservationScore(observations) {
   let weightedScore = 0;
   let totalWeight = 0;
@@ -2058,6 +2168,7 @@ function aggregate(trials, evaluationMode = "pass-fail") {
   const scoredTrialCount = cases.reduce((sum, testCase) => sum + (testCase.scoredTrialCount ?? 0), 0);
   const scoringCaseMeans = cases.map((testCase) => testCase.meanScore).filter((score) => score !== void 0);
   const scoringMeanScore = scoringCaseMeans.length === 0 ? void 0 : scoringCaseMeans.reduce((sum, score) => sum + score, 0) / scoringCaseMeans.length;
+  const incurredLatencies = incurred.map((trial) => trial.totalMetrics.durationMs);
   const aggregateValue = {
     trialCount: trials.length,
     evaluatedTrialCount: evaluated.length,
@@ -2071,13 +2182,15 @@ function aggregate(trials, evaluationMode = "pass-fail") {
     ...evaluationMode === "scoring" ? {
       scoredTrialCount,
       missingScoreTrialCount: trials.length - scoredTrialCount,
-      ...scoringMeanScore === void 0 ? {} : { meanScore: scoringMeanScore }
+      ...scoringMeanScore === void 0 ? {} : { meanScore: scoringMeanScore },
+      ...scoringCaseMeans.length === 0 ? {} : {
+        medianScore: median(scoringCaseMeans),
+        p95Score: percentile(scoringCaseMeans, 95)
+      }
     } : meanScore === void 0 ? {} : { meanScore },
-    averageLatencyMs: incurred.length === 0 ? 0 : incurred.reduce((sum, trial) => sum + trial.totalMetrics.durationMs, 0) / incurred.length,
-    p95LatencyMs: percentile(
-      incurred.map((trial) => trial.totalMetrics.durationMs),
-      95
-    ),
+    averageLatencyMs: incurred.length === 0 ? 0 : incurredLatencies.reduce((sum, durationMs) => sum + durationMs, 0) / incurred.length,
+    medianLatencyMs: median(incurredLatencies),
+    p95LatencyMs: percentile(incurredLatencies, 95),
     ...incurred.length === 0 || totalCostUsd === void 0 ? {} : { totalCostUsd, averageCostUsd: totalCostUsd / incurred.length },
     targetErrorRate: settled.length === 0 ? 0 : errors.length / settled.length,
     evaluatorErrorRate: settled.length === 0 ? 0 : settled.filter(
@@ -2405,8 +2518,8 @@ function createProvenance(project, suite, dataset, options) {
     accountingComplete: true
   };
 }
-async function runEvaluationSuite(options) {
-  var _a, _b, _c, _d, _e;
+function createEvaluationRunPlan(options) {
+  var _a;
   const suite = resolveSuite(options.evaluationData, options.suiteId);
   const purpose = options.purpose ?? "evaluation";
   const evaluationMode = getEvaluationSuiteMode(suite);
@@ -2416,6 +2529,110 @@ async function runEvaluationSuite(options) {
     );
   }
   validateSuite(options.project, suite, options.dataset, purpose);
+  const trialCount = Math.max(1, Math.floor(((_a = suite.configuration) == null ? void 0 : _a.trialCount) ?? 1));
+  const work = options.dataset.cases.filter((testCase) => testCase.enabled !== false).flatMap(
+    (testCase, caseIndex) => Array.from({ length: trialCount }, (_, trialIndex) => ({ testCase, caseIndex, trialIndex }))
+  );
+  return { suite, purpose, evaluationMode, work };
+}
+function finalizeEvaluationRun(options) {
+  var _a, _b;
+  const run = options.run;
+  const evaluationMode = getEvaluationSuiteMode(options.suite);
+  run.trials = options.trials.slice();
+  run.completedAt = options.completedAt ?? (/* @__PURE__ */ new Date()).toISOString();
+  const outcome = aggregate(run.trials, evaluationMode);
+  run.aggregate = outcome.aggregate;
+  run.provenance.accountingComplete = !run.trials.some((trial) => trial.totalMetrics.hasUnknownCost);
+  run.accountingStatus = run.provenance.accountingComplete ? "complete" : "partial";
+  if (run.accountingStatus === "partial" && !run.warnings.includes("Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated.")) {
+    run.warnings.push("Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated.");
+  }
+  if (options.canceled) {
+    run.executionStatus = "canceled";
+    run.qualityStatus = "not-evaluated";
+    run.qualityReason = { code: "canceled", message: "The evaluation run was canceled." };
+    return run;
+  }
+  if (run.purpose === "execution-benchmark") {
+    run.executionStatus = "completed";
+    run.qualityStatus = "not-evaluated";
+    run.qualityReason = { code: "benchmark", message: "This run measured execution without evaluating output quality." };
+    return run;
+  }
+  if (evaluationMode === "scoring") {
+    run.executionStatus = "completed";
+    if ((outcome.aggregate.missingScoreTrialCount ?? run.trials.length) > 0) {
+      run.qualityStatus = "unable-to-evaluate";
+      run.qualityReason = { code: "scores-incomplete", message: "One or more requested trials did not produce a usable score. Available averages are shown with coverage." };
+      run.warnings.push(`Score coverage is ${outcome.aggregate.scoredTrialCount ?? 0} of ${outcome.aggregate.trialCount} requested trials.`);
+    } else if ((outcome.aggregate.scoredTrialCount ?? 0) > 0) {
+      run.qualityStatus = "scored";
+      run.qualityReason = { code: "scores-complete", message: "Every requested trial produced a score. Overall score is the equal-weight average of case averages." };
+    } else {
+      run.qualityStatus = "unable-to-evaluate";
+      run.qualityReason = { code: "scores-incomplete", message: "No requested trial produced a usable score." };
+    }
+    return run;
+  }
+  const baseline = options.baseline ?? options.evaluationData.baselines.find((candidate) => candidate.suiteId === options.suite.id);
+  const usableBaseline = baseline && baselineCompatible(run.provenance, baseline) ? baseline : void 0;
+  if (baseline && !usableBaseline && !run.warnings.includes("The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed.")) {
+    run.warnings.push("The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed.");
+  }
+  run.thresholdResults = evaluateThresholdResults(outcome.aggregate, options.suite.thresholds, usableBaseline);
+  const hasPassRateThreshold = ((_a = options.suite.thresholds) == null ? void 0 : _a.some((threshold) => threshold.metric === "pass-rate")) ?? false;
+  const hasTargetErrorRateThreshold = ((_b = options.suite.thresholds) == null ? void 0 : _b.some((threshold) => threshold.metric === "target-error-rate")) ?? false;
+  const hasUngovernedCheckFailure = !hasPassRateThreshold && run.trials.some((trial) => trial.executionStatus === "completed" && trial.qualityStatus === "failed");
+  const hasUngovernedTargetError = !hasTargetErrorRateThreshold && run.trials.some((trial) => trial.executionStatus === "error");
+  const hasThresholdFailure = run.thresholdResults.some((result) => result.status === "failed");
+  const hasUnavailableEvidence = run.trials.some((trial) => trial.qualityStatus === "unable-to-evaluate") || run.thresholdResults.some((result) => result.status === "unavailable");
+  const hasAuthoritativePass = run.trials.some((trial) => trial.executionStatus === "completed" && trial.qualityStatus === "passed") || run.thresholdResults.some((result) => result.status === "passed");
+  run.executionStatus = "completed";
+  if (hasUngovernedCheckFailure || hasUngovernedTargetError || hasThresholdFailure) {
+    run.qualityStatus = "failed";
+    run.qualityReason = { code: hasThresholdFailure ? "thresholds-failed" : "checks-failed", message: hasThresholdFailure ? "One or more required aggregate thresholds failed." : "One or more target executions or required quality checks failed." };
+  } else if (hasUnavailableEvidence) {
+    run.qualityStatus = "unable-to-evaluate";
+    run.qualityReason = { code: run.thresholdResults.some((result) => result.status === "unavailable") ? "required-metric-unavailable" : "required-check-error", message: run.thresholdResults.some((result) => result.status === "unavailable") ? "A required aggregate metric could not be evaluated." : "A required quality check could not be evaluated." };
+  } else if (hasAuthoritativePass) {
+    run.qualityStatus = "passed";
+    run.qualityReason = { code: run.thresholdResults.length > 0 ? "thresholds-passed" : "checks-passed", message: run.thresholdResults.length > 0 ? "All required quality checks and aggregate thresholds passed." : "All required quality checks passed." };
+  } else {
+    run.qualityStatus = "not-evaluated";
+    run.qualityReason = { code: "no-completed-trials", message: "No completed trial produced an authoritative quality result." };
+  }
+  return run;
+}
+function createEvaluationRunShell(options) {
+  const plan = createEvaluationRunPlan(options);
+  const runId = options.runId ?? makeId("evaluation");
+  const run = {
+    version: 2,
+    id: runId,
+    projectId: options.project.metadata.id,
+    suiteId: plan.suite.id,
+    suiteName: plan.suite.name,
+    revision: 0,
+    startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    purpose: plan.purpose,
+    evaluationMode: plan.evaluationMode,
+    executionStatus: "running",
+    qualityStatus: "not-evaluated",
+    qualityReason: { code: "in-progress", message: "The run is still in progress." },
+    accountingStatus: "complete",
+    provenance: createProvenance(options.project, plan.suite, options.dataset, options),
+    trials: [],
+    thresholdResults: [],
+    warnings: [],
+    requestedTrialCount: plan.work.length
+  };
+  return { run, plan };
+}
+async function runEvaluationSuite(options) {
+  var _a, _b;
+  const plan = createEvaluationRunPlan(options);
+  const { suite, purpose, evaluationMode } = plan;
   const runId = options.runId ?? makeId("evaluation");
   const run = {
     version: 2,
@@ -2436,27 +2653,61 @@ async function runEvaluationSuite(options) {
     thresholdResults: [],
     warnings: []
   };
-  const trialCount = Math.max(1, Math.floor(((_a = suite.configuration) == null ? void 0 : _a.trialCount) ?? 1));
   const concurrency = Math.min(
     MAX_CONCURRENCY,
-    Math.max(1, Math.floor(((_b = suite.configuration) == null ? void 0 : _b.concurrency) ?? DEFAULT_CONCURRENCY))
+    Math.max(1, Math.floor(((_a = suite.configuration) == null ? void 0 : _a.concurrency) ?? DEFAULT_CONCURRENCY))
   );
-  const work = options.dataset.cases.filter((testCase) => testCase.enabled !== false).flatMap(
-    (testCase, caseIndex) => Array.from({ length: trialCount }, (_, trialIndex) => ({ testCase, caseIndex, trialIndex }))
-  );
-  const publish = () => {
-    var _a2;
+  const work = plan.work;
+  run.requestedTrialCount = work.length;
+  const settledTrials = Array.from({ length: work.length });
+  let settledTrialCount = 0;
+  const nextRevision = () => {
     run.revision = (run.revision ?? 0) + 1;
-    (_a2 = options.onUpdate) == null ? void 0 : _a2.call(options, structuredClone(run));
+    return run.revision;
+  };
+  const publishLegacySnapshot = () => {
+    if (!options.onUpdate) return;
+    run.trials = settledTrials.filter((trial) => trial !== void 0);
+    options.onUpdate(structuredClone(run));
+  };
+  const publishStarted = async () => {
+    const revision = nextRevision();
+    if (options.onEvent) {
+      const shell = structuredClone(run);
+      shell.trials = [];
+      await options.onEvent({ type: "run-started", revision, run: shell });
+    }
+    publishLegacySnapshot();
+  };
+  const publishSettledTrial = async (trial) => {
+    var _a2;
+    const revision = nextRevision();
+    await ((_a2 = options.onEvent) == null ? void 0 : _a2.call(options, {
+      type: "trial-settled",
+      revision,
+      runId: run.id,
+      projectId: run.projectId,
+      suiteId: run.suiteId,
+      requestedTrialCount: run.requestedTrialCount ?? work.length,
+      settledTrialCount,
+      trial: structuredClone(trial)
+    }));
+    publishLegacySnapshot();
   };
-  publish();
+  const publishFinalized = async () => {
+    var _a2;
+    const revision = nextRevision();
+    await ((_a2 = options.onEvent) == null ? void 0 : _a2.call(options, { type: "run-finalized", revision, run: structuredClone(run) }));
+    publishLegacySnapshot();
+  };
+  await publishStarted();
   const workPool = runEvaluationWorkPool({
     work,
     concurrency,
     signal: options.signal,
     execute: async (task) => {
       try {
-        return await runTrial({
+        return await runEvaluationTrial({
           ...task,
           project: options.project,
           suite,
@@ -2489,98 +2740,22 @@ async function runEvaluationSuite(options) {
         };
       }
     },
-    onSettled: (_result, _index, currentResults) => {
-      run.trials = currentResults.filter((result) => result !== void 0);
-      publish();
+    onSettled: async (result, index) => {
+      settledTrials[index] = result;
+      settledTrialCount += 1;
+      await publishSettledTrial(result);
     }
   });
   const results = await workPool;
-  run.trials = results.filter((result) => result !== void 0);
-  run.completedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const outcome = aggregate(run.trials, evaluationMode);
-  run.aggregate = outcome.aggregate;
-  run.provenance.accountingComplete = !run.trials.some((trial) => trial.totalMetrics.hasUnknownCost);
-  run.accountingStatus = run.provenance.accountingComplete ? "complete" : "partial";
-  if (run.accountingStatus === "partial") {
-    run.warnings.push(
-      "Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated."
-    );
-  }
-  if ((_c = options.signal) == null ? void 0 : _c.aborted) {
-    run.executionStatus = "canceled";
-    run.qualityStatus = "not-evaluated";
-    run.qualityReason = { code: "canceled", message: "The evaluation run was canceled." };
-  } else if (purpose === "execution-benchmark") {
-    run.executionStatus = "completed";
-    run.qualityStatus = "not-evaluated";
-    run.qualityReason = {
-      code: "benchmark",
-      message: "This run measured execution without evaluating output quality."
-    };
-  } else if (evaluationMode === "scoring") {
-    run.executionStatus = "completed";
-    if ((outcome.aggregate.missingScoreTrialCount ?? run.trials.length) > 0) {
-      run.qualityStatus = "unable-to-evaluate";
-      run.qualityReason = {
-        code: "scores-incomplete",
-        message: "One or more requested trials did not produce a usable score. Available averages are shown with coverage."
-      };
-      run.warnings.push(
-        `Score coverage is ${outcome.aggregate.scoredTrialCount ?? 0} of ${outcome.aggregate.trialCount} requested trials.`
-      );
-    } else if ((outcome.aggregate.scoredTrialCount ?? 0) > 0) {
-      run.qualityStatus = "scored";
-      run.qualityReason = {
-        code: "scores-complete",
-        message: "Every requested trial produced a score. Overall score is the equal-weight average of case averages."
-      };
-    } else {
-      run.qualityStatus = "unable-to-evaluate";
-      run.qualityReason = { code: "scores-incomplete", message: "No requested trial produced a usable score." };
-    }
-  } else {
-    const baseline = options.baseline ?? options.evaluationData.baselines.find((candidate) => candidate.suiteId === suite.id);
-    const usableBaseline = baseline && baselineCompatible(run.provenance, baseline) ? baseline : void 0;
-    if (baseline && !usableBaseline)
-      run.warnings.push(
-        "The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed."
-      );
-    run.thresholdResults = evaluateThresholdResults(outcome.aggregate, suite.thresholds, usableBaseline);
-    const hasPassRateThreshold = ((_d = suite.thresholds) == null ? void 0 : _d.some((threshold) => threshold.metric === "pass-rate")) ?? false;
-    const hasTargetErrorRateThreshold = ((_e = suite.thresholds) == null ? void 0 : _e.some((threshold) => threshold.metric === "target-error-rate")) ?? false;
-    const hasUngovernedCheckFailure = !hasPassRateThreshold && run.trials.some((trial) => trial.executionStatus === "completed" && trial.qualityStatus === "failed");
-    const hasUngovernedTargetError = !hasTargetErrorRateThreshold && run.trials.some((trial) => trial.executionStatus === "error");
-    const hasThresholdFailure = run.thresholdResults.some((result) => result.status === "failed");
-    const hasUnavailableEvidence = run.trials.some((trial) => trial.qualityStatus === "unable-to-evaluate") || run.thresholdResults.some((result) => result.status === "unavailable");
-    const hasAuthoritativePass = run.trials.some((trial) => trial.executionStatus === "completed" && trial.qualityStatus === "passed") || run.thresholdResults.some((result) => result.status === "passed");
-    if (hasUngovernedCheckFailure || hasUngovernedTargetError || hasThresholdFailure) {
-      run.qualityStatus = "failed";
-      run.qualityReason = {
-        code: hasThresholdFailure ? "thresholds-failed" : "checks-failed",
-        message: hasThresholdFailure ? "One or more required aggregate thresholds failed." : "One or more target executions or required quality checks failed."
-      };
-    } else if (hasUnavailableEvidence) {
-      run.qualityStatus = "unable-to-evaluate";
-      run.qualityReason = {
-        code: run.thresholdResults.some((result) => result.status === "unavailable") ? "required-metric-unavailable" : "required-check-error",
-        message: run.thresholdResults.some((result) => result.status === "unavailable") ? "A required aggregate metric could not be evaluated." : "A required quality check could not be evaluated."
-      };
-    } else if (hasAuthoritativePass) {
-      run.qualityStatus = "passed";
-      run.qualityReason = {
-        code: run.thresholdResults.length > 0 ? "thresholds-passed" : "checks-passed",
-        message: run.thresholdResults.length > 0 ? "All required quality checks and aggregate thresholds passed." : "All required quality checks passed."
-      };
-    } else {
-      run.qualityStatus = "not-evaluated";
-      run.qualityReason = {
-        code: "no-completed-trials",
-        message: "No completed trial produced an authoritative quality result."
-      };
-    }
-    run.executionStatus = "completed";
-  }
-  publish();
+  finalizeEvaluationRun({
+    run,
+    trials: results.filter((result) => result !== void 0),
+    suite,
+    evaluationData: options.evaluationData,
+    baseline: options.baseline,
+    canceled: ((_b = options.signal) == null ? void 0 : _b.aborted) === true
+  });
+  await publishFinalized();
   return run;
 }
 async function runEvaluationCases(options) {
@@ -2619,93 +2794,131 @@ function createEvaluationBaselineSnapshot(run) {
   };
 }
 
-// src/store.ts
-var InMemoryEvaluationRunStore = class {
-  #runs = /* @__PURE__ */ new Map();
-  #datasetSnapshots = /* @__PURE__ */ new Map();
-  #recordings = /* @__PURE__ */ new Map();
-  async put(run) {
-    const normalized = normalizeEvaluationRun(run);
-    const key = `${normalized.projectId}/${normalized.id}`;
-    const existing = this.#runs.get(key);
-    if (!shouldReplaceEvaluationRun(existing, normalized)) return;
-    this.#runs.set(key, structuredClone(normalized));
-  }
-  async get(input) {
-    const run = this.#runs.get(`${input.projectId}/${input.runId}`);
-    return run ? normalizeEvaluationRun(run) : void 0;
-  }
-  async list(input) {
-    return Array.from(this.#runs.values()).filter(
-      (run) => run.projectId === input.projectId && (input.suiteId === void 0 || run.suiteId === input.suiteId)
-    ).sort((left, right) => right.startedAt.localeCompare(left.startedAt)).map(normalizeEvaluationRun);
-  }
-  async delete(input) {
-    this.#runs.delete(`${input.projectId}/${input.runId}`);
-    for (const [key, artifact] of this.#recordings) {
-      if (artifact.projectId === input.projectId && artifact.runId === input.runId) this.#recordings.delete(key);
+// src/datasetTransfer.ts
+var DATASET_EXPORT_VERSION = 1;
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function requireString(value, path) {
+  if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string.`);
+  return value;
+}
+function requireText(value, path) {
+  if (typeof value !== "string") throw new Error(`${path} must be a string.`);
+  return value;
+}
+function parseFields(value) {
+  if (!Array.isArray(value)) throw new Error("Evaluation dataset fields must be an array.");
+  const ids = /* @__PURE__ */ new Set();
+  return value.map((field, index) => {
+    if (!isRecord2(field)) throw new Error(`fields[${index}] must be an object.`);
+    const id = requireString(field.id, `fields[${index}].id`);
+    if (ids.has(id)) throw new Error(`Evaluation dataset has duplicate field id "${id}".`);
+    ids.add(id);
+    const role = field.role;
+    if (role !== "input" && role !== "expected" && role !== "metadata") {
+      throw new Error(`fields[${index}].role must be input, expected, or metadata.`);
     }
-  }
-  async putDatasetSnapshot(snapshot) {
-    assertEvaluationDatasetSnapshot(snapshot);
-    const key = `${snapshot.projectId}/${snapshot.fingerprint}`;
-    if (this.#datasetSnapshots.has(key)) return;
-    this.#datasetSnapshots.set(key, structuredClone(snapshot));
-  }
-  async getDatasetSnapshot(input) {
-    const snapshot = this.#datasetSnapshots.get(`${input.projectId}/${input.fingerprint}`);
-    return snapshot ? structuredClone(snapshot) : void 0;
-  }
-  async putRecording(artifact) {
-    const key = `${artifact.projectId}/${artifact.reference.id}`;
-    const existing = this.#recordings.get(key);
-    if (existing && (existing.runId !== artifact.runId || existing.trialId !== artifact.trialId)) {
-      throw new Error("An evaluation recording ID cannot be reassigned to another run or trial.");
+    if (field.description !== void 0 && typeof field.description !== "string") {
+      throw new Error(`fields[${index}].description must be a string.`);
     }
-    const next = structuredClone(artifact);
-    if (existing) next.reference = structuredClone(existing.reference);
-    this.#recordings.set(key, next);
-  }
-  async getRecording(input) {
-    const artifact = this.#recordings.get(`${input.projectId}/${input.recordingId}`);
-    if (!artifact) return void 0;
-    if (artifact.reference.retention === "temporary" && artifact.reference.expiresAt && Date.parse(artifact.reference.expiresAt) <= Date.now()) {
-      this.#recordings.delete(`${input.projectId}/${input.recordingId}`);
-      return void 0;
+    if (field.required !== void 0 && typeof field.required !== "boolean") {
+      throw new Error(`fields[${index}].required must be a boolean.`);
     }
-    return structuredClone(artifact);
-  }
-  async updateRecordingRetention(input) {
-    const key = `${input.projectId}/${input.recordingId}`;
-    const artifact = this.#recordings.get(key);
-    if (!artifact) return;
-    artifact.reference = {
-      id: artifact.reference.id,
-      retention: input.retention,
-      ...input.expiresAt === void 0 ? {} : { expiresAt: input.expiresAt }
+    return {
+      id,
+      name: requireText(field.name, `fields[${index}].name`),
+      dataType: requireString(field.dataType, `fields[${index}].dataType`),
+      role,
+      ...typeof field.description === "string" ? { description: field.description } : {},
+      ...typeof field.required === "boolean" ? { required: field.required } : {}
     };
-  }
-  async promoteBaseline(input) {
-    for (const artifact of this.#recordings.values()) {
-      if (artifact.projectId !== input.projectId || artifact.runId !== input.runId) continue;
-      artifact.reference = { id: artifact.reference.id, retention: "baseline" };
+  });
+}
+function parseCases(value, fieldIds) {
+  if (!Array.isArray(value)) throw new Error("Evaluation dataset cases must be an array.");
+  const ids = /* @__PURE__ */ new Set();
+  return value.map((testCase, index) => {
+    if (!isRecord2(testCase)) throw new Error(`cases[${index}] must be an object.`);
+    const id = requireString(testCase.id, `cases[${index}].id`);
+    if (ids.has(id)) throw new Error(`Evaluation dataset has duplicate case id "${id}".`);
+    ids.add(id);
+    if (!isRecord2(testCase.values)) throw new Error(`cases[${index}].values must be an object.`);
+    for (const [fieldId, fieldValue] of Object.entries(testCase.values)) {
+      if (!fieldIds.has(fieldId)) throw new Error(`cases[${index}].values references unknown field "${fieldId}".`);
+      assertPortableJson(fieldValue, `cases[${index}].values.${fieldId}`);
+    }
+    if (testCase.tags !== void 0 && (!Array.isArray(testCase.tags) || testCase.tags.some((tag) => typeof tag !== "string"))) {
+      throw new Error(`cases[${index}].tags must be an array of strings.`);
     }
+    if (testCase.note !== void 0 && typeof testCase.note !== "string") throw new Error(`cases[${index}].note must be a string.`);
+    if (testCase.enabled !== void 0 && typeof testCase.enabled !== "boolean") throw new Error(`cases[${index}].enabled must be a boolean.`);
+    return {
+      id,
+      name: requireText(testCase.name, `cases[${index}].name`),
+      values: testCase.values,
+      ...typeof testCase.enabled === "boolean" ? { enabled: testCase.enabled } : {},
+      ...Array.isArray(testCase.tags) ? { tags: testCase.tags } : {},
+      ...typeof testCase.note === "string" ? { note: testCase.note } : {}
+    };
+  });
+}
+function validateEvaluationDataset(value, scope) {
+  if (!isRecord2(value)) throw new Error("Evaluation dataset must be an object.");
+  const fields = parseFields(value.fields);
+  const importedProjectId = value.projectId === void 0 ? void 0 : requireString(value.projectId, "dataset.projectId");
+  const projectId = (scope == null ? void 0 : scope.projectId) ?? importedProjectId;
+  const dataset = {
+    id: (scope == null ? void 0 : scope.id) ?? requireString(value.id, "dataset.id"),
+    name: requireText(value.name, "dataset.name"),
+    ...projectId === void 0 ? {} : { projectId },
+    fields,
+    cases: parseCases(value.cases, new Set(fields.map((field) => field.id)))
+  };
+  return { ...dataset, contentFingerprint: fingerprintEvaluationDataset(dataset) };
+}
+function localizeEvaluationDataset(value, id) {
+  const dataset = validateEvaluationDataset(value, id === void 0 ? void 0 : { id });
+  const { projectId: _legacyProjectId, ...localDataset } = dataset;
+  return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
+}
+function validateEvaluationDatasetForTransfer(value, scope) {
+  const dataset = validateEvaluationDataset(value, scope);
+  assertEvaluationDatasetValuesMatchDeclaredTypes(dataset);
+  return dataset;
+}
+function serializeEvaluationDatasetJson(dataset) {
+  const validated = validateEvaluationDatasetForTransfer(dataset);
+  return JSON.stringify({ version: DATASET_EXPORT_VERSION, dataset: validated }, null, 2);
+}
+function deserializeEvaluationDatasetJson(source, scope) {
+  let parsed;
+  try {
+    parsed = JSON.parse(source);
+  } catch {
+    throw new Error("Evaluation dataset JSON is not valid JSON.");
   }
-};
+  if (!isRecord2(parsed) || parsed.version !== DATASET_EXPORT_VERSION || !isRecord2(parsed.dataset)) {
+    throw new Error(`Expected an evaluation dataset export with version ${DATASET_EXPORT_VERSION}.`);
+  }
+  const dataset = validateEvaluationDatasetForTransfer(parsed.dataset, { id: scope.id });
+  const { projectId: _legacyProjectId, ...localDataset } = dataset;
+  return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
+}
 
 // src/serialization.ts
 var yaml = __toESM(require("yaml"), 1);
-function isRecord2(value) {
+function isRecord3(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 function requireRecord(value, path) {
-  if (!isRecord2(value)) throw new Error(`${path} must be an object.`);
+  if (!isRecord3(value)) throw new Error(`${path} must be an object.`);
   return value;
 }
-function requireString(value, path) {
+function requireString2(value, path) {
   if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path} must be a non-empty string.`);
 }
-function requireText(value, path) {
+function requireText2(value, path) {
   if (typeof value !== "string") throw new Error(`${path} must be a string.`);
 }
 function requireBoolean(value, path) {
@@ -2747,21 +2960,21 @@ var EVALUATOR_CONTEXT_INPUTS = /* @__PURE__ */ new Set(["case", "inputs", "expec
 function validateSuiteShape(value, index) {
   const path = `evaluations.suites[${index}]`;
   const suite = requireRecord(value, path);
-  for (const key of ["id", "targetGraphId", "datasetId"]) requireString(suite[key], `${path}.${key}`);
-  requireText(suite.name, `${path}.name`);
+  for (const key of ["id", "targetGraphId", "datasetId"]) requireString2(suite[key], `${path}.${key}`);
+  requireText2(suite.name, `${path}.name`);
   if (suite.evaluationMode !== void 0 && suite.evaluationMode !== "pass-fail" && suite.evaluationMode !== "scoring") {
     throw new Error(`${path}.evaluationMode must be "pass-fail" or "scoring".`);
   }
   for (const [bindingIndex, bindingValue] of requireArray(suite.inputBindings, `${path}.inputBindings`).entries()) {
     const binding = requireRecord(bindingValue, `${path}.inputBindings[${bindingIndex}]`);
-    requireString(binding.graphInputId, `${path}.inputBindings[${bindingIndex}].graphInputId`);
-    requireString(binding.datasetFieldId, `${path}.inputBindings[${bindingIndex}].datasetFieldId`);
+    requireString2(binding.graphInputId, `${path}.inputBindings[${bindingIndex}].graphInputId`);
+    requireString2(binding.datasetFieldId, `${path}.inputBindings[${bindingIndex}].datasetFieldId`);
   }
   for (const [assertionIndex, assertionValue] of requireArray(suite.assertions, `${path}.assertions`).entries()) {
     const assertionPath = `${path}.assertions[${assertionIndex}]`;
     const assertion = requireRecord(assertionValue, assertionPath);
-    for (const key of ["id", "outputPath"]) requireString(assertion[key], `${assertionPath}.${key}`);
-    requireText(assertion.name, `${assertionPath}.name`);
+    for (const key of ["id", "outputPath"]) requireString2(assertion[key], `${assertionPath}.${key}`);
+    requireText2(assertion.name, `${assertionPath}.name`);
     if (typeof assertion.operator !== "string" || !ASSERTION_OPERATORS.has(assertion.operator)) {
       throw new Error(`${assertionPath}.operator is not supported.`);
     }
@@ -2770,7 +2983,7 @@ function validateSuiteShape(value, index) {
     if (expected.kind === "literal") {
       if (!Object.hasOwn(expected, "value")) throw new Error(`${assertionPath}.expected.value is required.`);
     } else if (expected.kind === "dataset-field") {
-      requireString(expected.fieldId, `${assertionPath}.expected.fieldId`);
+      requireString2(expected.fieldId, `${assertionPath}.expected.fieldId`);
     } else {
       throw new Error(`${assertionPath}.expected.kind must be "literal" or "dataset-field".`);
     }
@@ -2778,8 +2991,8 @@ function validateSuiteShape(value, index) {
   for (const [evaluatorIndex, evaluatorValue] of requireArray(suite.evaluators, `${path}.evaluators`).entries()) {
     const evaluatorPath = `${path}.evaluators[${evaluatorIndex}]`;
     const evaluator = requireRecord(evaluatorValue, evaluatorPath);
-    for (const key of ["id", "graphId"]) requireString(evaluator[key], `${evaluatorPath}.${key}`);
-    requireText(evaluator.name, `${evaluatorPath}.name`);
+    for (const key of ["id", "graphId"]) requireString2(evaluator[key], `${evaluatorPath}.${key}`);
+    requireText2(evaluator.name, `${evaluatorPath}.name`);
     validateOptionalPrimitive(evaluator, "required", "boolean", evaluatorPath);
     validateOptionalPrimitive(evaluator, "runOnTargetError", "boolean", evaluatorPath);
     validateOptionalPrimitive(evaluator, "scoreWeight", "number", evaluatorPath);
@@ -2790,10 +3003,10 @@ function validateSuiteShape(value, index) {
       ).entries()) {
         const bindingPath = `${evaluatorPath}.inputBindings[${bindingIndex}]`;
         const binding = requireRecord(bindingValue, bindingPath);
-        requireString(binding.graphInputId, `${bindingPath}.graphInputId`);
+        requireString2(binding.graphInputId, `${bindingPath}.graphInputId`);
         const source = requireRecord(binding.source, `${bindingPath}.source`);
-        if (source.kind === "dataset-field") requireString(source.fieldId, `${bindingPath}.source.fieldId`);
-        else if (source.kind === "target-output") requireString(source.outputId, `${bindingPath}.source.outputId`);
+        if (source.kind === "dataset-field") requireString2(source.fieldId, `${bindingPath}.source.fieldId`);
+        else if (source.kind === "target-output") requireString2(source.outputId, `${bindingPath}.source.outputId`);
         else if (source.kind === "context") {
           if (typeof source.context !== "string" || !EVALUATOR_CONTEXT_INPUTS.has(source.context)) {
             throw new Error(`${bindingPath}.source.context is not supported.`);
@@ -2808,7 +3021,7 @@ function validateSuiteShape(value, index) {
     for (const [thresholdIndex, thresholdValue2] of requireArray(suite.thresholds, `${path}.thresholds`).entries()) {
       const thresholdPath = `${path}.thresholds[${thresholdIndex}]`;
       const threshold = requireRecord(thresholdValue2, thresholdPath);
-      for (const key of ["id", "metric", "operator"]) requireString(threshold[key], `${thresholdPath}.${key}`);
+      for (const key of ["id", "metric", "operator"]) requireString2(threshold[key], `${thresholdPath}.${key}`);
       requireFiniteNumber(threshold.value, `${thresholdPath}.value`);
     }
   }
@@ -2823,7 +3036,7 @@ function validateSuiteShape(value, index) {
   validateOptionalPrimitive(suite, "description", "string", path);
   if (suite.tags !== void 0) {
     for (const [tagIndex, tag] of requireArray(suite.tags, `${path}.tags`).entries()) {
-      requireString(tag, `${path}.tags[${tagIndex}]`);
+      requireString2(tag, `${path}.tags[${tagIndex}]`);
     }
   }
 }
@@ -2834,7 +3047,7 @@ function validateNumericRecord(value, path) {
 function validateBaselineShape(value, index) {
   const path = `evaluations.baselines[${index}]`;
   const baseline = requireRecord(value, path);
-  for (const key of ["id", "suiteId", "createdAt"]) requireString(baseline[key], `${path}.${key}`);
+  for (const key of ["id", "suiteId", "createdAt"]) requireString2(baseline[key], `${path}.${key}`);
   validateOptionalPrimitive(baseline, "sourceRunId", "string", path);
   if (baseline.evaluationMode !== void 0 && baseline.evaluationMode !== "pass-fail" && baseline.evaluationMode !== "scoring") {
     throw new Error(`${path}.evaluationMode must be "pass-fail" or "scoring".`);
@@ -2847,14 +3060,14 @@ function validateBaselineShape(value, index) {
     "targetFingerprint",
     "executionMode"
   ]) {
-    requireString(provenance[key], `${path}.provenance.${key}`);
+    requireString2(provenance[key], `${path}.provenance.${key}`);
   }
   const evaluatorFingerprints = requireRecord(
     provenance.evaluatorFingerprints,
     `${path}.provenance.evaluatorFingerprints`
   );
   for (const [key, fingerprint2] of Object.entries(evaluatorFingerprints)) {
-    requireString(fingerprint2, `${path}.provenance.evaluatorFingerprints.${key}`);
+    requireString2(fingerprint2, `${path}.provenance.evaluatorFingerprints.${key}`);
   }
   requireBoolean(provenance.accountingComplete, `${path}.provenance.accountingComplete`);
   const aggregate2 = requireRecord(baseline.aggregate, `${path}.aggregate`);
@@ -2878,6 +3091,9 @@ function validateBaselineShape(value, index) {
     "notEvaluatedTrialCount",
     "unableToEvaluateTrialCount",
     "meanScore",
+    "medianScore",
+    "p95Score",
+    "medianLatencyMs",
     "totalCostUsd",
     "averageCostUsd",
     "scoredTrialCount",
@@ -2889,8 +3105,8 @@ function validateBaselineShape(value, index) {
   for (const [caseIndex, caseValue] of requireArray(baseline.cases, `${path}.cases`).entries()) {
     const casePath = `${path}.cases[${caseIndex}]`;
     const caseAggregate = requireRecord(caseValue, casePath);
-    requireString(caseAggregate.caseId, `${casePath}.caseId`);
-    requireString(caseAggregate.caseName, `${casePath}.caseName`);
+    requireString2(caseAggregate.caseId, `${casePath}.caseId`);
+    requireString2(caseAggregate.caseName, `${casePath}.caseName`);
     validateNumericRecord(caseAggregate.metrics, `${casePath}.metrics`);
     for (const key of [
       "passRate",
@@ -2944,117 +3160,160 @@ function deserializeEvaluationProjectDataFromString(data) {
   return deserializeEvaluationProjectData(yaml.parse(data));
 }
 
-// src/datasetTransfer.ts
-var DATASET_EXPORT_VERSION = 1;
-function isRecord3(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function requireString2(value, path) {
-  if (typeof value !== "string" || value.length === 0) throw new Error(`${path} must be a non-empty string.`);
-  return value;
-}
-function requireText2(value, path) {
-  if (typeof value !== "string") throw new Error(`${path} must be a string.`);
-  return value;
+// src/library.ts
+function createEmptyEvaluationLibrary() {
+  return {
+    version: 1,
+    data: createEmptyEvaluationProjectData(),
+    datasets: [],
+    migratedLegacyProjectIds: []
+  };
 }
-function parseFields(value) {
-  if (!Array.isArray(value)) throw new Error("Evaluation dataset fields must be an array.");
-  const ids = /* @__PURE__ */ new Set();
-  return value.map((field, index) => {
-    if (!isRecord3(field)) throw new Error(`fields[${index}] must be an object.`);
-    const id = requireString2(field.id, `fields[${index}].id`);
-    if (ids.has(id)) throw new Error(`Evaluation dataset has duplicate field id "${id}".`);
-    ids.add(id);
-    const role = field.role;
-    if (role !== "input" && role !== "expected" && role !== "metadata") {
-      throw new Error(`fields[${index}].role must be input, expected, or metadata.`);
-    }
-    if (field.description !== void 0 && typeof field.description !== "string") {
-      throw new Error(`fields[${index}].description must be a string.`);
-    }
-    if (field.required !== void 0 && typeof field.required !== "boolean") {
-      throw new Error(`fields[${index}].required must be a boolean.`);
+function normalizeEvaluationLibrary(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return createEmptyEvaluationLibrary();
+  const candidate = value;
+  let data = createEmptyEvaluationProjectData();
+  try {
+    data = deserializeEvaluationProjectData(candidate.data);
+  } catch {
+    const dataCandidate = typeof candidate.data === "object" && candidate.data !== null && !Array.isArray(candidate.data) ? candidate.data : void 0;
+    const suites2 = Array.isArray(dataCandidate == null ? void 0 : dataCandidate.suites) ? dataCandidate.suites.flatMap((suite) => {
+      try {
+        return [deserializeEvaluationProjectData({ version: 1, suites: [suite], baselines: [] }).suites[0]];
+      } catch {
+        return [];
+      }
+    }) : [];
+    const baselines = Array.isArray(dataCandidate == null ? void 0 : dataCandidate.baselines) ? dataCandidate.baselines.flatMap((baseline) => {
+      try {
+        return [deserializeEvaluationProjectData({ version: 1, suites: [], baselines: [baseline] }).baselines[0]];
+      } catch {
+        return [];
+      }
+    }) : [];
+    const suiteIds2 = new Set(suites2.map((suite) => suite.id));
+    data = { version: 1, suites: suites2, baselines: baselines.filter((baseline) => suiteIds2.has(baseline.suiteId)) };
+  }
+  const datasets = Array.isArray(candidate.datasets) ? candidate.datasets.flatMap((dataset) => {
+    try {
+      return [localizeEvaluationDataset(dataset)];
+    } catch {
+      return [];
     }
-    return {
-      id,
-      name: requireText2(field.name, `fields[${index}].name`),
-      dataType: requireString2(field.dataType, `fields[${index}].dataType`),
-      role,
-      ...typeof field.description === "string" ? { description: field.description } : {},
-      ...typeof field.required === "boolean" ? { required: field.required } : {}
-    };
+  }) : [];
+  const suites = uniqueById(data.suites);
+  const suiteIds = new Set(suites.map((suite) => suite.id));
+  return {
+    version: 1,
+    data: {
+      ...data,
+      suites,
+      baselines: uniqueById(data.baselines).filter((baseline) => suiteIds.has(baseline.suiteId))
+    },
+    datasets: uniqueById(datasets),
+    migratedLegacyProjectIds: Array.isArray(candidate.migratedLegacyProjectIds) ? Array.from(new Set(candidate.migratedLegacyProjectIds.filter((id) => typeof id === "string"))) : []
+  };
+}
+function uniqueById(values) {
+  const seen = /* @__PURE__ */ new Set();
+  return values.filter((value) => {
+    if (seen.has(value.id)) return false;
+    seen.add(value.id);
+    return true;
   });
 }
-function parseCases(value, fieldIds) {
-  if (!Array.isArray(value)) throw new Error("Evaluation dataset cases must be an array.");
-  const ids = /* @__PURE__ */ new Set();
-  return value.map((testCase, index) => {
-    if (!isRecord3(testCase)) throw new Error(`cases[${index}] must be an object.`);
-    const id = requireString2(testCase.id, `cases[${index}].id`);
-    if (ids.has(id)) throw new Error(`Evaluation dataset has duplicate case id "${id}".`);
-    ids.add(id);
-    if (!isRecord3(testCase.values)) throw new Error(`cases[${index}].values must be an object.`);
-    for (const [fieldId, fieldValue] of Object.entries(testCase.values)) {
-      if (!fieldIds.has(fieldId)) throw new Error(`cases[${index}].values references unknown field "${fieldId}".`);
-      assertPortableJson(fieldValue, `cases[${index}].values.${fieldId}`);
+
+// src/store.ts
+var InMemoryEvaluationRunStore = class {
+  #runs = /* @__PURE__ */ new Map();
+  #datasetSnapshots = /* @__PURE__ */ new Map();
+  #recordings = /* @__PURE__ */ new Map();
+  #library = createEmptyEvaluationLibrary();
+  async getLibrary() {
+    return structuredClone(this.#library);
+  }
+  async putLibrary(library) {
+    this.#library = structuredClone(normalizeEvaluationLibrary(library));
+  }
+  async put(run) {
+    const normalized = normalizeEvaluationRun(run);
+    const key = `${normalized.projectId}/${normalized.id}`;
+    const existing = this.#runs.get(key);
+    const next = reconcileEvaluationRunSnapshots(existing, normalized);
+    if (next === existing) return;
+    this.#runs.set(key, structuredClone(next));
+  }
+  async updateRunName(input) {
+    const key = `${input.projectId}/${input.runId}`;
+    const existing = this.#runs.get(key);
+    if (!existing) return void 0;
+    const renamed = normalizeEvaluationRun({ ...existing, name: input.name });
+    this.#runs.set(key, structuredClone(renamed));
+    return structuredClone(renamed);
+  }
+  async get(input) {
+    const run = this.#runs.get(`${input.projectId}/${input.runId}`);
+    return run ? normalizeEvaluationRun(run) : void 0;
+  }
+  async list(input) {
+    return Array.from(this.#runs.values()).filter(
+      (run) => run.projectId === input.projectId && (input.suiteId === void 0 || run.suiteId === input.suiteId)
+    ).sort((left, right) => right.startedAt.localeCompare(left.startedAt)).map(normalizeEvaluationRun);
+  }
+  async delete(input) {
+    this.#runs.delete(`${input.projectId}/${input.runId}`);
+    for (const [key, artifact] of this.#recordings) {
+      if (artifact.projectId === input.projectId && artifact.runId === input.runId) this.#recordings.delete(key);
     }
-    if (testCase.tags !== void 0 && (!Array.isArray(testCase.tags) || testCase.tags.some((tag) => typeof tag !== "string"))) {
-      throw new Error(`cases[${index}].tags must be an array of strings.`);
+  }
+  async putDatasetSnapshot(snapshot) {
+    assertEvaluationDatasetSnapshot(snapshot);
+    const key = `${snapshot.projectId}/${snapshot.fingerprint}`;
+    if (this.#datasetSnapshots.has(key)) return;
+    this.#datasetSnapshots.set(key, structuredClone(snapshot));
+  }
+  async getDatasetSnapshot(input) {
+    const snapshot = this.#datasetSnapshots.get(`${input.projectId}/${input.fingerprint}`);
+    return snapshot ? structuredClone(snapshot) : void 0;
+  }
+  async putRecording(artifact) {
+    assertEvaluationRecordingArtifact(artifact);
+    const key = `${artifact.projectId}/${artifact.reference.id}`;
+    const existing = this.#recordings.get(key);
+    if (existing && (existing.runId !== artifact.runId || existing.trialId !== artifact.trialId)) {
+      throw new Error("An evaluation recording ID cannot be reassigned to another run or trial.");
     }
-    if (testCase.note !== void 0 && typeof testCase.note !== "string") throw new Error(`cases[${index}].note must be a string.`);
-    if (testCase.enabled !== void 0 && typeof testCase.enabled !== "boolean") throw new Error(`cases[${index}].enabled must be a boolean.`);
-    return {
-      id,
-      name: requireText2(testCase.name, `cases[${index}].name`),
-      values: testCase.values,
-      ...typeof testCase.enabled === "boolean" ? { enabled: testCase.enabled } : {},
-      ...Array.isArray(testCase.tags) ? { tags: testCase.tags } : {},
-      ...typeof testCase.note === "string" ? { note: testCase.note } : {}
+    const next = structuredClone(artifact);
+    if (existing) next.reference = structuredClone(existing.reference);
+    this.#recordings.set(key, next);
+  }
+  async getRecording(input) {
+    const artifact = this.#recordings.get(`${input.projectId}/${input.recordingId}`);
+    if (!artifact) return void 0;
+    if (artifact.reference.retention === "temporary" && artifact.reference.expiresAt && Date.parse(artifact.reference.expiresAt) <= Date.now()) {
+      this.#recordings.delete(`${input.projectId}/${input.recordingId}`);
+      return void 0;
+    }
+    return structuredClone(artifact);
+  }
+  async updateRecordingRetention(input) {
+    const key = `${input.projectId}/${input.recordingId}`;
+    const artifact = this.#recordings.get(key);
+    if (!artifact) return false;
+    artifact.reference = {
+      id: artifact.reference.id,
+      retention: input.retention,
+      ...input.expiresAt === void 0 ? {} : { expiresAt: input.expiresAt }
     };
-  });
-}
-function validateEvaluationDataset(value, scope) {
-  if (!isRecord3(value)) throw new Error("Evaluation dataset must be an object.");
-  const fields = parseFields(value.fields);
-  const importedProjectId = value.projectId === void 0 ? void 0 : requireString2(value.projectId, "dataset.projectId");
-  const projectId = (scope == null ? void 0 : scope.projectId) ?? importedProjectId;
-  const dataset = {
-    id: (scope == null ? void 0 : scope.id) ?? requireString2(value.id, "dataset.id"),
-    name: requireText2(value.name, "dataset.name"),
-    ...projectId === void 0 ? {} : { projectId },
-    fields,
-    cases: parseCases(value.cases, new Set(fields.map((field) => field.id)))
-  };
-  return { ...dataset, contentFingerprint: fingerprintEvaluationDataset(dataset) };
-}
-function localizeEvaluationDataset(value, id) {
-  const dataset = validateEvaluationDataset(value, id === void 0 ? void 0 : { id });
-  const { projectId: _legacyProjectId, ...localDataset } = dataset;
-  return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
-}
-function validateEvaluationDatasetForTransfer(value, scope) {
-  const dataset = validateEvaluationDataset(value, scope);
-  assertEvaluationDatasetValuesMatchDeclaredTypes(dataset);
-  return dataset;
-}
-function serializeEvaluationDatasetJson(dataset) {
-  const validated = validateEvaluationDatasetForTransfer(dataset);
-  return JSON.stringify({ version: DATASET_EXPORT_VERSION, dataset: validated }, null, 2);
-}
-function deserializeEvaluationDatasetJson(source, scope) {
-  let parsed;
-  try {
-    parsed = JSON.parse(source);
-  } catch {
-    throw new Error("Evaluation dataset JSON is not valid JSON.");
+    return true;
   }
-  if (!isRecord3(parsed) || parsed.version !== DATASET_EXPORT_VERSION || !isRecord3(parsed.dataset)) {
-    throw new Error(`Expected an evaluation dataset export with version ${DATASET_EXPORT_VERSION}.`);
+  async promoteBaseline(input) {
+    for (const artifact of this.#recordings.values()) {
+      if (artifact.projectId !== input.projectId || artifact.runId !== input.runId) continue;
+      artifact.reference = { id: artifact.reference.id, retention: "baseline" };
+    }
   }
-  const dataset = validateEvaluationDatasetForTransfer(parsed.dataset, { id: scope.id });
-  const { projectId: _legacyProjectId, ...localDataset } = dataset;
-  return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
-}
+};
 
 // src/suiteTransfer.ts
 var SUITE_BUNDLE_EXPORT_VERSION = 1;
@@ -3106,30 +3365,139 @@ function deserializeEvaluationSuiteBundleJson(source, scope) {
     }
   };
 }
+
+// src/librarySync.ts
+function evaluationLibraryValueEquals(left, right) {
+  if (left === void 0 || right === void 0) return left === right;
+  return canonicalStringify(left) === canonicalStringify(right);
+}
+function getEvaluationSuiteBundle(library, id) {
+  const suite = library.data.suites.find((candidate) => candidate.id === id);
+  if (!suite) return void 0;
+  return {
+    suite,
+    baselines: library.data.baselines.filter((baseline) => baseline.suiteId === id)
+  };
+}
+function getEvaluationLibraryResource(library, kind, id) {
+  if (kind === "suite") return { kind, id, value: getEvaluationSuiteBundle(library, id) };
+  return { kind, id, value: library.datasets.find((dataset) => dataset.id === id) };
+}
+function resourceIds(library, kind) {
+  return new Set(kind === "suite" ? library.data.suites.map((suite) => suite.id) : library.datasets.map((dataset) => dataset.id));
+}
+function diffEvaluationLibraryMutation(previous, next, versions) {
+  const changes = [];
+  for (const kind of ["dataset", "suite"]) {
+    const ids = /* @__PURE__ */ new Set([...resourceIds(previous, kind), ...resourceIds(next, kind)]);
+    for (const id of [...ids].sort()) {
+      if (kind === "suite") {
+        const before = getEvaluationSuiteBundle(previous, id);
+        const after = getEvaluationSuiteBundle(next, id);
+        if (evaluationLibraryValueEquals(before, after)) continue;
+        const currentVersion = versions.suites[id];
+        if (before !== void 0 && currentVersion === void 0) {
+          throw new Error(`The suite "${id}" has no synchronization version.`);
+        }
+        const expectedVersion = before === void 0 ? null : currentVersion;
+        if (after === void 0) {
+          changes.push({ kind: "delete-suite", id, expectedVersion });
+        } else {
+          changes.push({
+            kind: "put-suite",
+            id,
+            expectedVersion,
+            suite: after.suite,
+            baselines: after.baselines
+          });
+        }
+      } else {
+        const before = previous.datasets.find((dataset) => dataset.id === id);
+        const after = next.datasets.find((dataset) => dataset.id === id);
+        if (evaluationLibraryValueEquals(before, after)) continue;
+        const currentVersion = versions.datasets[id];
+        if (before !== void 0 && currentVersion === void 0) {
+          throw new Error(`The dataset "${id}" has no synchronization version.`);
+        }
+        const expectedVersion = before === void 0 ? null : currentVersion;
+        if (after === void 0) {
+          changes.push({ kind: "delete-dataset", id, expectedVersion });
+        } else {
+          changes.push({ kind: "put-dataset", id, expectedVersion, dataset: after });
+        }
+      }
+    }
+  }
+  return changes.length === 0 ? void 0 : { changes };
+}
+function replaceById(items, value) {
+  const index = items.findIndex((item) => item.id === value.id);
+  if (index === -1) return [...items, value];
+  return items.map((item) => item.id === value.id ? value : item);
+}
+function applyEvaluationLibraryMutation(library, mutation) {
+  let suites = library.data.suites;
+  let baselines = library.data.baselines;
+  let datasets = library.datasets;
+  for (const change of mutation.changes) {
+    switch (change.kind) {
+      case "put-suite":
+        suites = replaceById(suites, change.suite);
+        baselines = [...baselines.filter((baseline) => baseline.suiteId !== change.id), ...change.baselines];
+        break;
+      case "delete-suite":
+        suites = suites.filter((suite) => suite.id !== change.id);
+        baselines = baselines.filter((baseline) => baseline.suiteId !== change.id);
+        break;
+      case "put-dataset":
+        datasets = replaceById(datasets, change.dataset);
+        break;
+      case "delete-dataset":
+        datasets = datasets.filter((dataset) => dataset.id !== change.id);
+        break;
+    }
+  }
+  return normalizeEvaluationLibrary({
+    version: 1,
+    data: { version: 1, suites, baselines },
+    datasets,
+    migratedLegacyProjectIds: library.migratedLegacyProjectIds
+  });
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   EvaluationGraphExecutionError,
   InMemoryEvaluationRunStore,
   LEGACY_EVALUATOR_INPUT_IDS,
+  applyEvaluationLibraryMutation,
   areEvaluationDataTypesCompatible,
   assertEvaluationDatasetSnapshot,
   assertEvaluationDatasetValuesMatchDeclaredTypes,
+  assertEvaluationRecordingArtifact,
   assertPortableJson,
   canonicalStringify,
   canonicalizePortableJson,
   clonePortableJson,
+  createEmptyEvaluationLibrary,
   createEmptyEvaluationProjectData,
   createEmptyMetrics,
   createEvaluationBaselineSnapshot,
+  createEvaluationRunPlan,
+  createEvaluationRunShell,
   deserializeEvaluationDatasetJson,
   deserializeEvaluationProjectData,
   deserializeEvaluationProjectDataFromString,
   deserializeEvaluationSuiteBundleJson,
+  diffEvaluationLibraryMutation,
   evaluateAssertion,
   evaluateThresholdResults,
+  evaluationLibraryValueEquals,
   finalizeEvaluationRecordingRetention,
+  finalizeEvaluationRun,
   fingerprint,
   fingerprintEvaluationDataset,
+  getEvaluationLibraryResource,
+  getEvaluationSuiteBundle,
   getEvaluationSuiteMode,
   getEvaluationTopLevelOutputId,
   hasAuthoritativeEvaluationCriteria,
@@ -3139,11 +3507,15 @@ function deserializeEvaluationSuiteBundleJson(source, scope) {
   localizeEvaluationDataset,
   normalizeEvaluationAggregate,
   normalizeEvaluationBaselineSnapshot,
+  normalizeEvaluationLibrary,
   normalizeEvaluationRun,
   normalizeEvaluationTrial,
   portableEvaluationDataTypes,
+  preserveEvaluationRunName,
+  reconcileEvaluationRunSnapshots,
   runEvaluationCases,
   runEvaluationSuite,
+  runEvaluationTrial,
   runEvaluationWorkPool,
   serializeEvaluationDatasetJson,
   serializeEvaluationProjectData,