@valerypopoff/rivet2-evaluations 2.3.0 → 2.4.0

package/dist/esm/runner.js

@@ -41,7 +41,7 @@ export async function runEvaluationWorkPool(options) {
             const work = options.work[index];
             const result = await options.execute(work, index);
             results[index] = result;
-            options.onSettled?.(result, index, results);
+            await options.onSettled?.(result, index, results);
         }
     };
     await Promise.all(Array.from({ length: Math.min(options.concurrency, options.work.length) }, worker));
@@ -744,8 +744,8 @@ async function runWithTimeout(run, timeoutMs, outer) {
         outer?.removeEventListener('abort', abort);
     }
 }
-async function runTrial(input) {
-    const { project, suite, dataset, purpose, testCase, caseIndex, trialIndex, runId, runGraph, signal } = input;
+export async function runEvaluationTrial(input) {
+    const { project, suite, dataset, purpose, testCase, caseIndex, trialIndex, runId, runGraph, signal, trialId } = input;
     const evaluationMode = getEvaluationSuiteMode(suite);
     const { inputs, expected } = resolveCaseInputs(suite, dataset, testCase, purpose);
     const seed = deriveSeed(suite.configuration?.seed, testCase.id, trialIndex);
@@ -792,7 +792,7 @@ async function runTrial(input) {
             const executionStatus = 'canceled';
             const qualityStatus = 'not-evaluated';
             return {
-                id: makeId('trial'),
+                id: trialId ?? makeId('trial'),
                 caseId: testCase.id,
                 caseName: testCase.name,
                 caseIndex,
@@ -919,7 +919,7 @@ async function runTrial(input) {
                 const executionStatus = 'canceled';
                 const qualityStatus = 'not-evaluated';
                 return {
-                    id: makeId('trial'),
+                    id: trialId ?? makeId('trial'),
                     caseId: testCase.id,
                     caseName: testCase.name,
                     caseIndex,
@@ -1015,7 +1015,7 @@ async function runTrial(input) {
     }
     const totalMetrics = mergeMetrics(targetMetrics, evaluatorMetrics);
     return {
-        id: makeId('trial'),
+        id: trialId ?? makeId('trial'),
         caseId: testCase.id,
         caseName: testCase.name,
         caseIndex,
@@ -1042,6 +1042,15 @@ function percentile(values, percentage) {
     const ordered = values.slice().sort((left, right) => left - right);
     return ordered[Math.min(ordered.length - 1, Math.ceil((percentage / 100) * ordered.length) - 1)];
 }
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const ordered = values.slice().sort((left, right) => left - right);
+    const upperIndex = Math.floor(ordered.length / 2);
+    if (ordered.length % 2 !== 0)
+        return ordered[upperIndex];
+    return (ordered[upperIndex - 1] + ordered[upperIndex]) / 2;
+}
 function meanObservationScore(observations) {
     let weightedScore = 0;
     let totalWeight = 0;
@@ -1141,6 +1150,7 @@ function aggregate(trials, evaluationMode = 'pass-fail') {
     const scoringMeanScore = scoringCaseMeans.length === 0
         ? undefined
         : scoringCaseMeans.reduce((sum, score) => sum + score, 0) / scoringCaseMeans.length;
+    const incurredLatencies = incurred.map((trial) => trial.totalMetrics.durationMs);
     const aggregateValue = {
         trialCount: trials.length,
         evaluatedTrialCount: evaluated.length,
@@ -1156,14 +1166,19 @@ function aggregate(trials, evaluationMode = 'pass-fail') {
                 scoredTrialCount,
                 missingScoreTrialCount: trials.length - scoredTrialCount,
                 ...(scoringMeanScore === undefined ? {} : { meanScore: scoringMeanScore }),
+                ...(scoringCaseMeans.length === 0
+                    ? {}
+                    : {
+                        medianScore: median(scoringCaseMeans),
+                        p95Score: percentile(scoringCaseMeans, 95),
+                    }),
             }
             : meanScore === undefined
                 ? {}
                 : { meanScore }),
-        averageLatencyMs: incurred.length === 0
-            ? 0
-            : incurred.reduce((sum, trial) => sum + trial.totalMetrics.durationMs, 0) / incurred.length,
-        p95LatencyMs: percentile(incurred.map((trial) => trial.totalMetrics.durationMs), 95),
+        averageLatencyMs: incurred.length === 0 ? 0 : incurredLatencies.reduce((sum, durationMs) => sum + durationMs, 0) / incurred.length,
+        medianLatencyMs: median(incurredLatencies),
+        p95LatencyMs: percentile(incurredLatencies, 95),
         ...(incurred.length === 0 || totalCostUsd === undefined
             ? {}
             : { totalCostUsd, averageCostUsd: totalCostUsd / incurred.length }),
@@ -1515,7 +1530,12 @@ function createProvenance(project, suite, dataset, options) {
         accountingComplete: true,
     };
 }
-export async function runEvaluationSuite(options) {
+/**
+ * Validates an immutable Evaluation submission and expands its enabled cases
+ * into deterministic work. Hosts can persist this plan before any graph side
+ * effect, while local callers continue to use runEvaluationSuite.
+ */
+export function createEvaluationRunPlan(options) {
     const suite = resolveSuite(options.evaluationData, options.suiteId);
     const purpose = options.purpose ?? 'evaluation';
     const evaluationMode = getEvaluationSuiteMode(suite);
@@ -1525,6 +1545,119 @@ export async function runEvaluationSuite(options) {
             : `Evaluation suite "${suite.name}" has no required quality check or threshold. Add a quality criterion or run it as an execution benchmark.`);
     }
     validateSuite(options.project, suite, options.dataset, purpose);
+    const trialCount = Math.max(1, Math.floor(suite.configuration?.trialCount ?? 1));
+    const work = options.dataset.cases
+        .filter((testCase) => testCase.enabled !== false)
+        .flatMap((testCase, caseIndex) => Array.from({ length: trialCount }, (_, trialIndex) => ({ testCase, caseIndex, trialIndex })));
+    return { suite, purpose, evaluationMode, work };
+}
+/**
+ * Applies the same aggregate, quality, accounting, and baseline semantics to
+ * locally scheduled and durably hosted trials. Callers must provide every
+ * settled trial in deterministic case/trial order; never synthesize retries.
+ */
+export function finalizeEvaluationRun(options) {
+    const run = options.run;
+    const evaluationMode = getEvaluationSuiteMode(options.suite);
+    run.trials = options.trials.slice();
+    run.completedAt = options.completedAt ?? new Date().toISOString();
+    const outcome = aggregate(run.trials, evaluationMode);
+    run.aggregate = outcome.aggregate;
+    run.provenance.accountingComplete = !run.trials.some((trial) => trial.totalMetrics.hasUnknownCost);
+    run.accountingStatus = run.provenance.accountingComplete ? 'complete' : 'partial';
+    if (run.accountingStatus === 'partial' && !run.warnings.includes('Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated.')) {
+        run.warnings.push('Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated.');
+    }
+    if (options.canceled) {
+        run.executionStatus = 'canceled';
+        run.qualityStatus = 'not-evaluated';
+        run.qualityReason = { code: 'canceled', message: 'The evaluation run was canceled.' };
+        return run;
+    }
+    if (run.purpose === 'execution-benchmark') {
+        run.executionStatus = 'completed';
+        run.qualityStatus = 'not-evaluated';
+        run.qualityReason = { code: 'benchmark', message: 'This run measured execution without evaluating output quality.' };
+        return run;
+    }
+    if (evaluationMode === 'scoring') {
+        run.executionStatus = 'completed';
+        if ((outcome.aggregate.missingScoreTrialCount ?? run.trials.length) > 0) {
+            run.qualityStatus = 'unable-to-evaluate';
+            run.qualityReason = { code: 'scores-incomplete', message: 'One or more requested trials did not produce a usable score. Available averages are shown with coverage.' };
+            run.warnings.push(`Score coverage is ${outcome.aggregate.scoredTrialCount ?? 0} of ${outcome.aggregate.trialCount} requested trials.`);
+        }
+        else if ((outcome.aggregate.scoredTrialCount ?? 0) > 0) {
+            run.qualityStatus = 'scored';
+            run.qualityReason = { code: 'scores-complete', message: 'Every requested trial produced a score. Overall score is the equal-weight average of case averages.' };
+        }
+        else {
+            run.qualityStatus = 'unable-to-evaluate';
+            run.qualityReason = { code: 'scores-incomplete', message: 'No requested trial produced a usable score.' };
+        }
+        return run;
+    }
+    const baseline = options.baseline ?? options.evaluationData.baselines.find((candidate) => candidate.suiteId === options.suite.id);
+    const usableBaseline = baseline && baselineCompatible(run.provenance, baseline) ? baseline : undefined;
+    if (baseline && !usableBaseline && !run.warnings.includes('The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed.')) {
+        run.warnings.push('The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed.');
+    }
+    run.thresholdResults = evaluateThresholdResults(outcome.aggregate, options.suite.thresholds, usableBaseline);
+    const hasPassRateThreshold = options.suite.thresholds?.some((threshold) => threshold.metric === 'pass-rate') ?? false;
+    const hasTargetErrorRateThreshold = options.suite.thresholds?.some((threshold) => threshold.metric === 'target-error-rate') ?? false;
+    const hasUngovernedCheckFailure = !hasPassRateThreshold && run.trials.some((trial) => trial.executionStatus === 'completed' && trial.qualityStatus === 'failed');
+    const hasUngovernedTargetError = !hasTargetErrorRateThreshold && run.trials.some((trial) => trial.executionStatus === 'error');
+    const hasThresholdFailure = run.thresholdResults.some((result) => result.status === 'failed');
+    const hasUnavailableEvidence = run.trials.some((trial) => trial.qualityStatus === 'unable-to-evaluate') || run.thresholdResults.some((result) => result.status === 'unavailable');
+    const hasAuthoritativePass = run.trials.some((trial) => trial.executionStatus === 'completed' && trial.qualityStatus === 'passed') || run.thresholdResults.some((result) => result.status === 'passed');
+    run.executionStatus = 'completed';
+    if (hasUngovernedCheckFailure || hasUngovernedTargetError || hasThresholdFailure) {
+        run.qualityStatus = 'failed';
+        run.qualityReason = { code: hasThresholdFailure ? 'thresholds-failed' : 'checks-failed', message: hasThresholdFailure ? 'One or more required aggregate thresholds failed.' : 'One or more target executions or required quality checks failed.' };
+    }
+    else if (hasUnavailableEvidence) {
+        run.qualityStatus = 'unable-to-evaluate';
+        run.qualityReason = { code: run.thresholdResults.some((result) => result.status === 'unavailable') ? 'required-metric-unavailable' : 'required-check-error', message: run.thresholdResults.some((result) => result.status === 'unavailable') ? 'A required aggregate metric could not be evaluated.' : 'A required quality check could not be evaluated.' };
+    }
+    else if (hasAuthoritativePass) {
+        run.qualityStatus = 'passed';
+        run.qualityReason = { code: run.thresholdResults.length > 0 ? 'thresholds-passed' : 'checks-passed', message: run.thresholdResults.length > 0 ? 'All required quality checks and aggregate thresholds passed.' : 'All required quality checks passed.' };
+    }
+    else {
+        run.qualityStatus = 'not-evaluated';
+        run.qualityReason = { code: 'no-completed-trials', message: 'No completed trial produced an authoritative quality result.' };
+    }
+    return run;
+}
+/** Creates a durable running shell only after the complete submission validates. */
+export function createEvaluationRunShell(options) {
+    const plan = createEvaluationRunPlan(options);
+    const runId = options.runId ?? makeId('evaluation');
+    const run = {
+        version: 2,
+        id: runId,
+        projectId: options.project.metadata.id,
+        suiteId: plan.suite.id,
+        suiteName: plan.suite.name,
+        revision: 0,
+        startedAt: new Date().toISOString(),
+        purpose: plan.purpose,
+        evaluationMode: plan.evaluationMode,
+        executionStatus: 'running',
+        qualityStatus: 'not-evaluated',
+        qualityReason: { code: 'in-progress', message: 'The run is still in progress.' },
+        accountingStatus: 'complete',
+        provenance: createProvenance(options.project, plan.suite, options.dataset, options),
+        trials: [],
+        thresholdResults: [],
+        warnings: [],
+        requestedTrialCount: plan.work.length,
+    };
+    return { run, plan };
+}
+export async function runEvaluationSuite(options) {
+    const plan = createEvaluationRunPlan(options);
+    const { suite, purpose, evaluationMode } = plan;
     const runId = options.runId ?? makeId('evaluation');
     const run = {
         version: 2,
@@ -1545,30 +1678,62 @@ export async function runEvaluationSuite(options) {
         thresholdResults: [],
         warnings: [],
     };
-    const trialCount = Math.max(1, Math.floor(suite.configuration?.trialCount ?? 1));
     const concurrency = Math.min(MAX_CONCURRENCY, Math.max(1, Math.floor(suite.configuration?.concurrency ?? DEFAULT_CONCURRENCY)));
-    const work = options.dataset.cases
-        .filter((testCase) => testCase.enabled !== false)
-        .flatMap((testCase, caseIndex) => Array.from({ length: trialCount }, (_, trialIndex) => ({ testCase, caseIndex, trialIndex })));
-    const publish = () => {
+    const work = plan.work;
+    run.requestedTrialCount = work.length;
+    const settledTrials = Array.from({ length: work.length });
+    let settledTrialCount = 0;
+    const nextRevision = () => {
         run.revision = (run.revision ?? 0) + 1;
-        // Progress callbacks and stores may retain earlier revisions. Publish a
-        // fully detached snapshot so later mutations to warnings, provenance,
-        // metrics, observations, or nested trial data cannot rewrite history in
-        // place behind the consumer's back.
-        options.onUpdate?.(structuredClone(run));
+        return run.revision;
+    };
+    const publishLegacySnapshot = () => {
+        // Legacy consumers may retain earlier revisions. Keep their snapshots
+        // detached, while incremental consumers avoid cloning all prior trials.
+        if (!options.onUpdate)
+            return;
+        run.trials = settledTrials.filter((trial) => trial !== undefined);
+        options.onUpdate(structuredClone(run));
+    };
+    const publishStarted = async () => {
+        const revision = nextRevision();
+        if (options.onEvent) {
+            const shell = structuredClone(run);
+            shell.trials = [];
+            await options.onEvent({ type: 'run-started', revision, run: shell });
+        }
+        publishLegacySnapshot();
+    };
+    const publishSettledTrial = async (trial) => {
+        const revision = nextRevision();
+        await options.onEvent?.({
+            type: 'trial-settled',
+            revision,
+            runId: run.id,
+            projectId: run.projectId,
+            suiteId: run.suiteId,
+            requestedTrialCount: run.requestedTrialCount ?? work.length,
+            settledTrialCount,
+            trial: structuredClone(trial),
+        });
+        publishLegacySnapshot();
+    };
+    const publishFinalized = async () => {
+        const revision = nextRevision();
+        await options.onEvent?.({ type: 'run-finalized', revision, run: structuredClone(run) });
+        publishLegacySnapshot();
     };
     // Publish the running shell before constructing the worker-pool promise:
     // async workers start immediately, so publishing afterwards can otherwise
     // let the first graph call race ahead of the run's initial state.
-    publish();
+    await publishStarted();
     const workPool = runEvaluationWorkPool({
         work,
         concurrency,
         signal: options.signal,
         execute: async (task) => {
             try {
-                return await runTrial({
+                return await runEvaluationTrial({
                     ...task,
                     project: options.project,
                     suite,
@@ -1612,115 +1777,22 @@ export async function runEvaluationSuite(options) {
                 };
             }
         },
-        onSettled: (_result, _index, currentResults) => {
-            run.trials = currentResults.filter((result) => result !== undefined);
-            publish();
+        onSettled: async (result, index) => {
+            settledTrials[index] = result;
+            settledTrialCount += 1;
+            await publishSettledTrial(result);
         },
     });
     const results = await workPool;
-    run.trials = results.filter((result) => result !== undefined);
-    run.completedAt = new Date().toISOString();
-    const outcome = aggregate(run.trials, evaluationMode);
-    run.aggregate = outcome.aggregate;
-    run.provenance.accountingComplete = !run.trials.some((trial) => trial.totalMetrics.hasUnknownCost);
-    run.accountingStatus = run.provenance.accountingComplete ? 'complete' : 'partial';
-    if (run.accountingStatus === 'partial') {
-        run.warnings.push('Some provider pricing was unavailable. Cost totals are unavailable, and cost requirements cannot be evaluated.');
-    }
-    if (options.signal?.aborted) {
-        run.executionStatus = 'canceled';
-        run.qualityStatus = 'not-evaluated';
-        run.qualityReason = { code: 'canceled', message: 'The evaluation run was canceled.' };
-    }
-    else if (purpose === 'execution-benchmark') {
-        run.executionStatus = 'completed';
-        run.qualityStatus = 'not-evaluated';
-        run.qualityReason = {
-            code: 'benchmark',
-            message: 'This run measured execution without evaluating output quality.',
-        };
-    }
-    else if (evaluationMode === 'scoring') {
-        run.executionStatus = 'completed';
-        if ((outcome.aggregate.missingScoreTrialCount ?? run.trials.length) > 0) {
-            run.qualityStatus = 'unable-to-evaluate';
-            run.qualityReason = {
-                code: 'scores-incomplete',
-                message: 'One or more requested trials did not produce a usable score. Available averages are shown with coverage.',
-            };
-            run.warnings.push(`Score coverage is ${outcome.aggregate.scoredTrialCount ?? 0} of ${outcome.aggregate.trialCount} requested trials.`);
-        }
-        else if ((outcome.aggregate.scoredTrialCount ?? 0) > 0) {
-            run.qualityStatus = 'scored';
-            run.qualityReason = {
-                code: 'scores-complete',
-                message: 'Every requested trial produced a score. Overall score is the equal-weight average of case averages.',
-            };
-        }
-        else {
-            run.qualityStatus = 'unable-to-evaluate';
-            run.qualityReason = { code: 'scores-incomplete', message: 'No requested trial produced a usable score.' };
-        }
-    }
-    else {
-        const baseline = options.baseline ?? options.evaluationData.baselines.find((candidate) => candidate.suiteId === suite.id);
-        const usableBaseline = baseline && baselineCompatible(run.provenance, baseline) ? baseline : undefined;
-        if (baseline && !usableBaseline)
-            run.warnings.push('The suite baseline is stale because its target, dataset, bindings, or evaluator definition changed.');
-        run.thresholdResults = evaluateThresholdResults(outcome.aggregate, suite.thresholds, usableBaseline);
-        const hasPassRateThreshold = suite.thresholds?.some((threshold) => threshold.metric === 'pass-rate') ?? false;
-        const hasTargetErrorRateThreshold = suite.thresholds?.some((threshold) => threshold.metric === 'target-error-rate') ?? false;
-        // Per-trial checks and target execution errors are strict by default. An
-        // author can deliberately replace either default with its matching
-        // aggregate tolerance, but one kind of threshold must never hide the
-        // other kind of failure.
-        const hasUngovernedCheckFailure = !hasPassRateThreshold &&
-            run.trials.some((trial) => trial.executionStatus === 'completed' && trial.qualityStatus === 'failed');
-        const hasUngovernedTargetError = !hasTargetErrorRateThreshold && run.trials.some((trial) => trial.executionStatus === 'error');
-        const hasThresholdFailure = run.thresholdResults.some((result) => result.status === 'failed');
-        const hasUnavailableEvidence = run.trials.some((trial) => trial.qualityStatus === 'unable-to-evaluate') ||
-            run.thresholdResults.some((result) => result.status === 'unavailable');
-        const hasAuthoritativePass = run.trials.some((trial) => trial.executionStatus === 'completed' && trial.qualityStatus === 'passed') ||
-            run.thresholdResults.some((result) => result.status === 'passed');
-        if (hasUngovernedCheckFailure || hasUngovernedTargetError || hasThresholdFailure) {
-            run.qualityStatus = 'failed';
-            run.qualityReason = {
-                code: hasThresholdFailure ? 'thresholds-failed' : 'checks-failed',
-                message: hasThresholdFailure
-                    ? 'One or more required aggregate thresholds failed.'
-                    : 'One or more target executions or required quality checks failed.',
-            };
-        }
-        else if (hasUnavailableEvidence) {
-            run.qualityStatus = 'unable-to-evaluate';
-            run.qualityReason = {
-                code: run.thresholdResults.some((result) => result.status === 'unavailable')
-                    ? 'required-metric-unavailable'
-                    : 'required-check-error',
-                message: run.thresholdResults.some((result) => result.status === 'unavailable')
-                    ? 'A required aggregate metric could not be evaluated.'
-                    : 'A required quality check could not be evaluated.',
-            };
-        }
-        else if (hasAuthoritativePass) {
-            run.qualityStatus = 'passed';
-            run.qualityReason = {
-                code: run.thresholdResults.length > 0 ? 'thresholds-passed' : 'checks-passed',
-                message: run.thresholdResults.length > 0
-                    ? 'All required quality checks and aggregate thresholds passed.'
-                    : 'All required quality checks passed.',
-            };
-        }
-        else {
-            run.qualityStatus = 'not-evaluated';
-            run.qualityReason = {
-                code: 'no-completed-trials',
-                message: 'No completed trial produced an authoritative quality result.',
-            };
-        }
-        run.executionStatus = 'completed';
-    }
-    publish();
+    finalizeEvaluationRun({
+        run,
+        trials: results.filter((result) => result !== undefined),
+        suite,
+        evaluationData: options.evaluationData,
+        baseline: options.baseline,
+        canceled: options.signal?.aborted === true,
+    });
+    await publishFinalized();
     return run;
 }
 export async function runEvaluationCases(options) {

package/dist/esm/serialization.js

@@ -209,6 +209,9 @@ function validateBaselineShape(value, index) {
         'notEvaluatedTrialCount',
         'unableToEvaluateTrialCount',
         'meanScore',
+        'medianScore',
+        'p95Score',
+        'medianLatencyMs',
         'totalCostUsd',
         'averageCostUsd',
         'scoredTrialCount',

package/dist/esm/store.js

@@ -1,16 +1,34 @@
-import { assertEvaluationDatasetSnapshot } from './canonical.js';
-import { normalizeEvaluationRun, shouldReplaceEvaluationRun } from './normalization.js';
+import { assertEvaluationDatasetSnapshot, assertEvaluationRecordingArtifact } from './canonical.js';
+import { normalizeEvaluationRun, reconcileEvaluationRunSnapshots } from './normalization.js';
+import { createEmptyEvaluationLibrary, normalizeEvaluationLibrary } from './library.js';
 export class InMemoryEvaluationRunStore {
     #runs = new Map();
     #datasetSnapshots = new Map();
     #recordings = new Map();
+    #library = createEmptyEvaluationLibrary();
+    async getLibrary() {
+        return structuredClone(this.#library);
+    }
+    async putLibrary(library) {
+        this.#library = structuredClone(normalizeEvaluationLibrary(library));
+    }
     async put(run) {
         const normalized = normalizeEvaluationRun(run);
         const key = `${normalized.projectId}/${normalized.id}`;
         const existing = this.#runs.get(key);
-        if (!shouldReplaceEvaluationRun(existing, normalized))
+        const next = reconcileEvaluationRunSnapshots(existing, normalized);
+        if (next === existing)
             return;
-        this.#runs.set(key, structuredClone(normalized));
+        this.#runs.set(key, structuredClone(next));
+    }
+    async updateRunName(input) {
+        const key = `${input.projectId}/${input.runId}`;
+        const existing = this.#runs.get(key);
+        if (!existing)
+            return undefined;
+        const renamed = normalizeEvaluationRun({ ...existing, name: input.name });
+        this.#runs.set(key, structuredClone(renamed));
+        return structuredClone(renamed);
     }
     async get(input) {
         const run = this.#runs.get(`${input.projectId}/${input.runId}`);
@@ -43,6 +61,7 @@ export class InMemoryEvaluationRunStore {
         return snapshot ? structuredClone(snapshot) : undefined;
     }
     async putRecording(artifact) {
+        assertEvaluationRecordingArtifact(artifact);
         const key = `${artifact.projectId}/${artifact.reference.id}`;
         const existing = this.#recordings.get(key);
         if (existing && (existing.runId !== artifact.runId || existing.trialId !== artifact.trialId)) {
@@ -71,12 +90,13 @@ export class InMemoryEvaluationRunStore {
         const key = `${input.projectId}/${input.recordingId}`;
         const artifact = this.#recordings.get(key);
         if (!artifact)
-            return;
+            return false;
         artifact.reference = {
             id: artifact.reference.id,
             retention: input.retention,
             ...(input.expiresAt === undefined ? {} : { expiresAt: input.expiresAt }),
         };
+        return true;
     }
     async promoteBaseline(input) {
         for (const artifact of this.#recordings.values()) {

package/dist/types/canonical.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationDatasetSnapshot, PortableJson } from './types.js';
+import type { EvaluationDatasetSnapshot, EvaluationRecordingArtifact, PortableJson } from './types.js';
 export declare function assertPortableJson(value: unknown, path?: string, stack?: Set<object>): asserts value is PortableJson;
 export declare function canonicalizePortableJson(value: unknown): PortableJson;
 /**
@@ -21,3 +21,5 @@ export declare function fingerprintEvaluationDataset(value: {
  * accepting a snapshot that can no longer prove which cases were run.
  */
 export declare function assertEvaluationDatasetSnapshot(snapshot: EvaluationDatasetSnapshot): void;
+/** Rejects malformed replay evidence at both write and durable-read boundaries. */
+export declare function assertEvaluationRecordingArtifact(value: unknown): asserts value is EvaluationRecordingArtifact;

package/dist/types/index.d.ts

@@ -8,3 +8,4 @@ export * from './datasetTransfer.js';
 export * from './suiteTransfer.js';
 export * from './dataTypes.js';
 export * from './normalization.js';
+export * from './library.js';

package/dist/types/library.d.ts

@@ -0,0 +1,8 @@
+import type { EvaluationLibrary } from './types.js';
+export declare function createEmptyEvaluationLibrary(): EvaluationLibrary;
+/**
+ * Repairs a library resource-by-resource so one malformed suite, baseline, or
+ * dataset cannot hide unrelated valid resources. The top-level persistence
+ * envelope is validated by the owning store before this function is called.
+ */
+export declare function normalizeEvaluationLibrary(value: unknown): EvaluationLibrary;

package/dist/types/normalization.d.ts

@@ -5,6 +5,21 @@ import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationRun, Ev
  * rule so persistence cannot disagree with what the workspace presents.
  */
 export declare function shouldReplaceEvaluationRun(existing: Pick<EvaluationRun, 'executionStatus' | 'revision'> | undefined, incoming: Pick<EvaluationRun, 'executionStatus' | 'revision'>): boolean;
+/**
+ * Execution snapshots never own a user-assigned run name. Preserve it when a
+ * runner publishes a newer progress or terminal revision after a rename.
+ */
+export declare function preserveEvaluationRunName(existing: Pick<EvaluationRun, 'name'> | undefined, incoming: EvaluationRun): EvaluationRun;
+/**
+ * Reconciles two snapshots of the same run without discarding evidence.
+ *
+ * Revisions remain authoritative. Within one revision, however, persistence
+ * and recording-retention updates can race with a terminal UI snapshot. Keep
+ * the snapshot with the fuller trial set, while still allowing a terminal
+ * snapshot to complete a progress snapshot and allowing recording references
+ * and warnings to enrich already-settled evidence.
+ */
+export declare function reconcileEvaluationRunSnapshots(existing: EvaluationRun | undefined, incoming: EvaluationRun): EvaluationRun;
 export declare function normalizeEvaluationTrial(value: EvaluationTrial | unknown, purpose?: EvaluationRunPurpose, evaluationMode?: EvaluationSuiteMode): EvaluationTrial;
 export declare function normalizeEvaluationAggregate(value: EvaluationAggregate | unknown, trials?: readonly EvaluationTrial[]): EvaluationAggregate;
 export declare function normalizeEvaluationBaselineSnapshot(value: EvaluationBaselineSnapshot | unknown): EvaluationBaselineSnapshot;

package/dist/types/runner.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationCaseAggregate, EvaluationDataset, EvaluationExecutionMetrics, EvaluationGraphRunner, EvaluationGraphEvaluator, EvaluationProjectData, EvaluationRun, EvaluationRunPurpose, EvaluationSuite, EvaluationSuiteMode, EvaluationThreshold, EvaluationThresholdResult } from './types.js';
+import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationCaseAggregate, EvaluationDataset, EvaluationExecutionMetrics, EvaluationGraphRunner, EvaluationGraphEvaluator, EvaluationProjectData, EvaluationRun, EvaluationRunEvent, EvaluationRunPurpose, EvaluationSuite, EvaluationSuiteMode, EvaluationThreshold, EvaluationThresholdResult, EvaluationTrial } from './types.js';
 import type { Project } from '@valerypopoff/rivet2-core';
 export declare const LEGACY_EVALUATOR_INPUT_IDS: readonly ["case", "inputs", "expected", "outputs", "run"];
 export type RunEvaluationSuiteOptions = {
@@ -15,6 +15,9 @@ export type RunEvaluationSuiteOptions = {
     purpose?: EvaluationRunPurpose;
     /** Explicit caller selection; otherwise the suite's saved baseline is used. */
     baseline?: EvaluationBaselineSnapshot;
+    /** Incremental live-run transport. Prefer this over onUpdate in new integrations. */
+    onEvent?: (event: EvaluationRunEvent) => void | Promise<void>;
+    /** @deprecated Prefer onEvent; this clones the complete accumulated run after every trial. */
     onUpdate?: (run: EvaluationRun) => void;
 };
 export declare function hasAuthoritativeEvaluationCriteria(suite: EvaluationSuite): boolean;
@@ -35,7 +38,7 @@ export type RunEvaluationWorkPoolOptions<TWork, TResult> = {
     concurrency: number;
     signal?: AbortSignal;
     execute: (work: TWork, index: number) => Promise<TResult>;
-    onSettled?: (result: TResult, index: number, results: readonly (TResult | undefined)[]) => void;
+    onSettled?: (result: TResult, index: number, results: readonly (TResult | undefined)[]) => void | Promise<void>;
 };
 export declare function runEvaluationWorkPool<TWork, TResult>(options: RunEvaluationWorkPoolOptions<TWork, TResult>): Promise<readonly (TResult | undefined)[]>;
 export declare function createEmptyMetrics(): EvaluationExecutionMetrics;
@@ -50,7 +53,60 @@ export declare function createEmptyMetrics(): EvaluationExecutionMetrics;
  * otherwise-successful recording in the suite.
  */
 export declare function finalizeEvaluationRecordingRetention(run: EvaluationRun, policy?: 'failures-and-baselines' | 'all', now?: number): EvaluationRun;
+export type RunEvaluationTrialOptions = {
+    project: Project;
+    suite: EvaluationSuite;
+    dataset: EvaluationDataset;
+    purpose: EvaluationRunPurpose;
+    testCase: EvaluationDataset['cases'][number];
+    caseIndex: number;
+    trialIndex: number;
+    runId: string;
+    runGraph: EvaluationGraphRunner;
+    signal?: AbortSignal;
+    /** Stable server-owned job id. Local callers may omit it. */
+    trialId?: string;
+};
+export declare function runEvaluationTrial(input: RunEvaluationTrialOptions): Promise<EvaluationTrial>;
 export declare function evaluateThresholdResults(aggregateValue: EvaluationAggregate, thresholds?: EvaluationThreshold[], baseline?: EvaluationBaselineSnapshot): EvaluationThresholdResult[];
+export type EvaluationRunWorkItem = {
+    testCase: EvaluationDataset['cases'][number];
+    caseIndex: number;
+    trialIndex: number;
+};
+export type EvaluationRunPlan = {
+    suite: EvaluationSuite;
+    purpose: EvaluationRunPurpose;
+    evaluationMode: EvaluationSuiteMode;
+    work: readonly EvaluationRunWorkItem[];
+};
+/**
+ * Validates an immutable Evaluation submission and expands its enabled cases
+ * into deterministic work. Hosts can persist this plan before any graph side
+ * effect, while local callers continue to use runEvaluationSuite.
+ */
+export declare function createEvaluationRunPlan(options: Pick<RunEvaluationSuiteOptions, 'project' | 'evaluationData' | 'dataset' | 'suiteId' | 'purpose'>): EvaluationRunPlan;
+export type FinalizeEvaluationRunOptions = {
+    run: EvaluationRun;
+    trials: readonly EvaluationTrial[];
+    suite: EvaluationSuite;
+    evaluationData: EvaluationProjectData;
+    baseline?: EvaluationBaselineSnapshot;
+    canceled?: boolean;
+    completedAt?: string;
+};
+/**
+ * Applies the same aggregate, quality, accounting, and baseline semantics to
+ * locally scheduled and durably hosted trials. Callers must provide every
+ * settled trial in deterministic case/trial order; never synthesize retries.
+ */
+export declare function finalizeEvaluationRun(options: FinalizeEvaluationRunOptions): EvaluationRun;
+export type CreateEvaluationRunShellOptions = Pick<RunEvaluationSuiteOptions, 'project' | 'evaluationData' | 'dataset' | 'suiteId' | 'purpose' | 'runId' | 'projectFingerprint' | 'executionMode'>;
+/** Creates a durable running shell only after the complete submission validates. */
+export declare function createEvaluationRunShell(options: CreateEvaluationRunShellOptions): {
+    run: EvaluationRun;
+    plan: EvaluationRunPlan;
+};
 export declare function runEvaluationSuite(options: RunEvaluationSuiteOptions): Promise<EvaluationRun>;
 export declare function runEvaluationCases(options: RunEvaluationSuiteOptions & {
     caseIds: readonly string[];