@valerypopoff/rivet2-evaluations 2.3.0

package/dist/esm/dataTypes.js

@@ -0,0 +1,103 @@
+import { assertPortableJson } from './canonical.js';
+export const portableEvaluationDataTypes = [
+    'any',
+    'any[]',
+    'string',
+    'string[]',
+    'date',
+    'date[]',
+    'time',
+    'time[]',
+    'datetime',
+    'datetime[]',
+    'number',
+    'number[]',
+    'vector',
+    'vector[]',
+    'boolean',
+    'boolean[]',
+    'object',
+    'object[]',
+];
+const portableEvaluationDataTypeSet = new Set(portableEvaluationDataTypes);
+export function isPortableEvaluationDataType(dataType) {
+    return portableEvaluationDataTypeSet.has(dataType);
+}
+function isObject(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function isFiniteNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value);
+}
+function isArrayOf(value, predicate) {
+    return Array.isArray(value) && value.every(predicate);
+}
+/**
+ * Checks a portable dataset value against the Rivet data type selected for an
+ * evaluation field. This is shared by authoring UI and execution validation so
+ * a value cannot look valid in the editor and then fail under a different rule
+ * in the runner.
+ */
+export function isEvaluationValueCompatibleWithDataType(value, dataType) {
+    try {
+        assertPortableJson(value);
+    }
+    catch {
+        return false;
+    }
+    if (dataType === 'any')
+        return true;
+    if (dataType === 'any[]')
+        return Array.isArray(value);
+    if (dataType === 'string' || dataType === 'date' || dataType === 'time' || dataType === 'datetime') {
+        return typeof value === 'string';
+    }
+    if (dataType === 'string[]' || dataType === 'date[]' || dataType === 'time[]' || dataType === 'datetime[]') {
+        return isArrayOf(value, (item) => typeof item === 'string');
+    }
+    if (dataType === 'number')
+        return isFiniteNumber(value);
+    if (dataType === 'number[]' || dataType === 'vector')
+        return isArrayOf(value, isFiniteNumber);
+    if (dataType === 'vector[]') {
+        return isArrayOf(value, (item) => isArrayOf(item, isFiniteNumber));
+    }
+    if (dataType === 'boolean')
+        return typeof value === 'boolean';
+    if (dataType === 'boolean[]')
+        return isArrayOf(value, (item) => typeof item === 'boolean');
+    if (dataType === 'object')
+        return isObject(value);
+    if (dataType === 'object[]')
+        return isArrayOf(value, isObject);
+    // Functions, binary/media values, and provider-specific structured types
+    // do not have a safe portable dataset representation here. Unknown future
+    // types must also fail closed until their exact JSON contract is defined.
+    return false;
+}
+/** Shared declared-type check for graph/dataset binding authoring and execution validation. */
+export function areEvaluationDataTypesCompatible(sourceDataType, targetDataType) {
+    return sourceDataType === 'any' || targetDataType === 'any' || sourceDataType === targetDataType;
+}
+/**
+ * Enforces the declared portable JSON contract at explicit transfer
+ * boundaries. Durable local drafts remain recoverable even if an older build
+ * wrote an unsupported type; imports and exports fail with the precise field
+ * and case that needs repair instead of deferring the error until execution.
+ */
+export function assertEvaluationDatasetValuesMatchDeclaredTypes(dataset) {
+    const fieldsById = new Map(dataset.fields.map((field) => [field.id, field]));
+    for (const [fieldIndex, field] of dataset.fields.entries()) {
+        if (!isPortableEvaluationDataType(field.dataType)) {
+            throw new Error(`fields[${fieldIndex}].dataType "${field.dataType}" has no portable evaluation JSON representation.`);
+        }
+    }
+    for (const [caseIndex, testCase] of dataset.cases.entries()) {
+        for (const [fieldId, value] of Object.entries(testCase.values)) {
+            const field = fieldsById.get(fieldId);
+            if (field && !isEvaluationValueCompatibleWithDataType(value, field.dataType)) {
+                throw new Error(`cases[${caseIndex}].values.${fieldId} is not compatible with declared type "${field.dataType}".`);
+            }
+        }
+    }
+}

package/dist/esm/datasetTransfer.js

@@ -0,0 +1,146 @@
+import { assertPortableJson, fingerprintEvaluationDataset } from './canonical.js';
+import { assertEvaluationDatasetValuesMatchDeclaredTypes } from './dataTypes.js';
+const DATASET_EXPORT_VERSION = 1;
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function requireString(value, path) {
+    if (typeof value !== 'string' || value.length === 0)
+        throw new Error(`${path} must be a non-empty string.`);
+    return value;
+}
+function requireText(value, path) {
+    if (typeof value !== 'string')
+        throw new Error(`${path} must be a string.`);
+    return value;
+}
+function parseFields(value) {
+    if (!Array.isArray(value))
+        throw new Error('Evaluation dataset fields must be an array.');
+    const ids = new Set();
+    return value.map((field, index) => {
+        if (!isRecord(field))
+            throw new Error(`fields[${index}] must be an object.`);
+        const id = requireString(field.id, `fields[${index}].id`);
+        if (ids.has(id))
+            throw new Error(`Evaluation dataset has duplicate field id "${id}".`);
+        ids.add(id);
+        const role = field.role;
+        if (role !== 'input' && role !== 'expected' && role !== 'metadata') {
+            throw new Error(`fields[${index}].role must be input, expected, or metadata.`);
+        }
+        if (field.description !== undefined && typeof field.description !== 'string') {
+            throw new Error(`fields[${index}].description must be a string.`);
+        }
+        if (field.required !== undefined && typeof field.required !== 'boolean') {
+            throw new Error(`fields[${index}].required must be a boolean.`);
+        }
+        return {
+            id,
+            name: requireText(field.name, `fields[${index}].name`),
+            dataType: requireString(field.dataType, `fields[${index}].dataType`),
+            role,
+            ...(typeof field.description === 'string' ? { description: field.description } : {}),
+            ...(typeof field.required === 'boolean' ? { required: field.required } : {}),
+        };
+    });
+}
+function parseCases(value, fieldIds) {
+    if (!Array.isArray(value))
+        throw new Error('Evaluation dataset cases must be an array.');
+    const ids = new Set();
+    return value.map((testCase, index) => {
+        if (!isRecord(testCase))
+            throw new Error(`cases[${index}] must be an object.`);
+        const id = requireString(testCase.id, `cases[${index}].id`);
+        if (ids.has(id))
+            throw new Error(`Evaluation dataset has duplicate case id "${id}".`);
+        ids.add(id);
+        if (!isRecord(testCase.values))
+            throw new Error(`cases[${index}].values must be an object.`);
+        for (const [fieldId, fieldValue] of Object.entries(testCase.values)) {
+            if (!fieldIds.has(fieldId))
+                throw new Error(`cases[${index}].values references unknown field "${fieldId}".`);
+            assertPortableJson(fieldValue, `cases[${index}].values.${fieldId}`);
+        }
+        if (testCase.tags !== undefined && (!Array.isArray(testCase.tags) || testCase.tags.some((tag) => typeof tag !== 'string'))) {
+            throw new Error(`cases[${index}].tags must be an array of strings.`);
+        }
+        if (testCase.note !== undefined && typeof testCase.note !== 'string')
+            throw new Error(`cases[${index}].note must be a string.`);
+        if (testCase.enabled !== undefined && typeof testCase.enabled !== 'boolean')
+            throw new Error(`cases[${index}].enabled must be a boolean.`);
+        return {
+            id,
+            name: requireText(testCase.name, `cases[${index}].name`),
+            values: testCase.values,
+            ...(typeof testCase.enabled === 'boolean' ? { enabled: testCase.enabled } : {}),
+            ...(Array.isArray(testCase.tags) ? { tags: testCase.tags } : {}),
+            ...(typeof testCase.note === 'string' ? { note: testCase.note } : {}),
+        };
+    });
+}
+/**
+ * Validates durable evaluation datasets and portable imports. `projectId` is
+ * accepted solely for backwards-compatible migration from project-owned
+ * datasets; newly created local datasets omit it.
+ */
+export function validateEvaluationDataset(value, scope) {
+    if (!isRecord(value))
+        throw new Error('Evaluation dataset must be an object.');
+    const fields = parseFields(value.fields);
+    const importedProjectId = value.projectId === undefined ? undefined : requireString(value.projectId, 'dataset.projectId');
+    const projectId = scope?.projectId ?? importedProjectId;
+    const dataset = {
+        id: scope?.id ?? requireString(value.id, 'dataset.id'),
+        name: requireText(value.name, 'dataset.name'),
+        ...(projectId === undefined ? {} : { projectId }),
+        fields,
+        cases: parseCases(value.cases, new Set(fields.map((field) => field.id))),
+    };
+    return { ...dataset, contentFingerprint: fingerprintEvaluationDataset(dataset) };
+}
+/**
+ * Converts a legacy project-owned dataset into the application-local form.
+ * Field and case identity stay stable, so existing suite bindings continue to
+ * work after migration.
+ */
+export function localizeEvaluationDataset(value, id) {
+    const dataset = validateEvaluationDataset(value, id === undefined ? undefined : { id });
+    const { projectId: _legacyProjectId, ...localDataset } = dataset;
+    return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
+}
+/** Strict validation for explicit portable import and export boundaries. */
+export function validateEvaluationDatasetForTransfer(value, scope) {
+    const dataset = validateEvaluationDataset(value, scope);
+    assertEvaluationDatasetValuesMatchDeclaredTypes(dataset);
+    return dataset;
+}
+/**
+ * Portable interchange envelope for a complete evaluation dataset. CSV is
+ * useful for case rows; JSON is the lossless form that also carries field
+ * roles and Rivet data types.
+ */
+export function serializeEvaluationDatasetJson(dataset) {
+    const validated = validateEvaluationDatasetForTransfer(dataset);
+    return JSON.stringify({ version: DATASET_EXPORT_VERSION, dataset: validated }, null, 2);
+}
+/**
+ * Reads a dataset export into a destination dataset identity. Import keeps
+ * incoming field/case IDs but never lets a file overwrite another resource.
+ */
+export function deserializeEvaluationDatasetJson(source, scope) {
+    let parsed;
+    try {
+        parsed = JSON.parse(source);
+    }
+    catch {
+        throw new Error('Evaluation dataset JSON is not valid JSON.');
+    }
+    if (!isRecord(parsed) || parsed.version !== DATASET_EXPORT_VERSION || !isRecord(parsed.dataset)) {
+        throw new Error(`Expected an evaluation dataset export with version ${DATASET_EXPORT_VERSION}.`);
+    }
+    const dataset = validateEvaluationDatasetForTransfer(parsed.dataset, { id: scope.id });
+    const { projectId: _legacyProjectId, ...localDataset } = dataset;
+    return { ...localDataset, contentFingerprint: fingerprintEvaluationDataset(localDataset) };
+}

package/dist/esm/index.js

@@ -0,0 +1,10 @@
+export * from './types.js';
+export * from './canonical.js';
+export * from './assertions.js';
+export * from './runner.js';
+export * from './store.js';
+export * from './serialization.js';
+export * from './datasetTransfer.js';
+export * from './suiteTransfer.js';
+export * from './dataTypes.js';
+export * from './normalization.js';

package/dist/esm/normalization.js

@@ -0,0 +1,369 @@
+function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function isQualityStatus(value) {
+    return ['passed', 'failed', 'scored', 'not-evaluated', 'unable-to-evaluate'].includes(String(value));
+}
+function isEvaluationMode(value) {
+    return value === 'pass-fail' || value === 'scoring';
+}
+function isQualityStatusForMode(status, evaluationMode) {
+    return evaluationMode === 'scoring' ? status !== 'passed' && status !== 'failed' : status !== 'scored';
+}
+function isExecutionStatus(value) {
+    return ['completed', 'error', 'canceled'].includes(String(value));
+}
+function isRunExecutionStatus(value) {
+    return ['queued', 'running', 'completed', 'canceled', 'error'].includes(String(value));
+}
+function isTerminalRunExecutionStatus(status) {
+    return status === 'completed' || status === 'canceled' || status === 'error';
+}
+/**
+ * Orders snapshots for one run without allowing an equal-revision progress
+ * update to demote a terminal result. Writers and delayed UI reads share this
+ * rule so persistence cannot disagree with what the workspace presents.
+ */
+export function shouldReplaceEvaluationRun(existing, incoming) {
+    if (!existing)
+        return true;
+    const existingRevision = existing.revision ?? 0;
+    const incomingRevision = incoming.revision ?? 0;
+    if (incomingRevision !== existingRevision)
+        return incomingRevision > existingRevision;
+    return !isTerminalRunExecutionStatus(existing.executionStatus) || isTerminalRunExecutionStatus(incoming.executionStatus);
+}
+function isPurpose(value) {
+    return value === 'evaluation' || value === 'execution-benchmark';
+}
+function isAccountingStatus(value) {
+    return value === 'complete' || value === 'partial';
+}
+/** Run records keep scores normalized even though evaluator graphs use 0..100. */
+function isNormalizedScore(value) {
+    return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+function reasonForQualityStatus(status, executionStatus) {
+    if (executionStatus === 'canceled')
+        return { code: 'canceled', message: 'The execution was canceled.' };
+    if (executionStatus === 'error' && status === 'failed') {
+        return { code: 'target-error', message: 'The target graph failed before it could produce a valid result.' };
+    }
+    switch (status) {
+        case 'passed':
+            return { code: 'checks-passed', message: 'All required quality criteria passed.' };
+        case 'failed':
+            return { code: 'checks-failed', message: 'One or more required quality criteria failed.' };
+        case 'scored':
+            return { code: 'scores-complete', message: 'Every requested trial produced a score.' };
+        case 'not-evaluated':
+            return { code: 'no-trial-quality-checks', message: 'No per-trial quality check evaluated this result.' };
+        case 'unable-to-evaluate':
+            return { code: 'required-check-error', message: 'A required quality criterion could not be evaluated.' };
+    }
+}
+function reasonForQualityStatusInMode(status, evaluationMode, executionStatus) {
+    if (evaluationMode === 'scoring' && status === 'unable-to-evaluate') {
+        return { code: 'scores-incomplete', message: 'One or more requested trials did not produce a usable score.' };
+    }
+    return reasonForQualityStatus(status, executionStatus);
+}
+function isTargetExecutionError(value, observations) {
+    if (typeof value.error === 'string' && value.error.length > 0)
+        return true;
+    return observations.some((observation) => isRecord(observation) &&
+        observation.id === 'target-error' &&
+        observation.kind === 'assertion' &&
+        observation.status === 'error');
+}
+function legacyQualityFromObservations(observations, targetExecutionError) {
+    if (targetExecutionError)
+        return 'failed';
+    const required = observations.filter((observation) => isRecord(observation) && observation.required !== false);
+    if (required.some((observation) => observation.status === 'failed'))
+        return 'failed';
+    if (required.some((observation) => observation.status === 'error'))
+        return 'unable-to-evaluate';
+    if (required.some((observation) => observation.status === 'passed'))
+        return 'passed';
+    return 'not-evaluated';
+}
+/**
+ * Scoring runs are new enough that they do not need to preserve a legacy
+ * pass/fail quality label. Do still recover a useful status when a partial or
+ * hand-edited scoring record lacks its derived status, but never let an old
+ * `passed`/`failed` value leak into the scoring UI.
+ */
+function scoringQualityFromObservations(observations) {
+    const evaluatorObservations = observations.filter((observation) => isRecord(observation) && observation.kind === 'graph');
+    const hasCompleteScore = evaluatorObservations.length > 0 &&
+        evaluatorObservations.every((observation) => observation.status === 'scored' &&
+            typeof observation.score === 'number' &&
+            Number.isFinite(observation.score) &&
+            observation.score >= 0 &&
+            observation.score <= 1);
+    return hasCompleteScore ? 'scored' : 'unable-to-evaluate';
+}
+function isQualityReasonCompatible(reason, status, purpose, executionStatus) {
+    if (purpose === 'execution-benchmark') {
+        return executionStatus === 'canceled' ? reason.code === 'canceled' : reason.code === 'benchmark';
+    }
+    if (executionStatus === 'queued' || executionStatus === 'running')
+        return reason.code === 'in-progress';
+    if (executionStatus === 'canceled')
+        return reason.code === 'canceled';
+    switch (status) {
+        case 'passed':
+            return reason.code === 'checks-passed' || reason.code === 'thresholds-passed';
+        case 'failed':
+            return reason.code === 'checks-failed' || reason.code === 'thresholds-failed' || reason.code === 'target-error';
+        case 'scored':
+            return reason.code === 'scores-complete';
+        case 'not-evaluated':
+            return reason.code === 'no-trial-quality-checks' || reason.code === 'no-completed-trials';
+        case 'unable-to-evaluate':
+            return (reason.code === 'required-check-error' ||
+                reason.code === 'required-metric-unavailable' ||
+                reason.code === 'scores-incomplete');
+    }
+}
+function normalizeQualityReason(value, fallback, status, purpose, executionStatus) {
+    if (!isRecord(value) || typeof value.code !== 'string' || typeof value.message !== 'string')
+        return fallback;
+    const reason = value;
+    return isQualityReasonCompatible(reason, status, purpose, executionStatus) ? reason : fallback;
+}
+export function normalizeEvaluationTrial(value, purpose = 'evaluation', evaluationMode = 'pass-fail') {
+    if (!isRecord(value))
+        throw new Error('Evaluation trial must be an object.');
+    const legacyStatus = value.status;
+    const observations = Array.isArray(value.observations) ? value.observations : [];
+    const targetExecutionError = isTargetExecutionError(value, observations);
+    const executionStatus = isExecutionStatus(value.executionStatus)
+        ? value.executionStatus
+        : legacyStatus === 'canceled'
+            ? 'canceled'
+            : targetExecutionError
+                ? 'error'
+                : legacyStatus === 'error' &&
+                    !observations.some((observation) => isRecord(observation) && observation.required !== false && observation.status === 'error')
+                    ? 'error'
+                    : 'completed';
+    const qualityStatus = purpose === 'execution-benchmark' || executionStatus === 'canceled'
+        ? 'not-evaluated'
+        : executionStatus === 'error'
+            ? evaluationMode === 'scoring'
+                ? 'unable-to-evaluate'
+                : 'failed'
+            : evaluationMode === 'scoring'
+                ? scoringQualityFromObservations(observations)
+                : isQualityStatus(value.qualityStatus) && isQualityStatusForMode(value.qualityStatus, evaluationMode)
+                    ? value.qualityStatus
+                    : legacyQualityFromObservations(observations, targetExecutionError);
+    const normalized = structuredClone(value);
+    // Legacy runs stored a single status that conflated execution and quality.
+    // Read it above, then remove it so normalized v2 values cannot contradict
+    // their authoritative executionStatus/qualityStatus pair.
+    delete normalized.status;
+    return {
+        ...normalized,
+        executionStatus,
+        qualityStatus,
+        qualityReason: normalizeQualityReason(value.qualityReason, executionStatus === 'canceled'
+            ? { code: 'canceled', message: 'The execution was canceled.' }
+            : purpose === 'execution-benchmark'
+                ? { code: 'benchmark', message: 'This run measured execution without evaluating output quality.' }
+                : reasonForQualityStatusInMode(qualityStatus, evaluationMode, executionStatus), qualityStatus, purpose, executionStatus),
+    };
+}
+export function normalizeEvaluationAggregate(value, trials) {
+    if (!isRecord(value))
+        throw new Error('Evaluation aggregate must be an object.');
+    const normalized = structuredClone(value);
+    // Never let a graph-facing 0..100 value (or corrupt local data) masquerade
+    // as the persisted normalized score and render as an impossible 9900/100.
+    if (!isNormalizedScore(normalized.meanScore))
+        delete normalized.meanScore;
+    if (trials) {
+        const completed = trials.filter((trial) => trial.executionStatus === 'completed');
+        const evaluated = completed.filter((trial) => trial.qualityStatus === 'passed' || trial.qualityStatus === 'failed');
+        const passed = evaluated.filter((trial) => trial.qualityStatus === 'passed');
+        const failed = evaluated.filter((trial) => trial.qualityStatus === 'failed');
+        const notEvaluated = completed.filter((trial) => trial.qualityStatus === 'not-evaluated');
+        const unable = completed.filter((trial) => trial.qualityStatus === 'unable-to-evaluate');
+        const scored = completed.filter((trial) => trial.qualityStatus === 'scored');
+        return {
+            ...normalized,
+            trialCount: trials.length,
+            evaluatedTrialCount: evaluated.length,
+            notEvaluatedTrialCount: notEvaluated.length,
+            unableToEvaluateTrialCount: unable.length,
+            passedTrialCount: passed.length,
+            failedTrialCount: failed.length,
+            erroredTrialCount: trials.filter((trial) => trial.executionStatus === 'error').length,
+            canceledTrialCount: trials.filter((trial) => trial.executionStatus === 'canceled').length,
+            ...(scored.length > 0 || normalized.scoredTrialCount !== undefined
+                ? { scoredTrialCount: scored.length, missingScoreTrialCount: trials.length - scored.length }
+                : {}),
+            passRate: evaluated.length === 0 ? 0 : passed.length / evaluated.length,
+        };
+    }
+    const trialCount = Number(value.trialCount ?? 0);
+    const passed = Number(value.passedTrialCount ?? 0);
+    const failed = Number(value.failedTrialCount ?? 0);
+    const errored = Number(value.erroredTrialCount ?? 0);
+    const canceled = Number(value.canceledTrialCount ?? 0);
+    const evaluated = Number(value.evaluatedTrialCount ?? passed + failed);
+    const unable = Number(value.unableToEvaluateTrialCount ?? 0);
+    const notEvaluated = Number(value.notEvaluatedTrialCount ?? Math.max(0, trialCount - evaluated - unable - errored - canceled));
+    return {
+        ...normalized,
+        evaluatedTrialCount: evaluated,
+        notEvaluatedTrialCount: notEvaluated,
+        unableToEvaluateTrialCount: unable,
+        passRate: evaluated === 0 ? 0 : passed / evaluated,
+    };
+}
+function withoutAuthoritativeCost(aggregate) {
+    const normalized = { ...aggregate };
+    delete normalized.totalCostUsd;
+    delete normalized.averageCostUsd;
+    return normalized;
+}
+export function normalizeEvaluationBaselineSnapshot(value) {
+    if (!isRecord(value))
+        throw new Error('Evaluation baseline snapshot must be an object.');
+    const accountingStatus = isAccountingStatus(value.accountingStatus)
+        ? value.accountingStatus
+        : isRecord(value.provenance) && value.provenance.accountingComplete === false
+            ? 'partial'
+            : 'complete';
+    const aggregate = normalizeEvaluationAggregate(value.aggregate);
+    const purpose = isPurpose(value.purpose) ? value.purpose : 'evaluation';
+    const evaluationMode = isEvaluationMode(value.evaluationMode) ? value.evaluationMode : 'pass-fail';
+    const hasCompleteScoringAggregate = aggregate.trialCount > 0 &&
+        aggregate.scoredTrialCount === aggregate.trialCount &&
+        aggregate.missingScoreTrialCount === 0 &&
+        aggregate.meanScore !== undefined;
+    const qualityStatus = purpose === 'execution-benchmark'
+        ? 'not-evaluated'
+        : evaluationMode === 'scoring'
+            ? hasCompleteScoringAggregate
+                ? 'scored'
+                : 'unable-to-evaluate'
+            : isQualityStatus(value.qualityStatus) && isQualityStatusForMode(value.qualityStatus, evaluationMode)
+                ? value.qualityStatus
+                : aggregate.failedTrialCount > 0 || aggregate.erroredTrialCount > 0
+                    ? 'failed'
+                    : aggregate.evaluatedTrialCount > 0
+                        ? 'passed'
+                        : 'not-evaluated';
+    const qualityReasonFallback = purpose === 'execution-benchmark'
+        ? { code: 'benchmark', message: 'This run measured execution without evaluating output quality.' }
+        : reasonForQualityStatusInMode(qualityStatus, evaluationMode);
+    return {
+        ...structuredClone(value),
+        purpose,
+        evaluationMode,
+        qualityStatus,
+        qualityReason: normalizeQualityReason(value.qualityReason, qualityReasonFallback, qualityStatus, purpose),
+        accountingStatus,
+        aggregate: accountingStatus === 'partial' ? withoutAuthoritativeCost(aggregate) : aggregate,
+    };
+}
+export function normalizeEvaluationRun(value) {
+    if (!isRecord(value))
+        throw new Error('Evaluation run must be an object.');
+    const purpose = isPurpose(value.purpose) ? value.purpose : 'evaluation';
+    const evaluationMode = isEvaluationMode(value.evaluationMode) ? value.evaluationMode : 'pass-fail';
+    const executionStatus = isRunExecutionStatus(value.executionStatus)
+        ? value.executionStatus
+        : typeof value.completedAt === 'string'
+            ? 'completed'
+            : 'running';
+    const trials = Array.isArray(value.trials)
+        ? value.trials.map((trial) => normalizeEvaluationTrial(trial, purpose, evaluationMode))
+        : [];
+    const accountingStatus = isAccountingStatus(value.accountingStatus)
+        ? value.accountingStatus
+        : isRecord(value.provenance) && value.provenance.accountingComplete === false
+            ? 'partial'
+            : 'complete';
+    let qualityStatus;
+    if (purpose === 'execution-benchmark' || executionStatus === 'canceled') {
+        qualityStatus = 'not-evaluated';
+    }
+    else if (executionStatus === 'error') {
+        qualityStatus = evaluationMode === 'scoring' ? 'unable-to-evaluate' : 'failed';
+    }
+    else if (executionStatus === 'queued' || executionStatus === 'running') {
+        qualityStatus = 'not-evaluated';
+    }
+    else if (evaluationMode === 'scoring') {
+        const scored = trials.filter((trial) => trial.qualityStatus === 'scored').length;
+        qualityStatus = scored > 0 && scored === trials.length ? 'scored' : 'unable-to-evaluate';
+    }
+    else if (isQualityStatus(value.qualityStatus) && isQualityStatusForMode(value.qualityStatus, evaluationMode)) {
+        qualityStatus = value.qualityStatus;
+    }
+    else if (trials.some((trial) => trial.executionStatus === 'error' || trial.qualityStatus === 'failed')) {
+        qualityStatus = 'failed';
+    }
+    else if (trials.some((trial) => trial.qualityStatus === 'unable-to-evaluate')) {
+        qualityStatus = 'unable-to-evaluate';
+    }
+    else if (trials.some((trial) => trial.qualityStatus === 'passed')) {
+        qualityStatus = 'passed';
+    }
+    else if (value.verdict === 'fail') {
+        // Old runs could fail an aggregate threshold without having a per-trial
+        // quality observation. Preserve that authoritative failure. Only legacy
+        // success-without-evidence is reinterpreted as Not evaluated.
+        qualityStatus = 'failed';
+    }
+    else {
+        // Legacy runs used `pass` for a graph that merely completed. If no stored
+        // required check proves quality, reinterpret it as Not evaluated instead
+        // of preserving the misleading pass label.
+        qualityStatus = 'not-evaluated';
+    }
+    const qualityReasonFallback = executionStatus === 'canceled'
+        ? { code: 'canceled', message: 'The evaluation run was canceled.' }
+        : executionStatus === 'queued' || executionStatus === 'running'
+            ? { code: 'in-progress', message: 'The run is still in progress.' }
+            : purpose === 'execution-benchmark'
+                ? { code: 'benchmark', message: 'This run measured execution without evaluating output quality.' }
+                : (executionStatus === 'error' || trials.some((trial) => trial.executionStatus === 'error')) &&
+                    qualityStatus === 'failed'
+                    ? { code: 'target-error', message: 'One or more target graph executions failed.' }
+                    : reasonForQualityStatusInMode(qualityStatus, evaluationMode);
+    const provenance = isRecord(value.provenance)
+        ? {
+            ...structuredClone(value.provenance),
+            accountingComplete: accountingStatus === 'complete',
+        }
+        : value.provenance;
+    const normalized = structuredClone(value);
+    // As with legacy trial.status, the old verdict is input-only compatibility
+    // data. New v2 values expose one unambiguous qualityStatus instead.
+    delete normalized.verdict;
+    const aggregate = value.aggregate === undefined ? undefined : normalizeEvaluationAggregate(value.aggregate, trials);
+    return {
+        ...normalized,
+        version: 2,
+        purpose,
+        evaluationMode,
+        executionStatus,
+        qualityStatus,
+        qualityReason: normalizeQualityReason(value.qualityReason, qualityReasonFallback, qualityStatus, purpose, executionStatus),
+        accountingStatus,
+        provenance: provenance,
+        trials,
+        thresholdResults: Array.isArray(value.thresholdResults)
+            ? structuredClone(value.thresholdResults)
+            : [],
+        ...(aggregate === undefined
+            ? {}
+            : { aggregate: accountingStatus === 'partial' ? withoutAuthoritativeCost(aggregate) : aggregate }),
+    };
+}