@valerypopoff/rivet2-evaluations 2.3.0 → 2.4.0

package/dist/esm/canonical.js

@@ -100,10 +100,46 @@ export function fingerprintEvaluationDataset(value) {
  * accepting a snapshot that can no longer prove which cases were run.
  */
 export function assertEvaluationDatasetSnapshot(snapshot) {
-    if (snapshot.dataset.projectId !== undefined && snapshot.dataset.projectId !== snapshot.projectId) {
+    if (typeof snapshot.projectId !== 'string' ||
+        snapshot.projectId.length === 0 ||
+        typeof snapshot.fingerprint !== 'string' ||
+        snapshot.fingerprint.length === 0 ||
+        typeof snapshot.createdAt !== 'string' ||
+        !Number.isFinite(Date.parse(snapshot.createdAt))) {
+        throw new Error('An evaluation dataset snapshot has invalid identity or creation time.');
+    }
+    if (snapshot.dataset.projectId !== snapshot.projectId) {
         throw new Error('An evaluation dataset snapshot must belong to its project.');
     }
     if (fingerprintEvaluationDataset(snapshot.dataset) !== snapshot.fingerprint) {
         throw new Error('An evaluation dataset snapshot fingerprint must match its dataset ID, fields, and cases.');
     }
 }
+/** Rejects malformed replay evidence at both write and durable-read boundaries. */
+export function assertEvaluationRecordingArtifact(value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        throw new Error('An evaluation recording artifact must be an object.');
+    }
+    const artifact = value;
+    if (typeof artifact.projectId !== 'string' ||
+        artifact.projectId.length === 0 ||
+        typeof artifact.runId !== 'string' ||
+        artifact.runId.length === 0 ||
+        typeof artifact.trialId !== 'string' ||
+        artifact.trialId.length === 0 ||
+        typeof artifact.serialized !== 'string' ||
+        typeof artifact.createdAt !== 'string' ||
+        !Number.isFinite(Date.parse(artifact.createdAt))) {
+        throw new Error('An evaluation recording artifact has invalid identity, payload, or creation time.');
+    }
+    const reference = artifact.reference;
+    if (typeof reference !== 'object' ||
+        reference === null ||
+        typeof reference.id !== 'string' ||
+        reference.id.length === 0 ||
+        !['temporary', 'failure', 'baseline', 'retained'].includes(reference.retention ?? '') ||
+        (reference.expiresAt !== undefined &&
+            (typeof reference.expiresAt !== 'string' || !Number.isFinite(Date.parse(reference.expiresAt))))) {
+        throw new Error('An evaluation recording artifact has an invalid retention reference.');
+    }
+}

package/dist/esm/index.js

@@ -8,3 +8,4 @@ export * from './datasetTransfer.js';
 export * from './suiteTransfer.js';
 export * from './dataTypes.js';
 export * from './normalization.js';
+export * from './library.js';

package/dist/esm/library.js

@@ -0,0 +1,84 @@
+import { localizeEvaluationDataset } from './datasetTransfer.js';
+import { createEmptyEvaluationProjectData, deserializeEvaluationProjectData } from './serialization.js';
+export function createEmptyEvaluationLibrary() {
+    return {
+        version: 1,
+        data: createEmptyEvaluationProjectData(),
+        datasets: [],
+        migratedLegacyProjectIds: [],
+    };
+}
+/**
+ * Repairs a library resource-by-resource so one malformed suite, baseline, or
+ * dataset cannot hide unrelated valid resources. The top-level persistence
+ * envelope is validated by the owning store before this function is called.
+ */
+export function normalizeEvaluationLibrary(value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value))
+        return createEmptyEvaluationLibrary();
+    const candidate = value;
+    let data = createEmptyEvaluationProjectData();
+    try {
+        data = deserializeEvaluationProjectData(candidate.data);
+    }
+    catch {
+        const dataCandidate = typeof candidate.data === 'object' && candidate.data !== null && !Array.isArray(candidate.data)
+            ? candidate.data
+            : undefined;
+        const suites = Array.isArray(dataCandidate?.suites)
+            ? dataCandidate.suites.flatMap((suite) => {
+                try {
+                    return [deserializeEvaluationProjectData({ version: 1, suites: [suite], baselines: [] }).suites[0]];
+                }
+                catch {
+                    return [];
+                }
+            })
+            : [];
+        const baselines = Array.isArray(dataCandidate?.baselines)
+            ? dataCandidate.baselines.flatMap((baseline) => {
+                try {
+                    return [deserializeEvaluationProjectData({ version: 1, suites: [], baselines: [baseline] }).baselines[0]];
+                }
+                catch {
+                    return [];
+                }
+            })
+            : [];
+        const suiteIds = new Set(suites.map((suite) => suite.id));
+        data = { version: 1, suites, baselines: baselines.filter((baseline) => suiteIds.has(baseline.suiteId)) };
+    }
+    const datasets = Array.isArray(candidate.datasets)
+        ? candidate.datasets.flatMap((dataset) => {
+            try {
+                return [localizeEvaluationDataset(dataset)];
+            }
+            catch {
+                return [];
+            }
+        })
+        : [];
+    const suites = uniqueById(data.suites);
+    const suiteIds = new Set(suites.map((suite) => suite.id));
+    return {
+        version: 1,
+        data: {
+            ...data,
+            suites,
+            baselines: uniqueById(data.baselines).filter((baseline) => suiteIds.has(baseline.suiteId)),
+        },
+        datasets: uniqueById(datasets),
+        migratedLegacyProjectIds: Array.isArray(candidate.migratedLegacyProjectIds)
+            ? Array.from(new Set(candidate.migratedLegacyProjectIds.filter((id) => typeof id === 'string')))
+            : [],
+    };
+}
+function uniqueById(values) {
+    const seen = new Set();
+    return values.filter((value) => {
+        if (seen.has(value.id))
+            return false;
+        seen.add(value.id);
+        return true;
+    });
+}

package/dist/esm/normalization.js

@@ -19,6 +19,14 @@ function isRunExecutionStatus(value) {
 function isTerminalRunExecutionStatus(status) {
     return status === 'completed' || status === 'canceled' || status === 'error';
 }
+function reconcileRecordingReference(primary, secondary) {
+    if (!primary)
+        return secondary;
+    if (!secondary)
+        return primary;
+    const rank = (retention) => retention === 'temporary' ? 0 : retention === 'failure' ? 1 : 2;
+    return rank(secondary.retention) > rank(primary.retention) ? secondary : primary;
+}
 /**
  * Orders snapshots for one run without allowing an equal-revision progress
  * update to demote a terminal result. Writers and delayed UI reads share this
@@ -31,7 +39,80 @@ export function shouldReplaceEvaluationRun(existing, incoming) {
     const incomingRevision = incoming.revision ?? 0;
     if (incomingRevision !== existingRevision)
         return incomingRevision > existingRevision;
-    return !isTerminalRunExecutionStatus(existing.executionStatus) || isTerminalRunExecutionStatus(incoming.executionStatus);
+    return (!isTerminalRunExecutionStatus(existing.executionStatus) || isTerminalRunExecutionStatus(incoming.executionStatus));
+}
+/**
+ * Execution snapshots never own a user-assigned run name. Preserve it when a
+ * runner publishes a newer progress or terminal revision after a rename.
+ */
+export function preserveEvaluationRunName(existing, incoming) {
+    return existing?.name !== undefined && incoming.name === undefined ? { ...incoming, name: existing.name } : incoming;
+}
+/**
+ * Reconciles two snapshots of the same run without discarding evidence.
+ *
+ * Revisions remain authoritative. Within one revision, however, persistence
+ * and recording-retention updates can race with a terminal UI snapshot. Keep
+ * the snapshot with the fuller trial set, while still allowing a terminal
+ * snapshot to complete a progress snapshot and allowing recording references
+ * and warnings to enrich already-settled evidence.
+ */
+export function reconcileEvaluationRunSnapshots(existing, incoming) {
+    if (!existing)
+        return incoming;
+    const existingRevision = existing.revision ?? 0;
+    const incomingRevision = incoming.revision ?? 0;
+    if (incomingRevision < existingRevision)
+        return existing;
+    if (incomingRevision > existingRevision)
+        return preserveEvaluationRunName(existing, incoming);
+    if (!shouldReplaceEvaluationRun(existing, incoming))
+        return existing;
+    const existingTerminal = isTerminalRunExecutionStatus(existing.executionStatus);
+    const incomingTerminal = isTerminalRunExecutionStatus(incoming.executionStatus);
+    const existingTrials = existing.trials ?? [];
+    const incomingTrials = incoming.trials ?? [];
+    const incomingIsPrimary = incomingTerminal !== existingTerminal ? incomingTerminal : incomingTrials.length >= existingTrials.length;
+    const primary = preserveEvaluationRunName(existing, incomingIsPrimary ? incoming : existing);
+    const secondary = incomingIsPrimary ? existing : incoming;
+    const primaryTrials = primary.trials ?? [];
+    const secondaryTrialList = secondary.trials ?? [];
+    const name = existing.name ?? incoming.name;
+    const primaryTrialIds = new Set(primaryTrials.map((trial) => trial.id));
+    const secondaryTrials = new Map(secondaryTrialList.map((trial) => [trial.id, trial]));
+    return {
+        ...primary,
+        trials: [
+            ...primaryTrials.map((trial) => {
+                const other = secondaryTrials.get(trial.id);
+                if (!other)
+                    return trial;
+                const secondaryObservations = new Map((other.observations ?? []).map((observation) => [observation.id, observation]));
+                const primaryObservationIds = new Set((trial.observations ?? []).map((observation) => observation.id));
+                const recording = reconcileRecordingReference(trial.recording, other.recording);
+                return {
+                    ...trial,
+                    ...(recording === undefined ? {} : { recording }),
+                    observations: [
+                        ...(trial.observations ?? []).map((observation) => {
+                            const otherObservation = secondaryObservations.get(observation.id);
+                            const observationRecording = reconcileRecordingReference(observation.recording, otherObservation?.recording);
+                            return observationRecording === observation.recording
+                                ? observation
+                                : {
+                                    ...observation,
+                                    ...(observationRecording === undefined ? {} : { recording: observationRecording }),
+                                };
+                        }),
+                        ...(other.observations ?? []).filter((observation) => !primaryObservationIds.has(observation.id)),
+                    ],
+                };
+            }),
+            ...secondaryTrialList.filter((trial) => !primaryTrialIds.has(trial.id)),
+        ],
+        ...(name === undefined ? {} : { name }),
+        warnings: [...new Set([...(primary.warnings ?? []), ...(secondary.warnings ?? [])])],
+    };
 }
 function isPurpose(value) {
     return value === 'evaluation' || value === 'execution-benchmark';
@@ -184,6 +265,15 @@ export function normalizeEvaluationAggregate(value, trials) {
     // as the persisted normalized score and render as an impossible 9900/100.
     if (!isNormalizedScore(normalized.meanScore))
         delete normalized.meanScore;
+    if (!isNormalizedScore(normalized.medianScore))
+        delete normalized.medianScore;
+    if (!isNormalizedScore(normalized.p95Score))
+        delete normalized.p95Score;
+    if (typeof normalized.medianLatencyMs !== 'number' ||
+        !Number.isFinite(normalized.medianLatencyMs) ||
+        normalized.medianLatencyMs < 0) {
+        delete normalized.medianLatencyMs;
+    }
     if (trials) {
         const completed = trials.filter((trial) => trial.executionStatus === 'completed');
         const evaluated = completed.filter((trial) => trial.qualityStatus === 'passed' || trial.qualityStatus === 'failed');
@@ -347,6 +437,14 @@ export function normalizeEvaluationRun(value) {
     // As with legacy trial.status, the old verdict is input-only compatibility
     // data. New v2 values expose one unambiguous qualityStatus instead.
     delete normalized.verdict;
+    const name = typeof value.name === 'string' && value.name.trim().length > 0 ? value.name.trim() : undefined;
+    delete normalized.name;
+    const requestedTrialCount = typeof value.requestedTrialCount === 'number' &&
+        Number.isSafeInteger(value.requestedTrialCount) &&
+        value.requestedTrialCount >= 0
+        ? value.requestedTrialCount
+        : undefined;
+    delete normalized.requestedTrialCount;
     const aggregate = value.aggregate === undefined ? undefined : normalizeEvaluationAggregate(value.aggregate, trials);
     return {
         ...normalized,
@@ -362,6 +460,8 @@ export function normalizeEvaluationRun(value) {
         thresholdResults: Array.isArray(value.thresholdResults)
             ? structuredClone(value.thresholdResults)
             : [],
+        ...(name === undefined ? {} : { name }),
+        ...(requestedTrialCount === undefined ? {} : { requestedTrialCount }),
         ...(aggregate === undefined
             ? {}
             : { aggregate: accountingStatus === 'partial' ? withoutAuthoritativeCost(aggregate) : aggregate }),