@valerypopoff/rivet2-evaluations 2.3.0

package/dist/esm/serialization.js

@@ -0,0 +1,277 @@
+import * as yaml from 'yaml';
+import { assertPortableJson } from './canonical.js';
+import { normalizeEvaluationBaselineSnapshot } from './normalization.js';
+function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function requireRecord(value, path) {
+    if (!isRecord(value))
+        throw new Error(`${path} must be an object.`);
+    return value;
+}
+function requireString(value, path) {
+    if (typeof value !== 'string' || value.trim().length === 0)
+        throw new Error(`${path} must be a non-empty string.`);
+}
+function requireText(value, path) {
+    if (typeof value !== 'string')
+        throw new Error(`${path} must be a string.`);
+}
+function requireBoolean(value, path) {
+    if (typeof value !== 'boolean')
+        throw new Error(`${path} must be a boolean.`);
+}
+function requireFiniteNumber(value, path) {
+    if (typeof value !== 'number' || !Number.isFinite(value))
+        throw new Error(`${path} must be a finite number.`);
+}
+function requireArray(value, path) {
+    if (!Array.isArray(value))
+        throw new Error(`${path} must be an array.`);
+    return value;
+}
+/**
+ * Selection is editor workspace state, not executable project data. Older
+ * project attachments can contain these fields, and previous editors could
+ * put an explicit `undefined` here, which is not portable JSON. They have no
+ * effect on project behavior, so discard both fields before validation.
+ */
+function stripLegacyEditorSelections(data) {
+    const { selectedSuiteId: _selectedSuiteId, selectedDatasetId: _selectedDatasetId, ...projectData } = data;
+    return projectData;
+}
+function validateOptionalPrimitive(record, key, type, path) {
+    const value = record[key];
+    if (value === undefined)
+        return;
+    if (type === 'number')
+        requireFiniteNumber(value, `${path}.${key}`);
+    else if (typeof value !== type)
+        throw new Error(`${path}.${key} must be a ${type}.`);
+}
+const ASSERTION_OPERATORS = new Set([
+    'equals',
+    'not-equals',
+    'contains',
+    'matches-regex',
+    'type-is',
+    'json-schema',
+    'number-at-least',
+    'number-at-most',
+    'number-between',
+    'array-includes',
+    'set-overlaps',
+    'contains-any',
+    'contains-all',
+]);
+const EVALUATOR_CONTEXT_INPUTS = new Set(['case', 'inputs', 'expected', 'outputs', 'run']);
+function validateSuiteShape(value, index) {
+    const path = `evaluations.suites[${index}]`;
+    const suite = requireRecord(value, path);
+    for (const key of ['id', 'targetGraphId', 'datasetId'])
+        requireString(suite[key], `${path}.${key}`);
+    requireText(suite.name, `${path}.name`);
+    if (suite.evaluationMode !== undefined &&
+        suite.evaluationMode !== 'pass-fail' &&
+        suite.evaluationMode !== 'scoring') {
+        throw new Error(`${path}.evaluationMode must be "pass-fail" or "scoring".`);
+    }
+    for (const [bindingIndex, bindingValue] of requireArray(suite.inputBindings, `${path}.inputBindings`).entries()) {
+        const binding = requireRecord(bindingValue, `${path}.inputBindings[${bindingIndex}]`);
+        requireString(binding.graphInputId, `${path}.inputBindings[${bindingIndex}].graphInputId`);
+        requireString(binding.datasetFieldId, `${path}.inputBindings[${bindingIndex}].datasetFieldId`);
+    }
+    for (const [assertionIndex, assertionValue] of requireArray(suite.assertions, `${path}.assertions`).entries()) {
+        const assertionPath = `${path}.assertions[${assertionIndex}]`;
+        const assertion = requireRecord(assertionValue, assertionPath);
+        for (const key of ['id', 'outputPath'])
+            requireString(assertion[key], `${assertionPath}.${key}`);
+        requireText(assertion.name, `${assertionPath}.name`);
+        if (typeof assertion.operator !== 'string' || !ASSERTION_OPERATORS.has(assertion.operator)) {
+            throw new Error(`${assertionPath}.operator is not supported.`);
+        }
+        validateOptionalPrimitive(assertion, 'required', 'boolean', assertionPath);
+        const expected = requireRecord(assertion.expected, `${assertionPath}.expected`);
+        if (expected.kind === 'literal') {
+            if (!Object.hasOwn(expected, 'value'))
+                throw new Error(`${assertionPath}.expected.value is required.`);
+        }
+        else if (expected.kind === 'dataset-field') {
+            requireString(expected.fieldId, `${assertionPath}.expected.fieldId`);
+        }
+        else {
+            throw new Error(`${assertionPath}.expected.kind must be "literal" or "dataset-field".`);
+        }
+    }
+    for (const [evaluatorIndex, evaluatorValue] of requireArray(suite.evaluators, `${path}.evaluators`).entries()) {
+        const evaluatorPath = `${path}.evaluators[${evaluatorIndex}]`;
+        const evaluator = requireRecord(evaluatorValue, evaluatorPath);
+        for (const key of ['id', 'graphId'])
+            requireString(evaluator[key], `${evaluatorPath}.${key}`);
+        requireText(evaluator.name, `${evaluatorPath}.name`);
+        validateOptionalPrimitive(evaluator, 'required', 'boolean', evaluatorPath);
+        validateOptionalPrimitive(evaluator, 'runOnTargetError', 'boolean', evaluatorPath);
+        validateOptionalPrimitive(evaluator, 'scoreWeight', 'number', evaluatorPath);
+        if (evaluator.inputBindings !== undefined) {
+            for (const [bindingIndex, bindingValue] of requireArray(evaluator.inputBindings, `${evaluatorPath}.inputBindings`).entries()) {
+                const bindingPath = `${evaluatorPath}.inputBindings[${bindingIndex}]`;
+                const binding = requireRecord(bindingValue, bindingPath);
+                requireString(binding.graphInputId, `${bindingPath}.graphInputId`);
+                const source = requireRecord(binding.source, `${bindingPath}.source`);
+                if (source.kind === 'dataset-field')
+                    requireString(source.fieldId, `${bindingPath}.source.fieldId`);
+                else if (source.kind === 'target-output')
+                    requireString(source.outputId, `${bindingPath}.source.outputId`);
+                else if (source.kind === 'context') {
+                    if (typeof source.context !== 'string' || !EVALUATOR_CONTEXT_INPUTS.has(source.context)) {
+                        throw new Error(`${bindingPath}.source.context is not supported.`);
+                    }
+                }
+                else {
+                    throw new Error(`${bindingPath}.source.kind is not supported.`);
+                }
+            }
+        }
+    }
+    if (suite.thresholds !== undefined) {
+        for (const [thresholdIndex, thresholdValue] of requireArray(suite.thresholds, `${path}.thresholds`).entries()) {
+            const thresholdPath = `${path}.thresholds[${thresholdIndex}]`;
+            const threshold = requireRecord(thresholdValue, thresholdPath);
+            for (const key of ['id', 'metric', 'operator'])
+                requireString(threshold[key], `${thresholdPath}.${key}`);
+            requireFiniteNumber(threshold.value, `${thresholdPath}.value`);
+        }
+    }
+    if (suite.configuration !== undefined) {
+        const configuration = requireRecord(suite.configuration, `${path}.configuration`);
+        for (const key of ['trialCount', 'concurrency', 'timeoutMs', 'seed']) {
+            validateOptionalPrimitive(configuration, key, 'number', `${path}.configuration`);
+        }
+        validateOptionalPrimitive(configuration, 'seedGraphInputId', 'string', `${path}.configuration`);
+        validateOptionalPrimitive(configuration, 'recordingRetention', 'string', `${path}.configuration`);
+    }
+    validateOptionalPrimitive(suite, 'description', 'string', path);
+    if (suite.tags !== undefined) {
+        for (const [tagIndex, tag] of requireArray(suite.tags, `${path}.tags`).entries()) {
+            requireString(tag, `${path}.tags[${tagIndex}]`);
+        }
+    }
+}
+function validateNumericRecord(value, path) {
+    const record = requireRecord(value, path);
+    for (const [key, entry] of Object.entries(record))
+        requireFiniteNumber(entry, `${path}.${key}`);
+}
+function validateBaselineShape(value, index) {
+    const path = `evaluations.baselines[${index}]`;
+    const baseline = requireRecord(value, path);
+    for (const key of ['id', 'suiteId', 'createdAt'])
+        requireString(baseline[key], `${path}.${key}`);
+    validateOptionalPrimitive(baseline, 'sourceRunId', 'string', path);
+    if (baseline.evaluationMode !== undefined &&
+        baseline.evaluationMode !== 'pass-fail' &&
+        baseline.evaluationMode !== 'scoring') {
+        throw new Error(`${path}.evaluationMode must be "pass-fail" or "scoring".`);
+    }
+    const provenance = requireRecord(baseline.provenance, `${path}.provenance`);
+    for (const key of [
+        'projectFingerprint',
+        'suiteFingerprint',
+        'datasetFingerprint',
+        'targetFingerprint',
+        'executionMode',
+    ]) {
+        requireString(provenance[key], `${path}.provenance.${key}`);
+    }
+    const evaluatorFingerprints = requireRecord(provenance.evaluatorFingerprints, `${path}.provenance.evaluatorFingerprints`);
+    for (const [key, fingerprint] of Object.entries(evaluatorFingerprints)) {
+        requireString(fingerprint, `${path}.provenance.evaluatorFingerprints.${key}`);
+    }
+    requireBoolean(provenance.accountingComplete, `${path}.provenance.accountingComplete`);
+    const aggregate = requireRecord(baseline.aggregate, `${path}.aggregate`);
+    for (const key of [
+        'trialCount',
+        'passedTrialCount',
+        'failedTrialCount',
+        'erroredTrialCount',
+        'canceledTrialCount',
+        'passRate',
+        'averageLatencyMs',
+        'p95LatencyMs',
+        'targetErrorRate',
+        'evaluatorErrorRate',
+        'toolFailureRate',
+    ]) {
+        requireFiniteNumber(aggregate[key], `${path}.aggregate.${key}`);
+    }
+    for (const key of [
+        'evaluatedTrialCount',
+        'notEvaluatedTrialCount',
+        'unableToEvaluateTrialCount',
+        'meanScore',
+        'totalCostUsd',
+        'averageCostUsd',
+        'scoredTrialCount',
+        'missingScoreTrialCount',
+    ]) {
+        validateOptionalPrimitive(aggregate, key, 'number', `${path}.aggregate`);
+    }
+    validateNumericRecord(aggregate.metrics, `${path}.aggregate.metrics`);
+    for (const [caseIndex, caseValue] of requireArray(baseline.cases, `${path}.cases`).entries()) {
+        const casePath = `${path}.cases[${caseIndex}]`;
+        const caseAggregate = requireRecord(caseValue, casePath);
+        requireString(caseAggregate.caseId, `${casePath}.caseId`);
+        requireString(caseAggregate.caseName, `${casePath}.caseName`);
+        validateNumericRecord(caseAggregate.metrics, `${casePath}.metrics`);
+        for (const key of [
+            'passRate',
+            'meanScore',
+            'evaluatedTrialCount',
+            'passedTrialCount',
+            'failedTrialCount',
+            'notEvaluatedTrialCount',
+            'unableToEvaluateTrialCount',
+            'erroredTrialCount',
+            'canceledTrialCount',
+            'scoredTrialCount',
+            'missingScoreTrialCount',
+        ]) {
+            validateOptionalPrimitive(caseAggregate, key, 'number', casePath);
+        }
+    }
+}
+export function createEmptyEvaluationProjectData() {
+    return { version: 1, suites: [], baselines: [] };
+}
+export function serializeEvaluationProjectData(data) {
+    if (data.version !== 1)
+        throw new Error(`Unsupported evaluation data version: ${data.version}`);
+    const persistentData = stripLegacyEditorSelections(data);
+    assertPortableJson(persistentData);
+    return {
+        ...structuredClone(persistentData),
+        baselines: persistentData.baselines.map(normalizeEvaluationBaselineSnapshot),
+    };
+}
+export function deserializeEvaluationProjectData(data) {
+    if (data === null || typeof data !== 'object' || Array.isArray(data))
+        throw new Error('Evaluation project data must be an object.');
+    const candidate = data;
+    if (candidate.version !== 1 || !Array.isArray(candidate.suites) || !Array.isArray(candidate.baselines)) {
+        throw new Error('Unsupported or invalid evaluation project data.');
+    }
+    const persistentData = stripLegacyEditorSelections(candidate);
+    assertPortableJson(persistentData);
+    persistentData.suites.forEach(validateSuiteShape);
+    persistentData.baselines.forEach(validateBaselineShape);
+    return {
+        ...structuredClone(persistentData),
+        baselines: persistentData.baselines.map(normalizeEvaluationBaselineSnapshot),
+    };
+}
+export function serializeEvaluationProjectDataToString(data) {
+    return yaml.stringify(serializeEvaluationProjectData(data), null, 2);
+}
+export function deserializeEvaluationProjectDataFromString(data) {
+    return deserializeEvaluationProjectData(yaml.parse(data));
+}

package/dist/esm/store.js

@@ -0,0 +1,88 @@
+import { assertEvaluationDatasetSnapshot } from './canonical.js';
+import { normalizeEvaluationRun, shouldReplaceEvaluationRun } from './normalization.js';
+export class InMemoryEvaluationRunStore {
+    #runs = new Map();
+    #datasetSnapshots = new Map();
+    #recordings = new Map();
+    async put(run) {
+        const normalized = normalizeEvaluationRun(run);
+        const key = `${normalized.projectId}/${normalized.id}`;
+        const existing = this.#runs.get(key);
+        if (!shouldReplaceEvaluationRun(existing, normalized))
+            return;
+        this.#runs.set(key, structuredClone(normalized));
+    }
+    async get(input) {
+        const run = this.#runs.get(`${input.projectId}/${input.runId}`);
+        return run ? normalizeEvaluationRun(run) : undefined;
+    }
+    async list(input) {
+        return Array.from(this.#runs.values())
+            .filter((run) => run.projectId === input.projectId && (input.suiteId === undefined || run.suiteId === input.suiteId))
+            .sort((left, right) => right.startedAt.localeCompare(left.startedAt))
+            .map(normalizeEvaluationRun);
+    }
+    async delete(input) {
+        this.#runs.delete(`${input.projectId}/${input.runId}`);
+        for (const [key, artifact] of this.#recordings) {
+            if (artifact.projectId === input.projectId && artifact.runId === input.runId)
+                this.#recordings.delete(key);
+        }
+    }
+    async putDatasetSnapshot(snapshot) {
+        assertEvaluationDatasetSnapshot(snapshot);
+        const key = `${snapshot.projectId}/${snapshot.fingerprint}`;
+        // A content-addressed historical snapshot must never be rewritten by a
+        // later edit of the live evaluation dataset.
+        if (this.#datasetSnapshots.has(key))
+            return;
+        this.#datasetSnapshots.set(key, structuredClone(snapshot));
+    }
+    async getDatasetSnapshot(input) {
+        const snapshot = this.#datasetSnapshots.get(`${input.projectId}/${input.fingerprint}`);
+        return snapshot ? structuredClone(snapshot) : undefined;
+    }
+    async putRecording(artifact) {
+        const key = `${artifact.projectId}/${artifact.reference.id}`;
+        const existing = this.#recordings.get(key);
+        if (existing && (existing.runId !== artifact.runId || existing.trialId !== artifact.trialId)) {
+            throw new Error('An evaluation recording ID cannot be reassigned to another run or trial.');
+        }
+        const next = structuredClone(artifact);
+        // Artifact persistence never owns retention transitions. A duplicate or
+        // delayed write keeps whatever explicit update/promotion already decided.
+        if (existing)
+            next.reference = structuredClone(existing.reference);
+        this.#recordings.set(key, next);
+    }
+    async getRecording(input) {
+        const artifact = this.#recordings.get(`${input.projectId}/${input.recordingId}`);
+        if (!artifact)
+            return undefined;
+        if (artifact.reference.retention === 'temporary' &&
+            artifact.reference.expiresAt &&
+            Date.parse(artifact.reference.expiresAt) <= Date.now()) {
+            this.#recordings.delete(`${input.projectId}/${input.recordingId}`);
+            return undefined;
+        }
+        return structuredClone(artifact);
+    }
+    async updateRecordingRetention(input) {
+        const key = `${input.projectId}/${input.recordingId}`;
+        const artifact = this.#recordings.get(key);
+        if (!artifact)
+            return;
+        artifact.reference = {
+            id: artifact.reference.id,
+            retention: input.retention,
+            ...(input.expiresAt === undefined ? {} : { expiresAt: input.expiresAt }),
+        };
+    }
+    async promoteBaseline(input) {
+        for (const artifact of this.#recordings.values()) {
+            if (artifact.projectId !== input.projectId || artifact.runId !== input.runId)
+                continue;
+            artifact.reference = { id: artifact.reference.id, retention: 'baseline' };
+        }
+    }
+}

package/dist/esm/suiteTransfer.js

@@ -0,0 +1,63 @@
+import { localizeEvaluationDataset, validateEvaluationDatasetForTransfer } from './datasetTransfer.js';
+import { deserializeEvaluationProjectData } from './serialization.js';
+const SUITE_BUNDLE_EXPORT_VERSION = 1;
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function requireNonEmptyString(value, name) {
+    if (value.trim().length === 0)
+        throw new Error(`${name} must be a non-empty string.`);
+}
+function validateEvaluationSuite(value) {
+    return deserializeEvaluationProjectData({ version: 1, suites: [value], baselines: [] }).suites[0];
+}
+function validateBundle(value) {
+    if (!isRecord(value) ||
+        value.version !== SUITE_BUNDLE_EXPORT_VERSION ||
+        !isRecord(value.suite) ||
+        !isRecord(value.dataset)) {
+        throw new Error(`Expected an evaluation suite export with version ${SUITE_BUNDLE_EXPORT_VERSION}.`);
+    }
+    const suite = validateEvaluationSuite(value.suite);
+    const dataset = validateEvaluationDatasetForTransfer(value.dataset);
+    if (suite.datasetId !== dataset.id) {
+        throw new Error('The evaluation suite export must include the dataset referenced by its suite.');
+    }
+    return { suite, dataset };
+}
+/**
+ * A portable suite bundle includes the suite definition and its referenced
+ * dataset. It intentionally excludes graphs, baselines, recordings, and run
+ * history, which belong to the destination project or its run store.
+ */
+export function serializeEvaluationSuiteBundleJson(suite, dataset) {
+    const bundle = validateBundle({ version: SUITE_BUNDLE_EXPORT_VERSION, suite, dataset });
+    return JSON.stringify({ version: SUITE_BUNDLE_EXPORT_VERSION, ...bundle }, null, 2);
+}
+/**
+ * Imports a bundle as new local evaluation resources. Field and case IDs remain stable
+ * so the imported suite's bindings and expected-value references still point
+ * to its imported dataset; suite and dataset identities are always supplied by
+ * the destination to avoid overwriting existing resources.
+ */
+export function deserializeEvaluationSuiteBundleJson(source, scope) {
+    let parsed;
+    try {
+        parsed = JSON.parse(source);
+    }
+    catch {
+        throw new Error('Evaluation suite JSON is not valid JSON.');
+    }
+    requireNonEmptyString(scope.suiteId, 'suiteId');
+    requireNonEmptyString(scope.datasetId, 'datasetId');
+    const bundle = validateBundle(parsed);
+    const dataset = localizeEvaluationDataset(bundle.dataset, scope.datasetId);
+    return {
+        dataset,
+        suite: {
+            ...bundle.suite,
+            id: scope.suiteId,
+            datasetId: dataset.id,
+        },
+    };
+}

package/dist/esm/types.js

@@ -0,0 +1,16 @@
+/**
+ * Lets a target adapter preserve its metrics and replay reference when a
+ * graph itself fails. The runner still classifies that as a target failure.
+ */
+export class EvaluationGraphExecutionError extends Error {
+    metrics;
+    recording;
+    providerAttempts;
+    constructor(message, details = {}) {
+        super(message);
+        this.name = 'EvaluationGraphExecutionError';
+        this.metrics = details.metrics;
+        this.recording = details.recording;
+        this.providerAttempts = details.providerAttempts;
+    }
+}

package/dist/types/assertions.d.ts

@@ -0,0 +1,17 @@
+import type { EvaluationAssertion, EvaluationDatasetCase, EvaluationObservation, PortableJson } from './types.js';
+/**
+ * Validates the exact JSON-path subset supported by evaluation assertions.
+ * UI authoring and runtime execution intentionally share this parser so a
+ * path cannot pass preflight and then resolve with different grammar.
+ */
+export declare function isEvaluationOutputPathSyntaxValid(path: string): boolean;
+/** The Graph Output that owns an assertion path, including nested paths. */
+export declare function getEvaluationTopLevelOutputId(path: string): string | undefined;
+/**
+ * Validates author-controlled expected data without needing target outputs.
+ * The runner uses this before scheduling work; assertion execution uses the
+ * same function so direct callers cannot observe a looser contract.
+ */
+export declare function validateEvaluationAssertionExpectedValue(assertion: EvaluationAssertion, expected: PortableJson): void;
+export declare function validateEvaluationAssertionConfiguration(assertion: EvaluationAssertion, expected: PortableJson): void;
+export declare function evaluateAssertion(assertion: EvaluationAssertion, outputs: Record<string, PortableJson>, testCase: EvaluationDatasetCase): EvaluationObservation;

package/dist/types/canonical.d.ts

@@ -0,0 +1,23 @@
+import type { EvaluationDatasetSnapshot, PortableJson } from './types.js';
+export declare function assertPortableJson(value: unknown, path?: string, stack?: Set<object>): asserts value is PortableJson;
+export declare function canonicalizePortableJson(value: unknown): PortableJson;
+/**
+ * Evaluation inputs and observations cross concurrent graph-run boundaries.
+ * Clone them before handing them to an adapter so a Code or evaluator graph
+ * cannot mutate a dataset case, another trial, or the persisted run model.
+ */
+export declare function clonePortableJson<T extends PortableJson>(value: T): T;
+export declare function canonicalStringify(value: unknown): string;
+/** Stable, synchronously available content fingerprint for project-local identities. */
+export declare function fingerprint(value: unknown): string;
+export declare function fingerprintEvaluationDataset(value: {
+    id: unknown;
+    fields: unknown;
+    cases: unknown;
+}): string;
+/**
+ * A dataset fingerprint is the link between a compact run summary and its
+ * historical input cases. Stores must reject a mismatched key rather than
+ * accepting a snapshot that can no longer prove which cases were run.
+ */
+export declare function assertEvaluationDatasetSnapshot(snapshot: EvaluationDatasetSnapshot): void;

package/dist/types/dataTypes.d.ts

@@ -0,0 +1,19 @@
+import type { EvaluationDataset, PortableJson } from './types.js';
+export declare const portableEvaluationDataTypes: readonly ["any", "any[]", "string", "string[]", "date", "date[]", "time", "time[]", "datetime", "datetime[]", "number", "number[]", "vector", "vector[]", "boolean", "boolean[]", "object", "object[]"];
+export declare function isPortableEvaluationDataType(dataType: string): boolean;
+/**
+ * Checks a portable dataset value against the Rivet data type selected for an
+ * evaluation field. This is shared by authoring UI and execution validation so
+ * a value cannot look valid in the editor and then fail under a different rule
+ * in the runner.
+ */
+export declare function isEvaluationValueCompatibleWithDataType(value: PortableJson, dataType: string): boolean;
+/** Shared declared-type check for graph/dataset binding authoring and execution validation. */
+export declare function areEvaluationDataTypesCompatible(sourceDataType: string, targetDataType: string): boolean;
+/**
+ * Enforces the declared portable JSON contract at explicit transfer
+ * boundaries. Durable local drafts remain recoverable even if an older build
+ * wrote an unsupported type; imports and exports fail with the precise field
+ * and case that needs repair instead of deferring the error until execution.
+ */
+export declare function assertEvaluationDatasetValuesMatchDeclaredTypes(dataset: EvaluationDataset): void;

package/dist/types/datasetTransfer.d.ts

@@ -0,0 +1,26 @@
+import type { EvaluationDataset } from './types.js';
+/**
+ * Validates durable evaluation datasets and portable imports. `projectId` is
+ * accepted solely for backwards-compatible migration from project-owned
+ * datasets; newly created local datasets omit it.
+ */
+export declare function validateEvaluationDataset(value: unknown, scope?: Pick<EvaluationDataset, 'id' | 'projectId'>): EvaluationDataset;
+/**
+ * Converts a legacy project-owned dataset into the application-local form.
+ * Field and case identity stay stable, so existing suite bindings continue to
+ * work after migration.
+ */
+export declare function localizeEvaluationDataset(value: unknown, id?: string): EvaluationDataset;
+/** Strict validation for explicit portable import and export boundaries. */
+export declare function validateEvaluationDatasetForTransfer(value: unknown, scope?: Pick<EvaluationDataset, 'id' | 'projectId'>): EvaluationDataset;
+/**
+ * Portable interchange envelope for a complete evaluation dataset. CSV is
+ * useful for case rows; JSON is the lossless form that also carries field
+ * roles and Rivet data types.
+ */
+export declare function serializeEvaluationDatasetJson(dataset: EvaluationDataset): string;
+/**
+ * Reads a dataset export into a destination dataset identity. Import keeps
+ * incoming field/case IDs but never lets a file overwrite another resource.
+ */
+export declare function deserializeEvaluationDatasetJson(source: string, scope: Pick<EvaluationDataset, 'id'>): EvaluationDataset;

package/dist/types/index.d.ts

@@ -0,0 +1,10 @@
+export * from './types.js';
+export * from './canonical.js';
+export * from './assertions.js';
+export * from './runner.js';
+export * from './store.js';
+export * from './serialization.js';
+export * from './datasetTransfer.js';
+export * from './suiteTransfer.js';
+export * from './dataTypes.js';
+export * from './normalization.js';

package/dist/types/normalization.d.ts

@@ -0,0 +1,11 @@
+import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationRun, EvaluationRunPurpose, EvaluationSuiteMode, EvaluationTrial } from './types.js';
+/**
+ * Orders snapshots for one run without allowing an equal-revision progress
+ * update to demote a terminal result. Writers and delayed UI reads share this
+ * rule so persistence cannot disagree with what the workspace presents.
+ */
+export declare function shouldReplaceEvaluationRun(existing: Pick<EvaluationRun, 'executionStatus' | 'revision'> | undefined, incoming: Pick<EvaluationRun, 'executionStatus' | 'revision'>): boolean;
+export declare function normalizeEvaluationTrial(value: EvaluationTrial | unknown, purpose?: EvaluationRunPurpose, evaluationMode?: EvaluationSuiteMode): EvaluationTrial;
+export declare function normalizeEvaluationAggregate(value: EvaluationAggregate | unknown, trials?: readonly EvaluationTrial[]): EvaluationAggregate;
+export declare function normalizeEvaluationBaselineSnapshot(value: EvaluationBaselineSnapshot | unknown): EvaluationBaselineSnapshot;
+export declare function normalizeEvaluationRun(value: EvaluationRun | unknown): EvaluationRun;

package/dist/types/runner.d.ts

@@ -0,0 +1,66 @@
+import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationCaseAggregate, EvaluationDataset, EvaluationExecutionMetrics, EvaluationGraphRunner, EvaluationGraphEvaluator, EvaluationProjectData, EvaluationRun, EvaluationRunPurpose, EvaluationSuite, EvaluationSuiteMode, EvaluationThreshold, EvaluationThresholdResult } from './types.js';
+import type { Project } from '@valerypopoff/rivet2-core';
+export declare const LEGACY_EVALUATOR_INPUT_IDS: readonly ["case", "inputs", "expected", "outputs", "run"];
+export type RunEvaluationSuiteOptions = {
+    project: Project;
+    evaluationData: EvaluationProjectData;
+    dataset: EvaluationDataset;
+    suiteId: string;
+    runGraph: EvaluationGraphRunner;
+    projectFingerprint?: string;
+    executionMode?: string;
+    signal?: AbortSignal;
+    runId?: string;
+    /** An evaluation judges quality; a benchmark deliberately measures execution only. */
+    purpose?: EvaluationRunPurpose;
+    /** Explicit caller selection; otherwise the suite's saved baseline is used. */
+    baseline?: EvaluationBaselineSnapshot;
+    onUpdate?: (run: EvaluationRun) => void;
+};
+export declare function hasAuthoritativeEvaluationCriteria(suite: EvaluationSuite): boolean;
+/** Older saved suites predate scoring and therefore retain pass/fail semantics. */
+export declare function getEvaluationSuiteMode(suite: Pick<EvaluationSuite, 'evaluationMode'>): EvaluationSuiteMode;
+/** Existing evaluators with the complete reserved-input contract keep working without migration. */
+export declare function usesLegacyEvaluatorInputEnvelope(evaluator: Pick<EvaluationGraphEvaluator, 'inputBindings'>, graphInputIds: readonly string[]): boolean;
+/**
+ * Shared bounded work scheduler for evaluation adapters that need stable
+ * result ordering without serializing independent cases. It deliberately has
+ * no retry policy: callers own their authoritative execution semantics.
+ *
+ * The result array is aligned to `work`; entries that were still queued when
+ * the signal was canceled stay `undefined`. Completed work is never erased.
+ */
+export type RunEvaluationWorkPoolOptions<TWork, TResult> = {
+    work: readonly TWork[];
+    concurrency: number;
+    signal?: AbortSignal;
+    execute: (work: TWork, index: number) => Promise<TResult>;
+    onSettled?: (result: TResult, index: number, results: readonly (TResult | undefined)[]) => void;
+};
+export declare function runEvaluationWorkPool<TWork, TResult>(options: RunEvaluationWorkPoolOptions<TWork, TResult>): Promise<readonly (TResult | undefined)[]>;
+export declare function createEmptyMetrics(): EvaluationExecutionMetrics;
+/**
+ * Resolves the provisional recording references created while a suite is
+ * running into its final retention policy. The caller persists the returned
+ * references through its EvaluationRunStore.
+ *
+ * A successful candidate recording remains promotable for 24 hours by
+ * default. Failed/error trials are retained, and `all` pins every artifact.
+ * This deliberately operates per trial: one failed case must not pin every
+ * otherwise-successful recording in the suite.
+ */
+export declare function finalizeEvaluationRecordingRetention(run: EvaluationRun, policy?: 'failures-and-baselines' | 'all', now?: number): EvaluationRun;
+export declare function evaluateThresholdResults(aggregateValue: EvaluationAggregate, thresholds?: EvaluationThreshold[], baseline?: EvaluationBaselineSnapshot): EvaluationThresholdResult[];
+export declare function runEvaluationSuite(options: RunEvaluationSuiteOptions): Promise<EvaluationRun>;
+export declare function runEvaluationCases(options: RunEvaluationSuiteOptions & {
+    caseIds: readonly string[];
+}): Promise<EvaluationRun>;
+export declare function summarizeEvaluationRun(run: EvaluationRun): {
+    aggregate: EvaluationAggregate;
+    cases: EvaluationCaseAggregate[];
+} | undefined;
+/**
+ * Baselines deliberately retain aggregate/provenance only. Raw prompts,
+ * outputs, credentials, and replay artifacts remain in the run store.
+ */
+export declare function createEvaluationBaselineSnapshot(run: EvaluationRun): EvaluationBaselineSnapshot;

package/dist/types/serialization.d.ts

@@ -0,0 +1,6 @@
+import type { EvaluationProjectData } from './types.js';
+export declare function createEmptyEvaluationProjectData(): EvaluationProjectData;
+export declare function serializeEvaluationProjectData(data: EvaluationProjectData): EvaluationProjectData;
+export declare function deserializeEvaluationProjectData(data: unknown): EvaluationProjectData;
+export declare function serializeEvaluationProjectDataToString(data: EvaluationProjectData): string;
+export declare function deserializeEvaluationProjectDataFromString(data: string): EvaluationProjectData;