@valerypopoff/rivet2-evaluations 2.3.0 → 2.5.0

package/dist/types/runner.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationCaseAggregate, EvaluationDataset, EvaluationExecutionMetrics, EvaluationGraphRunner, EvaluationGraphEvaluator, EvaluationProjectData, EvaluationRun, EvaluationRunPurpose, EvaluationSuite, EvaluationSuiteMode, EvaluationThreshold, EvaluationThresholdResult } from './types.js';
+import type { EvaluationAggregate, EvaluationBaselineSnapshot, EvaluationCaseAggregate, EvaluationDataset, EvaluationExecutionMetrics, EvaluationGraphRunner, EvaluationGraphEvaluator, EvaluationProjectData, EvaluationRun, EvaluationRunEvent, EvaluationRunPurpose, EvaluationSuite, EvaluationSuiteMode, EvaluationThreshold, EvaluationThresholdResult, EvaluationTrial } from './types.js';
 import type { Project } from '@valerypopoff/rivet2-core';
 export declare const LEGACY_EVALUATOR_INPUT_IDS: readonly ["case", "inputs", "expected", "outputs", "run"];
 export type RunEvaluationSuiteOptions = {
@@ -15,6 +15,9 @@ export type RunEvaluationSuiteOptions = {
     purpose?: EvaluationRunPurpose;
     /** Explicit caller selection; otherwise the suite's saved baseline is used. */
     baseline?: EvaluationBaselineSnapshot;
+    /** Incremental live-run transport. Prefer this over onUpdate in new integrations. */
+    onEvent?: (event: EvaluationRunEvent) => void | Promise<void>;
+    /** @deprecated Prefer onEvent; this clones the complete accumulated run after every trial. */
     onUpdate?: (run: EvaluationRun) => void;
 };
 export declare function hasAuthoritativeEvaluationCriteria(suite: EvaluationSuite): boolean;
@@ -35,7 +38,7 @@ export type RunEvaluationWorkPoolOptions<TWork, TResult> = {
     concurrency: number;
     signal?: AbortSignal;
     execute: (work: TWork, index: number) => Promise<TResult>;
-    onSettled?: (result: TResult, index: number, results: readonly (TResult | undefined)[]) => void;
+    onSettled?: (result: TResult, index: number, results: readonly (TResult | undefined)[]) => void | Promise<void>;
 };
 export declare function runEvaluationWorkPool<TWork, TResult>(options: RunEvaluationWorkPoolOptions<TWork, TResult>): Promise<readonly (TResult | undefined)[]>;
 export declare function createEmptyMetrics(): EvaluationExecutionMetrics;
@@ -50,7 +53,60 @@ export declare function createEmptyMetrics(): EvaluationExecutionMetrics;
  * otherwise-successful recording in the suite.
  */
 export declare function finalizeEvaluationRecordingRetention(run: EvaluationRun, policy?: 'failures-and-baselines' | 'all', now?: number): EvaluationRun;
+export type RunEvaluationTrialOptions = {
+    project: Project;
+    suite: EvaluationSuite;
+    dataset: EvaluationDataset;
+    purpose: EvaluationRunPurpose;
+    testCase: EvaluationDataset['cases'][number];
+    caseIndex: number;
+    trialIndex: number;
+    runId: string;
+    runGraph: EvaluationGraphRunner;
+    signal?: AbortSignal;
+    /** Stable server-owned job id. Local callers may omit it. */
+    trialId?: string;
+};
+export declare function runEvaluationTrial(input: RunEvaluationTrialOptions): Promise<EvaluationTrial>;
 export declare function evaluateThresholdResults(aggregateValue: EvaluationAggregate, thresholds?: EvaluationThreshold[], baseline?: EvaluationBaselineSnapshot): EvaluationThresholdResult[];
+export type EvaluationRunWorkItem = {
+    testCase: EvaluationDataset['cases'][number];
+    caseIndex: number;
+    trialIndex: number;
+};
+export type EvaluationRunPlan = {
+    suite: EvaluationSuite;
+    purpose: EvaluationRunPurpose;
+    evaluationMode: EvaluationSuiteMode;
+    work: readonly EvaluationRunWorkItem[];
+};
+/**
+ * Validates an immutable Evaluation submission and expands its enabled cases
+ * into deterministic work. Hosts can persist this plan before any graph side
+ * effect, while local callers continue to use runEvaluationSuite.
+ */
+export declare function createEvaluationRunPlan(options: Pick<RunEvaluationSuiteOptions, 'project' | 'evaluationData' | 'dataset' | 'suiteId' | 'purpose'>): EvaluationRunPlan;
+export type FinalizeEvaluationRunOptions = {
+    run: EvaluationRun;
+    trials: readonly EvaluationTrial[];
+    suite: EvaluationSuite;
+    evaluationData: EvaluationProjectData;
+    baseline?: EvaluationBaselineSnapshot;
+    canceled?: boolean;
+    completedAt?: string;
+};
+/**
+ * Applies the same aggregate, quality, accounting, and baseline semantics to
+ * locally scheduled and durably hosted trials. Callers must provide every
+ * settled trial in deterministic case/trial order; never synthesize retries.
+ */
+export declare function finalizeEvaluationRun(options: FinalizeEvaluationRunOptions): EvaluationRun;
+export type CreateEvaluationRunShellOptions = Pick<RunEvaluationSuiteOptions, 'project' | 'evaluationData' | 'dataset' | 'suiteId' | 'purpose' | 'runId' | 'projectFingerprint' | 'executionMode'>;
+/** Creates a durable running shell only after the complete submission validates. */
+export declare function createEvaluationRunShell(options: CreateEvaluationRunShellOptions): {
+    run: EvaluationRun;
+    plan: EvaluationRunPlan;
+};
 export declare function runEvaluationSuite(options: RunEvaluationSuiteOptions): Promise<EvaluationRun>;
 export declare function runEvaluationCases(options: RunEvaluationSuiteOptions & {
     caseIds: readonly string[];

package/dist/types/store.d.ts

@@ -1,7 +1,14 @@
-import type { EvaluationDatasetSnapshot, EvaluationRecordingArtifact, EvaluationRun, EvaluationRunStore } from './types.js';
-export declare class InMemoryEvaluationRunStore implements EvaluationRunStore {
+import type { EvaluationDatasetSnapshot, EvaluationLibrary, EvaluationRecordingArtifact, EvaluationRun, EvaluationStore } from './types.js';
+export declare class InMemoryEvaluationRunStore implements EvaluationStore {
     #private;
+    getLibrary(): Promise<EvaluationLibrary>;
+    putLibrary(library: EvaluationLibrary): Promise<void>;
     put(run: EvaluationRun): Promise<void>;
+    updateRunName(input: {
+        projectId: string;
+        runId: string;
+        name?: string;
+    }): Promise<EvaluationRun | undefined>;
     get(input: {
         projectId: string;
         runId: string;
@@ -29,7 +36,7 @@ export declare class InMemoryEvaluationRunStore implements EvaluationRunStore {
         recordingId: string;
         retention: EvaluationRecordingArtifact['reference']['retention'];
         expiresAt?: string;
-    }): Promise<void>;
+    }): Promise<boolean>;
     promoteBaseline(input: {
         projectId: string;
         runId: string;

package/dist/types/types.d.ts

@@ -27,7 +27,7 @@ export type EvaluationDataset = {
      * Legacy project ownership retained only while importing older project
      * files. Locally persisted evaluation datasets intentionally omit it so
      * they can be reused with any currently open project.
-    */
+     */
     projectId?: ProjectId;
     name: string;
     fields: EvaluationDatasetField[];
@@ -158,6 +158,125 @@ export type EvaluationProjectData = {
     suites: EvaluationSuite[];
     baselines: EvaluationBaselineSnapshot[];
 };
+/**
+ * Rivet-local evaluation resources. This is deliberately outside project
+ * files so one Rivet instance can retain its evaluation library independently
+ * of whichever project is currently open.
+ */
+export type EvaluationLibrary = {
+    version: 1;
+    data: EvaluationProjectData;
+    datasets: EvaluationDataset[];
+    /** Legacy project IDs whose embedded evaluation resources were imported. */
+    migratedLegacyProjectIds: ProjectId[];
+};
+/**
+ * A collaboration boundary for the shared evaluation library. A suite owns
+ * its baselines because a baseline is only meaningful with that suite; a
+ * dataset is independently editable and reusable by many suites.
+ */
+export type EvaluationLibraryResourceKind = 'suite' | 'dataset';
+export type EvaluationLibraryResourceVersions = {
+    suites: Record<string, string>;
+    datasets: Record<string, string>;
+};
+/** A server-authoritative library snapshot used by collaborative hosts. */
+export type EvaluationLibrarySyncSnapshot = {
+    revision: number;
+    library: EvaluationLibrary;
+    resourceVersions: EvaluationLibraryResourceVersions;
+};
+/**
+ * A validated, external change token for the shared evaluation library.
+ * `epoch` identifies one SSE generation so reconnects can require a resync
+ * even when a server's revision counter has not advanced.
+ */
+export type EvaluationLibraryInvalidation = {
+    epoch: string;
+    revision: number;
+};
+/** The durable value for one independently synchronized library resource. */
+export type EvaluationLibrarySuiteBundle = {
+    suite: EvaluationSuite;
+    baselines: EvaluationBaselineSnapshot[];
+};
+export type EvaluationLibraryResourceValue = {
+    kind: 'suite';
+    id: string;
+    value: EvaluationLibrarySuiteBundle | undefined;
+} | {
+    kind: 'dataset';
+    id: string;
+    value: EvaluationDataset | undefined;
+};
+/**
+ * Every mutation is an all-or-nothing resource batch. Existing resources use
+ * their opaque version token; null means that creation requires the resource
+ * to remain absent.
+ */
+export type EvaluationLibraryMutationChange = {
+    kind: 'put-suite';
+    id: string;
+    expectedVersion: string | null;
+    suite: EvaluationSuite;
+    baselines: EvaluationBaselineSnapshot[];
+} | {
+    kind: 'delete-suite';
+    id: string;
+    expectedVersion: string;
+} | {
+    kind: 'put-dataset';
+    id: string;
+    expectedVersion: string | null;
+    dataset: EvaluationDataset;
+} | {
+    kind: 'delete-dataset';
+    id: string;
+    expectedVersion: string;
+};
+export type EvaluationLibraryMutation = {
+    changes: EvaluationLibraryMutationChange[];
+};
+export type EvaluationLibraryConflictResource = {
+    kind: EvaluationLibraryResourceKind;
+    id: string;
+    expectedVersion: string | null;
+    currentVersion: string | null;
+};
+/**
+ * A conflict is deliberately expressed with both values. Hosts use this to
+ * present an explicit choice instead of silently overwriting either editor.
+ */
+export type EvaluationLibraryConflictDraft = EvaluationLibraryConflictResource & {
+    local: EvaluationLibraryResourceValue;
+    server: EvaluationLibraryResourceValue;
+};
+export type EvaluationLibrarySyncIssue = {
+    id: string;
+    kind: 'conflict';
+    message: string;
+    conflicts: readonly EvaluationLibraryConflictDraft[];
+} | {
+    id: string;
+    kind: 'retryable' | 'failed';
+    message: string;
+};
+export type EvaluationLibraryConflictResolution = {
+    issueId: string;
+    kind: EvaluationLibraryResourceKind;
+    id: string;
+    action: 'use-server' | 'keep-mine-as-copy';
+};
+/** A host can expose this optional capability without changing local stores. */
+export type EvaluationLibrarySyncStore = {
+    getLibrarySyncSnapshot(): Promise<EvaluationLibrarySyncSnapshot>;
+    mutateLibrary(input: EvaluationLibraryMutation): Promise<EvaluationLibrarySyncSnapshot>;
+    subscribeLibraryInvalidation?(listener: (invalidation: EvaluationLibraryInvalidation) => void): () => void;
+    /** An unresolved hosted save requiring an explicit user decision. */
+    subscribeLibrarySyncIssue?(listener: (issue: EvaluationLibrarySyncIssue | undefined) => void): () => void;
+    resolveLibraryConflict?(input: EvaluationLibraryConflictResolution): Promise<EvaluationLibrary>;
+    retryLibrarySync?(): Promise<EvaluationLibrary>;
+};
 export type EvaluationObservationStatus = 'passed' | 'failed' | 'scored' | 'error' | 'skipped';
 export type EvaluationObservation = {
     id: string;
@@ -284,7 +403,13 @@ export type EvaluationAggregate = {
     /** Normalized internal score in the closed 0..1 range. */
     meanScore?: number;
     averageLatencyMs: number;
+    /** Optional so historical persisted runs without this newer factoid remain readable. */
+    medianLatencyMs?: number;
     p95LatencyMs: number;
+    /** Score distribution statistics use equal-weight per-case mean scores. */
+    medianScore?: number;
+    /** Score distribution statistics use equal-weight per-case mean scores. */
+    p95Score?: number;
     totalCostUsd?: number;
     averageCostUsd?: number;
     targetErrorRate: number;
@@ -307,6 +432,8 @@ export type EvaluationRun = {
     projectId: ProjectId;
     suiteId: string;
     suiteName: string;
+    /** Optional user-assigned label for distinguishing retained runs. */
+    name?: string;
     /**
      * Monotonically increases while a run is persisted. Stores use it to reject
      * a delayed live-progress write after a newer completed snapshot.
@@ -318,6 +445,8 @@ export type EvaluationRun = {
     purpose: EvaluationRunPurpose;
     /** Captures the suite semantics that produced this run. */
     evaluationMode?: EvaluationSuiteMode;
+    /** Planned target executions, retained so live progress does not depend on a later suite edit. */
+    requestedTrialCount?: number;
     executionStatus: 'queued' | 'running' | 'completed' | 'canceled' | 'error';
     qualityStatus: EvaluationQualityStatus;
     qualityReason: EvaluationQualityReason;
@@ -329,6 +458,31 @@ export type EvaluationRun = {
     trials: EvaluationTrial[];
     warnings: string[];
 };
+/**
+ * Incremental live-run protocol. A started event carries only the empty run
+ * shell, a settled event carries one immutable trial, and the terminal event
+ * carries the complete reporting shape once. Consumers that still need full
+ * snapshots can use RunEvaluationSuiteOptions.onUpdate during the compatibility
+ * period.
+ */
+export type EvaluationRunEvent = {
+    type: 'run-started';
+    revision: number;
+    run: EvaluationRun;
+} | {
+    type: 'trial-settled';
+    revision: number;
+    runId: string;
+    projectId: ProjectId;
+    suiteId: string;
+    requestedTrialCount: number;
+    settledTrialCount: number;
+    trial: EvaluationTrial;
+} | {
+    type: 'run-finalized';
+    revision: number;
+    run: EvaluationRun;
+};
 export type EvaluationRecordingReference = {
     id: string;
     retention: 'temporary' | 'failure' | 'baseline' | 'retained';
@@ -386,6 +540,12 @@ export type EvaluationGraphRunner = (input: {
 }) => Promise<EvaluationGraphExecution>;
 export type EvaluationRunStore = {
     put(run: EvaluationRun): Promise<void>;
+    /** Sets a user-assigned run name; omit the name to restore the Unnamed label. */
+    updateRunName(input: {
+        projectId: ProjectId;
+        runId: string;
+        name?: string;
+    }): Promise<EvaluationRun | undefined>;
     get(input: {
         projectId: ProjectId;
         runId: string;
@@ -417,12 +577,38 @@ export type EvaluationRunStore = {
         recordingId: string;
         retention: EvaluationRecordingReference['retention'];
         expiresAt?: string;
-    }): Promise<void>;
+    }): Promise<boolean>;
     /** Pins all artifacts belonging to the run before its compact baseline is saved in the project. */
     promoteBaseline(input: {
         projectId: ProjectId;
         runId: string;
     }): Promise<void>;
+    /**
+     * Optional V2 capability. Persists an idempotent incremental lifecycle event
+     * so a host can recover settled work after a process interruption.
+     */
+    applyRunEvent?(event: EvaluationRunEvent): Promise<void>;
+};
+export type EvaluationStoreInitialization = {
+    /** A recoverable persistence warning that a host may surface to the user. */
+    warning?: string;
+};
+/**
+ * Complete persistence boundary for Evaluations. Desktop, browser, and hosted
+ * Rivet instances implement this same contract so definitions and operational
+ * evidence cannot silently land in different stores.
+ */
+export type EvaluationStore = EvaluationRunStore & {
+    initialize?(): Promise<EvaluationStoreInitialization | void>;
+    getLibrary(): Promise<EvaluationLibrary>;
+    putLibrary(library: EvaluationLibrary): Promise<void>;
+    /** Optional hosted capability for resource-scoped collaborative saves. */
+    getLibrarySyncSnapshot?: EvaluationLibrarySyncStore['getLibrarySyncSnapshot'];
+    mutateLibrary?: EvaluationLibrarySyncStore['mutateLibrary'];
+    subscribeLibraryInvalidation?: EvaluationLibrarySyncStore['subscribeLibraryInvalidation'];
+    subscribeLibrarySyncIssue?: EvaluationLibrarySyncStore['subscribeLibrarySyncIssue'];
+    resolveLibraryConflict?: EvaluationLibrarySyncStore['resolveLibraryConflict'];
+    retryLibrarySync?: EvaluationLibrarySyncStore['retryLibrarySync'];
 };
 export type EvaluationReporter = {
     report(run: EvaluationRun): Promise<void> | void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@valerypopoff/rivet2-evaluations",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "license": "MIT",
   "main": "dist/cjs/bundle.cjs",
   "module": "dist/esm/index.js",
@@ -36,7 +36,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@valerypopoff/rivet2-core": "^2.3.0",
+    "@valerypopoff/rivet2-core": "^2.5.0",
     "yaml": "^2.9.0"
   }
 }