@valerypopoff/rivet2-evaluations 2.3.0 → 2.4.0

package/dist/types/store.d.ts

@@ -1,7 +1,14 @@
-import type { EvaluationDatasetSnapshot, EvaluationRecordingArtifact, EvaluationRun, EvaluationRunStore } from './types.js';
-export declare class InMemoryEvaluationRunStore implements EvaluationRunStore {
+import type { EvaluationDatasetSnapshot, EvaluationLibrary, EvaluationRecordingArtifact, EvaluationRun, EvaluationStore } from './types.js';
+export declare class InMemoryEvaluationRunStore implements EvaluationStore {
     #private;
+    getLibrary(): Promise<EvaluationLibrary>;
+    putLibrary(library: EvaluationLibrary): Promise<void>;
     put(run: EvaluationRun): Promise<void>;
+    updateRunName(input: {
+        projectId: string;
+        runId: string;
+        name?: string;
+    }): Promise<EvaluationRun | undefined>;
     get(input: {
         projectId: string;
         runId: string;
@@ -29,7 +36,7 @@ export declare class InMemoryEvaluationRunStore implements EvaluationRunStore {
         recordingId: string;
         retention: EvaluationRecordingArtifact['reference']['retention'];
         expiresAt?: string;
-    }): Promise<void>;
+    }): Promise<boolean>;
     promoteBaseline(input: {
         projectId: string;
         runId: string;

package/dist/types/types.d.ts

@@ -27,7 +27,7 @@ export type EvaluationDataset = {
      * Legacy project ownership retained only while importing older project
      * files. Locally persisted evaluation datasets intentionally omit it so
      * they can be reused with any currently open project.
-    */
+     */
     projectId?: ProjectId;
     name: string;
     fields: EvaluationDatasetField[];
@@ -158,6 +158,18 @@ export type EvaluationProjectData = {
     suites: EvaluationSuite[];
     baselines: EvaluationBaselineSnapshot[];
 };
+/**
+ * Rivet-local evaluation resources. This is deliberately outside project
+ * files so one Rivet instance can retain its evaluation library independently
+ * of whichever project is currently open.
+ */
+export type EvaluationLibrary = {
+    version: 1;
+    data: EvaluationProjectData;
+    datasets: EvaluationDataset[];
+    /** Legacy project IDs whose embedded evaluation resources were imported. */
+    migratedLegacyProjectIds: ProjectId[];
+};
 export type EvaluationObservationStatus = 'passed' | 'failed' | 'scored' | 'error' | 'skipped';
 export type EvaluationObservation = {
     id: string;
@@ -284,7 +296,13 @@ export type EvaluationAggregate = {
     /** Normalized internal score in the closed 0..1 range. */
     meanScore?: number;
     averageLatencyMs: number;
+    /** Optional so historical persisted runs without this newer factoid remain readable. */
+    medianLatencyMs?: number;
     p95LatencyMs: number;
+    /** Score distribution statistics use equal-weight per-case mean scores. */
+    medianScore?: number;
+    /** Score distribution statistics use equal-weight per-case mean scores. */
+    p95Score?: number;
     totalCostUsd?: number;
     averageCostUsd?: number;
     targetErrorRate: number;
@@ -307,6 +325,8 @@ export type EvaluationRun = {
     projectId: ProjectId;
     suiteId: string;
     suiteName: string;
+    /** Optional user-assigned label for distinguishing retained runs. */
+    name?: string;
     /**
      * Monotonically increases while a run is persisted. Stores use it to reject
      * a delayed live-progress write after a newer completed snapshot.
@@ -318,6 +338,8 @@ export type EvaluationRun = {
     purpose: EvaluationRunPurpose;
     /** Captures the suite semantics that produced this run. */
     evaluationMode?: EvaluationSuiteMode;
+    /** Planned target executions, retained so live progress does not depend on a later suite edit. */
+    requestedTrialCount?: number;
     executionStatus: 'queued' | 'running' | 'completed' | 'canceled' | 'error';
     qualityStatus: EvaluationQualityStatus;
     qualityReason: EvaluationQualityReason;
@@ -329,6 +351,31 @@ export type EvaluationRun = {
     trials: EvaluationTrial[];
     warnings: string[];
 };
+/**
+ * Incremental live-run protocol. A started event carries only the empty run
+ * shell, a settled event carries one immutable trial, and the terminal event
+ * carries the complete reporting shape once. Consumers that still need full
+ * snapshots can use RunEvaluationSuiteOptions.onUpdate during the compatibility
+ * period.
+ */
+export type EvaluationRunEvent = {
+    type: 'run-started';
+    revision: number;
+    run: EvaluationRun;
+} | {
+    type: 'trial-settled';
+    revision: number;
+    runId: string;
+    projectId: ProjectId;
+    suiteId: string;
+    requestedTrialCount: number;
+    settledTrialCount: number;
+    trial: EvaluationTrial;
+} | {
+    type: 'run-finalized';
+    revision: number;
+    run: EvaluationRun;
+};
 export type EvaluationRecordingReference = {
     id: string;
     retention: 'temporary' | 'failure' | 'baseline' | 'retained';
@@ -386,6 +433,12 @@ export type EvaluationGraphRunner = (input: {
 }) => Promise<EvaluationGraphExecution>;
 export type EvaluationRunStore = {
     put(run: EvaluationRun): Promise<void>;
+    /** Sets a user-assigned run name; omit the name to restore the Unnamed label. */
+    updateRunName(input: {
+        projectId: ProjectId;
+        runId: string;
+        name?: string;
+    }): Promise<EvaluationRun | undefined>;
     get(input: {
         projectId: ProjectId;
         runId: string;
@@ -417,12 +470,31 @@ export type EvaluationRunStore = {
         recordingId: string;
         retention: EvaluationRecordingReference['retention'];
         expiresAt?: string;
-    }): Promise<void>;
+    }): Promise<boolean>;
     /** Pins all artifacts belonging to the run before its compact baseline is saved in the project. */
     promoteBaseline(input: {
         projectId: ProjectId;
         runId: string;
     }): Promise<void>;
+    /**
+     * Optional V2 capability. Persists an idempotent incremental lifecycle event
+     * so a host can recover settled work after a process interruption.
+     */
+    applyRunEvent?(event: EvaluationRunEvent): Promise<void>;
+};
+export type EvaluationStoreInitialization = {
+    /** A recoverable persistence warning that a host may surface to the user. */
+    warning?: string;
+};
+/**
+ * Complete persistence boundary for Evaluations. Desktop, browser, and hosted
+ * Rivet instances implement this same contract so definitions and operational
+ * evidence cannot silently land in different stores.
+ */
+export type EvaluationStore = EvaluationRunStore & {
+    initialize?(): Promise<EvaluationStoreInitialization | void>;
+    getLibrary(): Promise<EvaluationLibrary>;
+    putLibrary(library: EvaluationLibrary): Promise<void>;
 };
 export type EvaluationReporter = {
     report(run: EvaluationRun): Promise<void> | void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@valerypopoff/rivet2-evaluations",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "license": "MIT",
   "main": "dist/cjs/bundle.cjs",
   "module": "dist/esm/index.js",
@@ -36,7 +36,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@valerypopoff/rivet2-core": "^2.3.0",
+    "@valerypopoff/rivet2-core": "^2.4.0",
     "yaml": "^2.9.0"
   }
 }