niceeval 0.16.3-canary.99 → 0.17.1

package/dist/inspection/results.d.mts

@@ -2383,58 +2383,17 @@ export declare const InspectionRunValueSchema: Schema.Struct<{
     }>>;
 }>;
 export type InspectionRunValue = Schema.Schema.Type<typeof InspectionRunValueSchema>;
-export declare const InspectionRunResultSchema: Schema.Struct<{
-    readonly value: Schema.Struct<{
-        readonly runId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
-        readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-        readonly state: Schema.Literals<readonly ["active", "completed", "interrupted", "failed"]>;
-        readonly context: Schema.optional<Schema.Struct<{
-            readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-            readonly execution: Schema.Codec<import("../record/model/run-context.mjs").RunExecutionContext, import("../record/model/run-context.mjs").RunExecutionContext, never, never>;
-            readonly labels: Schema.Codec<Readonly<Record<string, string>>, Readonly<Record<string, string>>, never, never>;
-        }>>;
-        readonly startedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
-        readonly completedAt: Schema.optional<Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>>;
-        readonly expectedSlots: Schema.$Array<Schema.Struct<{
-            readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
-            readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
-            readonly attemptOrdinal: Schema.Number;
-            readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
-        }>>;
-    }>;
-    readonly members: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
-        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
-        readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
-        readonly attempt: Schema.Struct<{
-            readonly originRunId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
-            readonly attemptId: Schema.Codec<import("../record/index.mjs").AttemptId, string, never, never>;
-        }>;
-    }>, Schema.Struct<{
-        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
-        readonly action: Schema.Literals<readonly ["not-dispatched", "interrupted"]>;
-        readonly attempt: Schema.Null;
-        readonly absenceReason: Schema.Literals<readonly ["early-exit-satisfied", "budget-exhausted", "stopped-by-failure", "interrupted-before-publication", "dispatch-failed"]>;
-    }>]>>;
-    readonly attempts: Schema.$Array<Schema.Struct<{
-        readonly attemptId: Schema.Codec<import("../record/index.mjs").AttemptId, string, never, never>;
-        readonly originRunId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
-        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
-        readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
-        readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
-        readonly outcome: Schema.Literals<readonly ["completed", "errored", "cancelled", "interrupted"]>;
-    }>>;
-}>;
-export type InspectionRunResult = Schema.Schema.Type<typeof InspectionRunResultSchema>;
 export declare const InspectionRunListResultSchema: Schema.$Array<Schema.Struct<{
-    readonly runId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
-    readonly state: Schema.Literal<"completed">;
-    readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-    readonly invocationId: Schema.Null;
-    readonly startedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
-    readonly completedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
+    readonly runId: Schema.String;
+    readonly invocationId: Schema.String;
+    readonly experimentId: Schema.String;
+    readonly state: Schema.Literals<readonly ["active", "completed", "interrupted", "failed"]>;
+    readonly startedAt: Schema.String;
+    readonly completedAt: Schema.optional<Schema.String>;
     readonly coverage: Schema.Struct<{
         readonly published: Schema.Number;
         readonly expected: Schema.Number;
+        readonly missing: Schema.Number;
     }>;
 }>>;
 export type InspectionRunListResult = Schema.Schema.Type<typeof InspectionRunListResultSchema>;
@@ -2451,53 +2410,6 @@ export declare const InspectionScoredValueSchema: Schema.Union<readonly [Schema.
     readonly unavailable: Schema.Number;
 }>]>;
 export type InspectionScoredValue = Schema.Schema.Type<typeof InspectionScoredValueSchema>;
-export declare const InspectionRunSummaryResultSchema: Schema.Struct<{
-    readonly runs: Schema.$Array<Schema.Struct<{
-        readonly runId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
-        readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-        readonly context: Schema.Struct<{
-            readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-            readonly execution: Schema.Codec<import("../record/model/run-context.mjs").RunExecutionContext, import("../record/model/run-context.mjs").RunExecutionContext, never, never>;
-            readonly labels: Schema.Codec<Readonly<Record<string, string>>, Readonly<Record<string, string>>, never, never>;
-        }>;
-        readonly startedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
-        readonly completedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
-        readonly expectedSlots: Schema.$Array<Schema.Struct<{
-            readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
-            readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
-            readonly attemptOrdinal: Schema.Number;
-            readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
-        }>>;
-    }>>;
-    readonly denominator: Schema.Struct<{
-        readonly expected: Schema.Number;
-        readonly observed: Schema.Number;
-    }>;
-    readonly members: Schema.$Array<Schema.Struct<{
-        readonly runId: Schema.String;
-        readonly slotId: Schema.String;
-        readonly evalId: Schema.String;
-        readonly attemptOrdinal: Schema.Number;
-        readonly executionIdentityDigest: Schema.String;
-        readonly state: Schema.Literals<readonly ["executed", "carried", "accepted", "not-dispatched", "interrupted", "missing"]>;
-        readonly locator: Schema.NullOr<Schema.String>;
-        readonly outcome: Schema.NullOr<Schema.Literals<readonly ["completed", "errored", "cancelled", "interrupted"]>>;
-        readonly verdict: Schema.NullOr<Schema.Literals<readonly ["passed", "failed", "errored", "skipped"]>>;
-        readonly score: Schema.optional<Schema.Union<readonly [Schema.Struct<{
-            readonly state: Schema.Literal<"not-scored">;
-        }>, Schema.Struct<{
-            readonly state: Schema.Literal<"complete">;
-            readonly earned: Schema.Number;
-            readonly possible: Schema.Number;
-        }>, Schema.Struct<{
-            readonly state: Schema.Literal<"unavailable">;
-            readonly earned: Schema.Number;
-            readonly possible: Schema.Number;
-            readonly unavailable: Schema.Number;
-        }>]>>;
-    }>>;
-}>;
-export type InspectionRunSummaryResult = Schema.Schema.Type<typeof InspectionRunSummaryResultSchema>;
 export declare const InspectionAttemptResultSchema: Schema.Struct<{
     readonly core: Schema.Struct<{
         readonly attemptId: Schema.Codec<import("../record/index.mjs").AttemptId, string, never, never>;
@@ -4004,6 +3916,60 @@ declare const JudgeUsageSchema: Schema.Union<readonly [Schema.Union<readonly [Sc
 }>]>;
 export type JudgeUsage = Schema.Schema.Type<typeof JudgeUsageSchema>;
 export declare const InspectionAttemptUsageResultSchema: Schema.Struct<{
+    readonly cacheRead: Schema.Struct<{
+        readonly total: Schema.Struct<{
+            rate: Schema.Union<readonly [Schema.Struct<{
+                readonly state: Schema.Literal<"exact">;
+                readonly value: Schema.Number;
+            }>, Schema.Struct<{
+                readonly state: Schema.Literal<"unavailable">;
+                readonly reason: Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>;
+            }>]>;
+            recordedCalls: Schema.Number;
+            coveredCalls: Schema.Number;
+            missingCacheReadCalls: Schema.Number;
+            missingInputTotalCalls: Schema.Number;
+            unknownStatusCalls: Schema.Number;
+            collectionComplete: Schema.Boolean;
+            reasons: Schema.$Array<Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>>;
+        }>;
+        readonly modelSlots: Schema.$Array<Schema.Struct<{
+            readonly rate: Schema.Union<readonly [Schema.Struct<{
+                readonly state: Schema.Literal<"exact">;
+                readonly value: Schema.Number;
+            }>, Schema.Struct<{
+                readonly state: Schema.Literal<"unavailable">;
+                readonly reason: Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>;
+            }>]>;
+            readonly recordedCalls: Schema.Number;
+            readonly coveredCalls: Schema.Number;
+            readonly missingCacheReadCalls: Schema.Number;
+            readonly missingInputTotalCalls: Schema.Number;
+            readonly unknownStatusCalls: Schema.Number;
+            readonly collectionComplete: Schema.Boolean;
+            readonly reasons: Schema.$Array<Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>>;
+            readonly modelSlot: Schema.NullOr<Schema.String>;
+        }>>;
+        readonly models: Schema.$Array<Schema.Struct<{
+            readonly rate: Schema.Union<readonly [Schema.Struct<{
+                readonly state: Schema.Literal<"exact">;
+                readonly value: Schema.Number;
+            }>, Schema.Struct<{
+                readonly state: Schema.Literal<"unavailable">;
+                readonly reason: Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>;
+            }>]>;
+            readonly recordedCalls: Schema.Number;
+            readonly coveredCalls: Schema.Number;
+            readonly missingCacheReadCalls: Schema.Number;
+            readonly missingInputTotalCalls: Schema.Number;
+            readonly unknownStatusCalls: Schema.Number;
+            readonly collectionComplete: Schema.Boolean;
+            readonly reasons: Schema.$Array<Schema.Literals<readonly ["usage-not-recorded", "source-invalid", "adapter-usage-required", "collection-incomplete", "call-status-unknown", "no-calls", "cache-read-missing", "input-total-missing", "zero-input-total", "unsafe-token-total"]>>;
+            readonly modelSlot: Schema.NullOr<Schema.String>;
+            readonly provider: Schema.NullOr<Schema.String>;
+            readonly model: Schema.NullOr<Schema.String>;
+        }>>;
+    }>;
     readonly configuredModels: Schema.Union<readonly [Schema.Struct<{
         readonly state: Schema.Literal<"not-recorded">;
     }>, Schema.Struct<{
@@ -4393,26 +4359,65 @@ export declare const InspectionAttemptUsageResultSchema: Schema.Struct<{
     readonly omittedObservationCount: Schema.Number;
 }>;
 export type InspectionAttemptUsageResult = Schema.Schema.Type<typeof InspectionAttemptUsageResultSchema>;
-export declare const InspectionRunOverviewResultSchema: Schema.Struct<{
-    readonly identity: Schema.Struct<{
+export declare const InspectionRunResultSchema: Schema.Struct<{
+    readonly value: Schema.Struct<{
         readonly runId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
         readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
-        readonly adapter: Schema.NullOr<Schema.Codec<import("../record/model/run-context.mjs").AdapterIdentity, import("../record/model/run-context.mjs").AdapterIdentity, never, never>>;
-    }>;
-    readonly state: Schema.Literals<readonly ["active", "completed", "interrupted", "failed"]>;
-    readonly startedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
-    readonly completedAt: Schema.optional<Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>>;
-    readonly denominator: Schema.Struct<{
-        readonly expected: Schema.Number;
-        readonly observed: Schema.Number;
-    }>;
-    readonly members: Schema.$Array<Schema.Struct<{
-        readonly slot: Schema.Struct<{
+        readonly state: Schema.Literals<readonly ["active", "completed", "interrupted", "failed"]>;
+        readonly context: Schema.optional<Schema.Struct<{
+            readonly experimentId: Schema.Codec<import("../record/model/identifiers.mjs").ExperimentId, string, never, never>;
+            readonly execution: Schema.Codec<import("../record/model/run-context.mjs").RunExecutionContext, import("../record/model/run-context.mjs").RunExecutionContext, never, never>;
+            readonly labels: Schema.Codec<Readonly<Record<string, string>>, Readonly<Record<string, string>>, never, never>;
+        }>>;
+        readonly startedAt: Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>;
+        readonly completedAt: Schema.optional<Schema.Codec<import("../record/model/identifiers.mjs").UtcMillis, number, never, never>>;
+        readonly expectedSlots: Schema.$Array<Schema.Struct<{
             readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
             readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
             readonly attemptOrdinal: Schema.Number;
             readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
+        }>>;
+    }>;
+    readonly members: Schema.$Array<Schema.Union<readonly [Schema.Struct<{
+        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
+        readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
+        readonly attempt: Schema.Struct<{
+            readonly originRunId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
+            readonly attemptId: Schema.Codec<import("../record/index.mjs").AttemptId, string, never, never>;
         }>;
+    }>, Schema.Struct<{
+        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
+        readonly action: Schema.Literals<readonly ["not-dispatched", "interrupted"]>;
+        readonly attempt: Schema.Null;
+        readonly absenceReason: Schema.Literals<readonly ["early-exit-satisfied", "budget-exhausted", "stopped-by-failure", "interrupted-before-publication", "dispatch-failed"]>;
+    }>]>>;
+    readonly attempts: Schema.$Array<Schema.Struct<{
+        readonly attemptId: Schema.Codec<import("../record/index.mjs").AttemptId, string, never, never>;
+        readonly originRunId: Schema.Codec<import("../record/model/identifiers.mjs").RunId, string, never, never>;
+        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
+        readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
+        readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
+        readonly outcome: Schema.Literals<readonly ["completed", "errored", "cancelled", "interrupted"]>;
+    }>>;
+    readonly denominator: Schema.Struct<{
+        readonly expected: Schema.Number;
+        readonly published: Schema.Number;
+        readonly missing: Schema.Number;
+    }>;
+    readonly slots: Schema.$Array<Schema.Struct<{
+        readonly publication: Schema.Union<readonly [Schema.Struct<{
+            readonly state: Schema.Literal<"pending">;
+        }>, Schema.Struct<{
+            readonly state: Schema.Literal<"published">;
+            readonly action: Schema.Literals<readonly ["executed", "carried", "accepted"]>;
+            readonly attemptId: Schema.String;
+            readonly attemptLocator: Schema.String;
+            readonly originRunId: Schema.String;
+            readonly originSlotId: Schema.String;
+        }>, Schema.Struct<{
+            readonly state: Schema.Literal<"absent">;
+            readonly reason: Schema.Literals<readonly ["early-exit-satisfied", "budget-exhausted", "stopped-by-failure", "interrupted-before-publication", "dispatch-failed"]>;
+        }>]>;
         readonly state: Schema.Literals<readonly ["executed", "carried", "accepted", "not-dispatched", "interrupted", "pending"]>;
         readonly locator: Schema.NullOr<Schema.String>;
         readonly relation: Schema.NullOr<Schema.Literals<readonly ["origin", "reference"]>>;
@@ -4687,6 +4692,10 @@ export declare const InspectionRunOverviewResultSchema: Schema.Struct<{
                 readonly reason: Schema.String;
             }>]>;
         }>]>>;
+        readonly slotId: Schema.Codec<import("../record/model/identifiers.mjs").SlotId, string, never, never>;
+        readonly evalId: Schema.Codec<import("../record/model/identifiers.mjs").EvalId, string, never, never>;
+        readonly attemptOrdinal: Schema.Number;
+        readonly executionIdentityDigest: Schema.Codec<import("../record/model/identifiers.mjs").ExecutionIdentityDigest, string, never, never>;
     }>>;
     readonly coverage: Schema.Struct<{
         readonly state: Schema.Literals<readonly ["complete", "partial", "not-recorded", "invalid", "unavailable"]>;
@@ -5070,7 +5079,7 @@ export declare const InspectionRunOverviewResultSchema: Schema.Struct<{
         }>]>;
     }>>;
 }>;
-export type InspectionRunOverviewResult = Schema.Schema.Type<typeof InspectionRunOverviewResultSchema>;
+export type InspectionRunResult = Schema.Schema.Type<typeof InspectionRunResultSchema>;
 export declare const InspectionAttemptTimingResultSchema: Schema.Struct<{
     readonly state: Schema.Literals<readonly ["complete", "partial", "not-recorded", "invalid"]>;
     readonly limitations: Schema.$Array<Schema.Union<readonly [Schema.Union<readonly [Schema.Struct<{
@@ -5175,14 +5184,9 @@ export declare const InspectionAttemptDiffResultSchema: Schema.Union<readonly [S
     }>>;
 }>]>;
 export type InspectionAttemptDiffResult = Schema.Schema.Type<typeof InspectionAttemptDiffResultSchema>;
-export interface InspectionSealedCutoff {
-    readonly kind: "inspection-sealed-cutoff";
-    readonly identity: string;
-    readonly runCount: number;
-    readonly runs: readonly {
-        readonly runId: string;
-        readonly logicalSealIdentity: string;
-    }[];
+export interface InspectionPublicationCutoff {
+    readonly storeGeneration: string;
+    readonly revision: number;
 }
 export interface InspectionSelectionAudit {
     readonly requestedRunIds: readonly string[];
@@ -5196,9 +5200,9 @@ export interface InspectionResultMetadata<Kind extends InspectionOperationId> {
     readonly operation: Kind;
     readonly source: {
         readonly kind: "project-record" | "external-record";
-        readonly sealedCutoffIdentity: string;
+        readonly publicationCutoffIdentity: string;
     };
-    readonly sealedCutoff: InspectionSealedCutoff;
+    readonly publicationCutoff: InspectionPublicationCutoff;
     readonly selection: InspectionSelectionAudit;
     readonly issues: readonly [];
     readonly evidence: {
@@ -5207,10 +5211,8 @@ export interface InspectionResultMetadata<Kind extends InspectionOperationId> {
 }
 export type InspectionOverviewDocument = InspectionSuccessDocumentFor<"overview.get">;
 export type InspectionExperimentDocument = InspectionSuccessDocumentFor<"experiment.get">;
-export type InspectionRunListDocument = InspectionSuccessDocumentFor<"runs.list">;
+export type InspectionRunListDocument = InspectionSuccessDocumentFor<"run.list">;
 export type InspectionRunDocument = InspectionSuccessDocumentFor<"run.get">;
-export type InspectionRunSummaryDocument = InspectionSuccessDocumentFor<"run.summary">;
-export type InspectionRunOverviewDocument = InspectionSuccessDocumentFor<"run.overview">;
 export type InspectionAttemptDocument = InspectionSuccessDocumentFor<"attempt.get">;
 export type InspectionAttemptSourcesDocument = InspectionSuccessDocumentFor<"attempt.sources">;
 export type InspectionAttemptTraceDocument = InspectionSuccessDocumentFor<"attempt.trace">;