niceeval 0.16.3-canary.99 → 0.17.0

package/dist/inspection/public.mjs

@@ -3937,6 +3937,54 @@ var AssertionDetailResultSchema = Schema.toType(Schema.Union([Schema.Struct({
 	check: DiagnosticNodeSchema,
 	matcher: MatcherProjectionSchema
 })]));
+//#endregion
+//#region src/o11y/cache-read.ts
+var Reason = Schema.Literals([
+	"usage-not-recorded",
+	"source-invalid",
+	"adapter-usage-required",
+	"collection-incomplete",
+	"call-status-unknown",
+	"no-calls",
+	"cache-read-missing",
+	"input-total-missing",
+	"zero-input-total",
+	"unsafe-token-total"
+]);
+var fields = {
+	rate: Schema.Union([Schema.Struct({
+		state: Schema.Literal("exact"),
+		value: Schema.Number.check(Schema.makeFilter((value) => Number.isFinite(value) && value >= 0 && value <= 1))
+	}), Schema.Struct({
+		state: Schema.Literal("unavailable"),
+		reason: Reason
+	})]),
+	recordedCalls: NonNegativeSafeIntegerSchema$2,
+	coveredCalls: NonNegativeSafeIntegerSchema$2,
+	missingCacheReadCalls: NonNegativeSafeIntegerSchema$2,
+	missingInputTotalCalls: NonNegativeSafeIntegerSchema$2,
+	unknownStatusCalls: NonNegativeSafeIntegerSchema$2,
+	collectionComplete: Schema.Boolean,
+	reasons: Schema.Array(Reason)
+};
+var consistent = (value) => {
+	const count = value.recordedCalls;
+	return count <= 4e3 && value.coveredCalls <= count && value.unknownStatusCalls <= count && value.missingCacheReadCalls <= count && value.missingInputTotalCalls <= count && value.coveredCalls <= count - Math.max(value.missingCacheReadCalls, value.missingInputTotalCalls) && value.coveredCalls >= count - value.missingCacheReadCalls - value.missingInputTotalCalls && new Set(value.reasons).size === value.reasons.length && (value.rate.state === "exact" ? value.reasons.length === 0 && value.collectionComplete && count > 0 && value.coveredCalls === count && value.unknownStatusCalls === 0 : value.rate.reason === value.reasons[0]);
+};
+var Summary = Schema.Struct(fields).check(Schema.makeFilter(consistent));
+var CacheReadReportSchema = Schema.Struct({
+	total: Summary,
+	modelSlots: Schema.Array(Schema.Struct({
+		modelSlot: Schema.NullOr(Schema.String),
+		...fields
+	}).check(Schema.makeFilter(consistent))),
+	models: Schema.Array(Schema.Struct({
+		modelSlot: Schema.NullOr(Schema.String),
+		provider: Schema.NullOr(Schema.String),
+		model: Schema.NullOr(Schema.String),
+		...fields
+	}).check(Schema.makeFilter(consistent)))
+}).check(Schema.makeFilter((value) => value.modelSlots.length <= 65 && value.models.length <= 4e3));
 Object.freeze({
 	errors: "all",
 	onExcessProperty: "error"
@@ -5846,23 +5894,7 @@ var InspectionRunValueSchema = Schema.Struct({
 	completedAt: Schema.optional(UtcMillisSchema),
 	expectedSlots: Schema.Array(RecordSlotIdentitySchema)
 });
-var InspectionRunResultSchema = Schema.Struct({
-	value: InspectionRunValueSchema,
-	members: Schema.Array(MemberDocumentSchema),
-	attempts: Schema.Array(AttemptDocumentSchema)
-});
-Schema.Array(Schema.Struct({
-	runId: RunIdSchema,
-	state: Schema.Literal("completed"),
-	experimentId: ExperimentIdSchema,
-	invocationId: Schema.Null,
-	startedAt: UtcMillisSchema,
-	completedAt: UtcMillisSchema,
-	coverage: Schema.Struct({
-		published: Schema.Number,
-		expected: Schema.Number
-	})
-}));
+var InspectionRunListResultSchema = Schema.Array(RunSummarySchema);
 var InspectionScoredValueSchema = Schema.Union([
 	Schema.Struct({ state: Schema.Literal("not-scored") }),
 	Schema.Struct({
@@ -5883,37 +5915,6 @@ var VerdictSchema = Schema.NullOr(Schema.Literals([
 	"errored",
 	"skipped"
 ]));
-var InspectionRunSummaryResultSchema = Schema.Struct({
-	runs: Schema.Array(RunDocumentSchema),
-	denominator: Schema.Struct({
-		expected: Schema.Number,
-		observed: Schema.Number
-	}),
-	members: Schema.Array(Schema.Struct({
-		runId: Schema.String,
-		slotId: Schema.String,
-		evalId: Schema.String,
-		attemptOrdinal: Schema.Number,
-		executionIdentityDigest: Schema.String,
-		state: Schema.Literals([
-			"executed",
-			"carried",
-			"accepted",
-			"not-dispatched",
-			"interrupted",
-			"missing"
-		]),
-		locator: Schema.NullOr(Schema.String),
-		outcome: Schema.NullOr(Schema.Literals([
-			"completed",
-			"errored",
-			"cancelled",
-			"interrupted"
-		])),
-		verdict: VerdictSchema,
-		score: Schema.optional(InspectionScoredValueSchema)
-	}))
-});
 var AttemptCoverageFactSchema = Schema.Struct({
 	channel: Schema.Literals([
 		"events",
@@ -6893,6 +6894,7 @@ var UsageTotalsSchema = Schema.Struct({
 	costs: Schema.optional(EffectiveCostTotalSchema)
 });
 var InspectionAttemptUsageResultSchema = Schema.Struct({
+	cacheRead: CacheReadReportSchema,
 	configuredModels: ConfiguredModelsSchema,
 	modelGroups: ModelGroupsSchema,
 	judgeUsage: JudgeUsageSchema,
@@ -6965,7 +6967,8 @@ var RunOverviewMemberLimitationSchema = Schema.Union([
 	})
 ]);
 var RunOverviewMemberSchema = Schema.Struct({
-	slot: RecordSlotIdentitySchema,
+	...RecordSlotIdentitySchema.fields,
+	publication: RunSlotPublicationSchema,
 	state: Schema.Literals([
 		"executed",
 		"carried",
@@ -6993,20 +6996,16 @@ var RunOverviewLocatedLimitationSchema = Schema.Struct({
 	locator: Schema.NullOr(Schema.String),
 	limitation: RunOverviewMemberLimitationSchema
 });
-var InspectionRunOverviewResultSchema = Schema.Struct({
-	identity: Schema.Struct({
-		runId: RunIdSchema,
-		experimentId: ExperimentIdSchema,
-		adapter: Schema.NullOr(AdapterIdentitySchema)
-	}),
-	state: RunStateSchema,
-	startedAt: UtcMillisSchema,
-	completedAt: Schema.optional(UtcMillisSchema),
+var InspectionRunResultSchema = Schema.Struct({
+	value: InspectionRunValueSchema,
+	members: Schema.Array(MemberDocumentSchema),
+	attempts: Schema.Array(AttemptDocumentSchema),
 	denominator: Schema.Struct({
 		expected: Schema.Number,
-		observed: Schema.Number
+		published: Schema.Number,
+		missing: Schema.Number
 	}),
-	members: Schema.Array(RunOverviewMemberSchema),
+	slots: Schema.Array(RunOverviewMemberSchema),
 	coverage: Schema.Struct({
 		state: RunOverviewStateSchema,
 		expectedMemberCount: Schema.Number,
@@ -7183,16 +7182,11 @@ var operation = (kind, fields) => Schema.Struct({
 });
 var SourceSchema = Schema.Struct({
 	kind: Schema.Literals(["project-record", "external-record"]),
-	sealedCutoffIdentity: Schema.String
+	publicationCutoffIdentity: Schema.String
 });
-var SealedCutoffSchema = Schema.Struct({
-	kind: Schema.Literal("inspection-sealed-cutoff"),
-	identity: Schema.String,
-	runCount: Schema.Number,
-	runs: Schema.Array(Schema.Struct({
-		runId: Schema.String,
-		logicalSealIdentity: Schema.String
-	}))
+var PublicationCutoffSchema = Schema.Struct({
+	storeGeneration: Schema.String,
+	revision: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0))
 });
 var SelectionSchema = Schema.Struct({
 	requestedRunIds: Schema.Array(Schema.String),
@@ -7204,7 +7198,7 @@ var SuccessMetadataSchema = Schema.Struct({
 	outcome: Schema.Literal("success"),
 	behaviorVersion: Schema.Literal(INSPECTION_BEHAVIOR_VERSION),
 	source: SourceSchema,
-	sealedCutoff: SealedCutoffSchema,
+	publicationCutoff: PublicationCutoffSchema,
 	selection: SelectionSchema,
 	issues: Schema.Tuple([]),
 	evidence: Schema.Struct({ refs: Schema.Array(Schema.String) })
@@ -7214,17 +7208,12 @@ var RunsListSuccessMetadataSchema = Schema.Struct({
 	outcome: Schema.Literal("success"),
 	behaviorVersion: Schema.Literal(INSPECTION_BEHAVIOR_VERSION),
 	source: SourceSchema,
-	sealedCutoff: Schema.Struct({
-		kind: Schema.Literal("inspection-sealed-cutoff"),
-		identity: Schema.String,
-		runCount: Schema.Number
-	}),
+	publicationCutoff: PublicationCutoffSchema,
 	selection: Schema.Struct({
 		requestedRunIds: Schema.Array(Schema.String),
 		selectedRunIds: Schema.Array(Schema.String),
 		missingRunIds: Schema.Array(Schema.String),
 		returnedRunCount: Schema.Number,
-		totalRunCount: Schema.Number,
 		truncated: Schema.Boolean
 	}),
 	issues: Schema.Tuple([]),
@@ -7253,19 +7242,6 @@ var ArtifactsResultSchema = Schema.Union([Schema.Struct({ state: Schema.Literal(
 	})),
 	contentsTruncated: Schema.Boolean
 })]);
-var SealedRunSummarySchema = Schema.Struct({
-	runId: Schema.String,
-	writerGeneration: Schema.String,
-	startedAt: Schema.String,
-	logicalSealIdentity: Schema.String,
-	slotCount: Schema.Number,
-	memberCount: Schema.Number,
-	attemptCount: Schema.Number,
-	attachmentCount: Schema.Number,
-	contentCount: Schema.Number,
-	contentByteLength: Schema.Number,
-	sealEntryCount: Schema.Number
-});
 var ComparisonMemberSchema = Schema.Struct({
 	key: Schema.String,
 	runId: Schema.String,
@@ -7329,10 +7305,10 @@ var inspectionProtocolRegistry = Object.freeze({
 			"judge-usage"
 		]
 	}),
-	"runs.list": spec({
-		request: operation("runs.list", { continuation: Schema.optional(Schema.String) }),
+	"run.list": spec({
+		request: operation("run.list", { continuation: Schema.optional(Schema.String) }),
 		result: {
-			runs: Schema.Array(SealedRunSummarySchema),
+			runs: InspectionRunListResultSchema,
 			continuation: Schema.optional(Schema.String)
 		},
 		factKinds: ["core"]
@@ -7340,24 +7316,12 @@ var inspectionProtocolRegistry = Object.freeze({
 	"run.get": spec({
 		request: operation("run.get", { runId: RunIdSchema }),
 		result: { run: InspectionRunResultSchema },
-		factKinds: ["core"]
-	}),
-	"run.summary": spec({
-		request: operation("run.summary", { runId: RunIdSchema }),
-		result: { summary: InspectionRunSummaryResultSchema },
 		factKinds: [
 			"core",
 			"assertions",
-			"agent-turns"
-		]
-	}),
-	"run.overview": spec({
-		request: operation("run.overview", { runId: RunIdSchema }),
-		result: { runOverview: InspectionRunOverviewResultSchema },
-		factKinds: [
-			"core",
-			"assertions",
-			"agent-turns"
+			"agent-turns",
+			"adapter-usage",
+			"judge-usage"
 		]
 	}),
 	"attempt.get": spec({
@@ -7512,7 +7476,7 @@ var successSchema = (id, entry) => {
 		operation: Schema.Literal(id),
 		...entry.result
 	});
-	return id === "runs.list" ? RunsListSuccessMetadataSchema.pipe(fields) : id === "runs.compare" ? CompareSuccessMetadataSchema.pipe(fields) : SuccessMetadataSchema.pipe(fields);
+	return id === "run.list" ? RunsListSuccessMetadataSchema.pipe(fields) : id === "runs.compare" ? CompareSuccessMetadataSchema.pipe(fields) : SuccessMetadataSchema.pipe(fields);
 };
 var InspectionSuccessDocumentSchema = Schema.Union(specs.map(([id, entry]) => successSchema(id, entry)));
 var explanationSchema = (id, entry) => {
@@ -7521,7 +7485,7 @@ var explanationSchema = (id, entry) => {
 		operation: Schema.Literal(id),
 		factKinds: Schema.Tuple(entry.factKinds.map((value) => Schema.Literal(value)))
 	});
-	if (id === "runs.list") return RunsListSuccessMetadataSchema.pipe(fields);
+	if (id === "run.list") return RunsListSuccessMetadataSchema.pipe(fields);
 	return SuccessMetadataSchema.pipe(fields);
 };
 var InspectionExplanationDocumentSchema = Schema.Union(specs.map(([id, entry]) => explanationSchema(id, entry)));

package/dist/inspection/results.cjs

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.InspectionAttemptDiffResultSchema = exports.InspectionAttemptTimingResultSchema = exports.InspectionRunOverviewResultSchema = exports.InspectionAttemptUsageResultSchema = exports.InspectionTraceDetailResultSchema = exports.InspectionTraceDetailItemSchema = exports.InspectionTraceResultSchema = exports.InspectionTraceItemSchema = exports.ExecutionDisplayPreviewSchema = exports.InspectionSourcesResultSchema = exports.InspectionAttemptResultSchema = exports.InspectionRunSummaryResultSchema = exports.InspectionScoredValueSchema = exports.InspectionRunListResultSchema = exports.InspectionRunResultSchema = exports.InspectionRunValueSchema = exports.InspectionExperimentResultSchema = exports.InspectionOverviewResultSchema = void 0;
+exports.InspectionAttemptDiffResultSchema = exports.InspectionAttemptTimingResultSchema = exports.InspectionRunResultSchema = exports.InspectionAttemptUsageResultSchema = exports.InspectionTraceDetailResultSchema = exports.InspectionTraceDetailItemSchema = exports.InspectionTraceResultSchema = exports.InspectionTraceItemSchema = exports.ExecutionDisplayPreviewSchema = exports.InspectionSourcesResultSchema = exports.InspectionAttemptResultSchema = exports.InspectionScoredValueSchema = exports.InspectionRunListResultSchema = exports.InspectionRunValueSchema = exports.InspectionExperimentResultSchema = exports.InspectionOverviewResultSchema = void 0;
+const cache_read_ts_1 = require("../o11y/cache-read.cjs");
 const effect_1 = require("effect");
 const schema_ts_1 = require("../record/family/adapter-usage/schema.cjs");
 const common_ts_1 = require("../record/family/common.cjs");
@@ -198,23 +199,7 @@ exports.InspectionRunValueSchema = effect_1.Schema.Struct({
     completedAt: effect_1.Schema.optional(identifiers_ts_1.UtcMillisSchema),
     expectedSlots: effect_1.Schema.Array(core_ts_1.RecordSlotIdentitySchema),
 });
-exports.InspectionRunResultSchema = effect_1.Schema.Struct({
-    value: exports.InspectionRunValueSchema,
-    members: effect_1.Schema.Array(core_ts_1.MemberDocumentSchema),
-    attempts: effect_1.Schema.Array(core_ts_1.AttemptDocumentSchema),
-});
-exports.InspectionRunListResultSchema = effect_1.Schema.Array(effect_1.Schema.Struct({
-    runId: identifiers_ts_1.RunIdSchema,
-    state: effect_1.Schema.Literal("completed"),
-    experimentId: identifiers_ts_1.ExperimentIdSchema,
-    invocationId: effect_1.Schema.Null,
-    startedAt: identifiers_ts_1.UtcMillisSchema,
-    completedAt: identifiers_ts_1.UtcMillisSchema,
-    coverage: effect_1.Schema.Struct({
-        published: effect_1.Schema.Number,
-        expected: effect_1.Schema.Number,
-    }),
-}));
+exports.InspectionRunListResultSchema = effect_1.Schema.Array(index_ts_2.RunSummarySchema);
 exports.InspectionScoredValueSchema = effect_1.Schema.Union([
     effect_1.Schema.Struct({ state: effect_1.Schema.Literal("not-scored") }),
     effect_1.Schema.Struct({ state: effect_1.Schema.Literal("complete"), earned: effect_1.Schema.Number, possible: effect_1.Schema.Number }),
@@ -226,24 +211,6 @@ exports.InspectionScoredValueSchema = effect_1.Schema.Union([
     }),
 ]);
 const VerdictSchema = effect_1.Schema.NullOr(effect_1.Schema.Literals(["passed", "failed", "errored", "skipped"]));
-exports.InspectionRunSummaryResultSchema = effect_1.Schema.Struct({
-    runs: effect_1.Schema.Array(core_ts_1.RunDocumentSchema),
-    denominator: effect_1.Schema.Struct({ expected: effect_1.Schema.Number, observed: effect_1.Schema.Number }),
-    members: effect_1.Schema.Array(effect_1.Schema.Struct({
-        runId: effect_1.Schema.String,
-        slotId: effect_1.Schema.String,
-        evalId: effect_1.Schema.String,
-        attemptOrdinal: effect_1.Schema.Number,
-        executionIdentityDigest: effect_1.Schema.String,
-        state: effect_1.Schema.Literals([
-            "executed", "carried", "accepted", "not-dispatched", "interrupted", "missing",
-        ]),
-        locator: effect_1.Schema.NullOr(effect_1.Schema.String),
-        outcome: effect_1.Schema.NullOr(effect_1.Schema.Literals(["completed", "errored", "cancelled", "interrupted"])),
-        verdict: VerdictSchema,
-        score: effect_1.Schema.optional(exports.InspectionScoredValueSchema),
-    })),
-});
 const AttemptCoverageFactSchema = effect_1.Schema.Struct({
     channel: effect_1.Schema.Literals(["events", "actions", "messages", "usage", "status", "data"]),
     status: effect_1.Schema.Literals(["complete", "partial", "unavailable"]),
@@ -927,6 +894,7 @@ const UsageTotalsSchema = effect_1.Schema.Struct({
     costs: effect_1.Schema.optional(EffectiveCostTotalSchema),
 });
 exports.InspectionAttemptUsageResultSchema = effect_1.Schema.Struct({
+    cacheRead: cache_read_ts_1.CacheReadReportSchema,
     configuredModels: ConfiguredModelsSchema,
     modelGroups: ModelGroupsSchema,
     judgeUsage: JudgeUsageSchema,
@@ -982,7 +950,8 @@ const RunOverviewMemberLimitationSchema = effect_1.Schema.Union([
     effect_1.Schema.Struct({ kind: effect_1.Schema.Literal("usage"), detail: UsageLimitationSchema }),
 ]);
 const RunOverviewMemberSchema = effect_1.Schema.Struct({
-    slot: core_ts_1.RecordSlotIdentitySchema,
+    ...core_ts_1.RecordSlotIdentitySchema.fields,
+    publication: index_ts_2.RunSlotPublicationSchema,
     state: effect_1.Schema.Literals([
         "executed", "carried", "accepted", "not-dispatched", "interrupted", "pending",
     ]),
@@ -1000,17 +969,12 @@ const RunOverviewLocatedLimitationSchema = effect_1.Schema.Struct({
     locator: effect_1.Schema.NullOr(effect_1.Schema.String),
     limitation: RunOverviewMemberLimitationSchema,
 });
-exports.InspectionRunOverviewResultSchema = effect_1.Schema.Struct({
-    identity: effect_1.Schema.Struct({
-        runId: identifiers_ts_1.RunIdSchema,
-        experimentId: identifiers_ts_1.ExperimentIdSchema,
-        adapter: effect_1.Schema.NullOr(run_context_ts_1.AdapterIdentitySchema),
-    }),
-    state: index_ts_2.RunStateSchema,
-    startedAt: identifiers_ts_1.UtcMillisSchema,
-    completedAt: effect_1.Schema.optional(identifiers_ts_1.UtcMillisSchema),
-    denominator: effect_1.Schema.Struct({ expected: effect_1.Schema.Number, observed: effect_1.Schema.Number }),
-    members: effect_1.Schema.Array(RunOverviewMemberSchema),
+exports.InspectionRunResultSchema = effect_1.Schema.Struct({
+    value: exports.InspectionRunValueSchema,
+    members: effect_1.Schema.Array(core_ts_1.MemberDocumentSchema),
+    attempts: effect_1.Schema.Array(core_ts_1.AttemptDocumentSchema),
+    denominator: effect_1.Schema.Struct({ expected: effect_1.Schema.Number, published: effect_1.Schema.Number, missing: effect_1.Schema.Number }),
+    slots: effect_1.Schema.Array(RunOverviewMemberSchema),
     coverage: effect_1.Schema.Struct({
         state: RunOverviewStateSchema,
         expectedMemberCount: effect_1.Schema.Number,