niceeval 0.17.1 → 0.19.0

package/dist/inspection/public.mjs

@@ -78,7 +78,8 @@ var NUMERIC_COMPARATORS = [
 	"less-than",
 	"at-most",
 	"greater-than",
-	"at-least"
+	"at-least",
+	"between"
 ];
 //#endregion
 //#region src/assertions/identity.ts
@@ -287,6 +288,16 @@ var NumericComparisonCriterionSchema = Schema.Struct({
 	data: Schema.Struct({
 		comparator: Schema.Literals(NUMERIC_COMPARATORS),
 		threshold: Schema.Finite,
+		upperThreshold: Schema.optional(Schema.Finite),
+		unit: Schema.optional(Schema.Literals([
+			"ms",
+			"seconds",
+			"tokens",
+			"usd",
+			"ratio",
+			"count",
+			"bytes"
+		])),
 		subject: Schema.Union([
 			Schema.Struct({ kind: Schema.Literal("explicit-value") }),
 			Schema.Struct({
@@ -319,7 +330,7 @@ var NumericComparisonCriterionSchema = Schema.Struct({
 				])
 			})
 		])
-	})
+	}).check(Schema.makeFilter((data) => data.comparator === "between" ? data.upperThreshold !== void 0 && data.upperThreshold >= data.threshold : data.upperThreshold === void 0, { message: "Only between comparisons have an ordered upper threshold." }))
 });
 var ScopeStatusCriterionSchema = Schema.Struct({
 	kind: Schema.Literal("builtin"),
@@ -2211,14 +2222,14 @@ var RunGetDocumentSchema = Schema.Struct({
 });
 Schema.Union([RunListDocumentSchema, RunGetDocumentSchema]);
 //#endregion
-//#region src/model-slots.ts
+//#region src/model-roles.ts
 var modelText = Schema.String.check(Schema.isPattern(/\S/u));
-var slotKey = Schema.String.check(Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\s\S])/u));
-Schema.Record(slotKey, Schema.Struct({
+var roleKey = Schema.String.check(Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\s\S])/u));
+Schema.Record(roleKey, Schema.Struct({
 	model: modelText,
 	reasoningEffort: Schema.optionalKey(modelText)
 })).check(Schema.makeFilter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 64));
-var ResolvedModelSlotsSchema = Schema.Record(slotKey, Schema.Struct({
+var ResolvedModelRolesSchema = Schema.Record(roleKey, Schema.Struct({
 	model: Schema.NullOr(modelText),
 	reasoningEffort: Schema.NullOr(modelText)
 })).check(Schema.makeFilter((value) => Object.keys(value).length <= 64));
@@ -2260,8 +2271,14 @@ var AdapterIdentitySchema = Schema.Struct({
 	behaviorRevision: Schema.NullOr(NonEmptyStringSchema),
 	cleanupTimeoutMs: Schema.optionalKey(AdapterCleanupTimeoutMsSchema)
 });
+var RunPluginIdentitySchema = Schema.Struct({
+	name: NonEmptyStringSchema,
+	key: NonEmptyStringSchema,
+	identity: RunContextJsonObjectSchema
+});
 var RunExecutionContextSchema = Schema.Struct({
-	models: Schema.optionalKey(ResolvedModelSlotsSchema),
+	plugins: Schema.optionalKey(Schema.Array(RunPluginIdentitySchema)),
+	models: Schema.optionalKey(ResolvedModelRolesSchema),
 	adapter: AdapterIdentitySchema,
 	model: Schema.NullOr(Schema.String),
 	reasoningEffort: Schema.NullOr(Schema.String),
@@ -2983,6 +3000,52 @@ Data.TaggedError("InspectionArtifactError");
 Schema.decodeUnknownResult(ReadRequestSchema, RecordExactParseOptions);
 Schema.decodeUnknownResult(Schema.Struct({ "$niceeval.record.content": Schema.String }), RecordExactParseOptions);
 //#endregion
+//#region src/inspection/planned-attempt.ts
+var { slotId: plannedAttemptId, ...positionFields } = RecordSlotIdentitySchema.fields;
+var PlannedAttemptSchema = Schema.Struct({
+	plannedAttemptId,
+	...positionFields
+});
+var { slotId: attemptPosition, ...attemptFields } = AttemptDocumentSchema.fields;
+var InspectionAttemptCoreSchema = Schema.Struct({
+	plannedAttemptId: attemptPosition,
+	...attemptFields
+}).check(Schema.makeFilter(({ plannedAttemptId, ...attempt }) => Schema.is(AttemptDocumentSchema)({
+	...attempt,
+	slotId: plannedAttemptId
+})));
+var { expectedSlots: _expectedSlots, ...runFields } = RunDocumentSchema.fields;
+var InspectionRunCoreSchema = Schema.Struct({
+	...runFields,
+	plannedAttempts: Schema.Array(PlannedAttemptSchema)
+}).check(Schema.makeFilter(({ plannedAttempts, ...run }) => Schema.is(RunDocumentSchema)({
+	...run,
+	expectedSlots: plannedAttempts.map(({ plannedAttemptId, ...position }) => ({
+		...position,
+		slotId: plannedAttemptId
+	}))
+})));
+var InspectionMemberSchema = Schema.Union(MemberDocumentSchema.members.map((member) => {
+	const { slotId, ...fields } = member.fields;
+	return Schema.Struct({
+		plannedAttemptId: slotId,
+		...fields
+	});
+})).check(Schema.makeFilter(({ plannedAttemptId, ...member }) => Schema.is(MemberDocumentSchema)({
+	...member,
+	slotId: plannedAttemptId
+})));
+var { originSlotId, ...publicationFields } = RunPublishedPublicationSchema.fields;
+var InspectionPublishedPublicationSchema = Schema.Struct({
+	...publicationFields,
+	originPlannedAttemptId: originSlotId
+});
+var PlannedAttemptPublicationSchema = Schema.Union([
+	RunPendingPublicationSchema,
+	InspectionPublishedPublicationSchema,
+	RunAbsentPublicationSchema
+]);
+//#endregion
 //#region src/inspection/artifact-list.ts
 var { content: _content, ...metadataFields } = ArtifactSchema.fields;
 var InspectionArtifactsPageLimitSchema = NonNegativeSafeIntegerSchema$2.pipe(Schema.check(Schema.makeFilter((value) => value > 0 && value <= 128)));
@@ -3181,7 +3244,7 @@ Object.freeze([
 	"niceeval.runner-diagnostics"
 ]);
 Object.freeze(["niceeval.runner-activities", "niceeval.runner-diagnostics"]);
-function isObject$1(value) {
+function isObject(value) {
 	return typeof value === "object" && value !== null;
 }
 function ownDataProperty(value, key) {
@@ -3189,7 +3252,7 @@ function ownDataProperty(value, key) {
 	return descriptor !== void 0 && "value" in descriptor ? descriptor.value : void 0;
 }
 function isReferenceFields(value) {
-	if (!isObject$1(value)) return false;
+	if (!isObject(value)) return false;
 	const family = ownDataProperty(value, "family");
 	const kind = ownDataProperty(value, "kind");
 	const id = ownDataProperty(value, "id");
@@ -3369,7 +3432,7 @@ function validateObservabilityCollection(collection) {
 }
 //#endregion
 //#region src/inspection/protocol-values.ts
-var QUERY_PROTOCOL = "niceeval.query/v1";
+var QUERY_PROTOCOL = "niceeval.query/v2";
 /** Selector semantics revision; independent from the wire protocol major. */
 var INSPECTION_BEHAVIOR_VERSION = "2.0.0";
 //#endregion
@@ -3938,6 +4001,40 @@ var AssertionDetailResultSchema = Schema.toType(Schema.Union([Schema.Struct({
 	matcher: MatcherProjectionSchema
 })]));
 //#endregion
+//#region src/inspection/lifecycle.ts
+function projectLifecyclePhase(phase) {
+	return phase.replace(/\.setup$/, ".before").replace(/\.teardown$/, ".after");
+}
+//#endregion
+//#region src/record/family/attempt-note/schema.ts
+var AttemptNoteLimits = Object.freeze({
+	maximumEntries: 64,
+	maximumJsonBytes: 65536
+});
+var NoteJsonSchema = Schema.suspend(() => Schema.Union([
+	Schema.Null,
+	Schema.Boolean,
+	Schema.Finite,
+	Schema.String,
+	Schema.Array(NoteJsonSchema),
+	Schema.Record(Schema.String, NoteJsonSchema)
+]));
+var AttemptNoteSchema = Schema.Struct({
+	label: Schema.String,
+	sequence: NonNegativeSafeIntegerSchema$2,
+	value: NoteJsonSchema,
+	unit: Schema.optional(Schema.Literals([
+		"ms",
+		"seconds",
+		"tokens",
+		"usd",
+		"ratio",
+		"count",
+		"bytes"
+	]))
+});
+var AttemptNotesSchema = Schema.Array(AttemptNoteSchema).check(Schema.makeFilter((notes) => notes.length <= AttemptNoteLimits.maximumEntries && new Set(notes.map((note) => note.label)).size === notes.length && notes.every((note, index) => note.sequence === index && (note.unit === void 0 || typeof note.value === "number") && new TextEncoder().encode(JSON.stringify(note)).byteLength <= AttemptNoteLimits.maximumJsonBytes)));
+//#endregion
 //#region src/o11y/cache-read.ts
 var Reason = Schema.Literals([
 	"usage-not-recorded",
@@ -3974,17 +4071,17 @@ var consistent = (value) => {
 var Summary = Schema.Struct(fields).check(Schema.makeFilter(consistent));
 var CacheReadReportSchema = Schema.Struct({
 	total: Summary,
-	modelSlots: Schema.Array(Schema.Struct({
-		modelSlot: Schema.NullOr(Schema.String),
+	roles: Schema.Array(Schema.Struct({
+		modelRole: Schema.NullOr(Schema.String),
 		...fields
 	}).check(Schema.makeFilter(consistent))),
 	models: Schema.Array(Schema.Struct({
-		modelSlot: Schema.NullOr(Schema.String),
+		modelRole: Schema.NullOr(Schema.String),
 		provider: Schema.NullOr(Schema.String),
 		model: Schema.NullOr(Schema.String),
 		...fields
 	}).check(Schema.makeFilter(consistent)))
-}).check(Schema.makeFilter((value) => value.modelSlots.length <= 65 && value.models.length <= 4e3));
+}).check(Schema.makeFilter((value) => value.roles.length <= 65 && value.models.length <= 4e3));
 Object.freeze({
 	errors: "all",
 	onExcessProperty: "error"
@@ -4713,7 +4810,7 @@ function validateJudgeUsageAttachment(value) {
 /** Tuple order is stable across hosts, with missing identities before strings. */
 function compareAdapterModelGroups(left, right) {
 	for (const key of [
-		"modelSlot",
+		"modelRole",
 		"provider",
 		"model"
 	]) {
@@ -5686,7 +5783,7 @@ var OverviewIssueSchema = Schema.Union([
 	Schema.Struct({
 		code: Schema.Literals(["attempt-origin-missing", "attempt-not-observed"]),
 		runId: Schema.String,
-		slotId: Schema.String,
+		plannedAttemptId: Schema.String,
 		evalId: Schema.String,
 		attemptOrdinal: Schema.Number
 	}),
@@ -5714,7 +5811,7 @@ var MetricSchema = Schema.Struct({
 	value: Schema.NullOr(Schema.Number),
 	samples: Schema.Number,
 	total: Schema.Number,
-	basis: Schema.Literals(["slot", "eval"]),
+	basis: Schema.Literals(["planned-attempt", "eval"]),
 	issues: Schema.Array(OverviewIssueSchema),
 	refs: Schema.Array(AttemptRefSchema),
 	unit: Schema.optional(Schema.Literals([
@@ -5802,7 +5899,7 @@ var AggregateFields = {
 	issues: Schema.Array(OverviewIssueSchema)
 };
 var OverviewPublishedMemberSchema = Schema.Struct({
-	...RunPublishedPublicationSchema.fields,
+	...InspectionPublishedPublicationSchema.fields,
 	verdict: Schema.NullOr(Schema.Literals([
 		"passed",
 		"failed",
@@ -5825,7 +5922,7 @@ var InspectionExecutionValueSchema = Schema.Union([
 var InspectionLabelsSchema = Schema.Record(Schema.String, InspectionExecutionValueSchema);
 var OverviewMemberSchema = Schema.Struct({
 	runId: Schema.String,
-	slotId: Schema.String,
+	plannedAttemptId: Schema.String,
 	evalId: Schema.String,
 	attemptOrdinal: Schema.Number,
 	labels: InspectionLabelsSchema,
@@ -5840,6 +5937,7 @@ var OverviewGroupSchema = Schema.Struct({
 	...AggregateFields
 });
 var OverviewExperimentSchema = Schema.Struct({
+	plugins: Schema.suspend(() => Schema.Array(RunPluginIdentitySchema)),
 	experimentId: Schema.String,
 	adapter: Schema.Union([Schema.Struct({
 		state: Schema.Literal("available"),
@@ -5863,6 +5961,15 @@ var InspectionOverviewResultSchema = Schema.Struct({
 	cells: Schema.Array(OverviewCellSchema)
 });
 var InspectionExperimentResultSchema = Schema.Struct({
+	notes: Schema.Array(Schema.Struct({
+		label: Schema.String,
+		unit: AttemptNoteSchema.fields.unit,
+		count: NonNegativeSafeIntegerSchema$2,
+		min: Schema.Finite,
+		median: Schema.Finite,
+		p90: Schema.Finite,
+		max: Schema.Finite
+	})),
 	costSummary: Schema.suspend(() => ExperimentCostSummarySchema),
 	modelUsage: Schema.Struct({
 		attempts: Schema.Array(Schema.Struct({
@@ -5892,7 +5999,7 @@ var InspectionRunValueSchema = Schema.Struct({
 	context: Schema.optional(RunContextSchema),
 	startedAt: UtcMillisSchema,
 	completedAt: Schema.optional(UtcMillisSchema),
-	expectedSlots: Schema.Array(RecordSlotIdentitySchema)
+	plannedAttempts: Schema.Array(PlannedAttemptSchema)
 });
 var InspectionRunListResultSchema = Schema.Array(RunSummarySchema);
 var InspectionScoredValueSchema = Schema.Union([
@@ -6015,12 +6122,13 @@ var AssertionEvidenceSchema = Schema.Union([
 	})
 ]);
 var InspectionAttemptResultSchema = Schema.Struct({
-	core: AttemptDocumentSchema,
+	notes: AttemptNotesSchema,
+	core: InspectionAttemptCoreSchema,
 	locator: Schema.String,
-	originRun: RunDocumentSchema,
+	originRun: InspectionRunCoreSchema,
 	targets: Schema.Array(Schema.Struct({
 		runId: Schema.String,
-		member: MemberDocumentSchema
+		member: InspectionMemberSchema
 	})),
 	evidence: AssertionEvidenceSchema,
 	assertions: AssertionIndexSchema,
@@ -6209,7 +6317,7 @@ var TraceDiagnosticsSchema = Schema.Struct({
 		sequence: Schema.Number,
 		turnId: Schema.NullOr(Schema.String),
 		phase: Schema.Literals([
-			"attempt.setup",
+			"attempt.before",
 			"sandbox.prepare",
 			"agent.ensure",
 			"eval.run",
@@ -6217,7 +6325,7 @@ var TraceDiagnosticsSchema = Schema.Struct({
 			"sandbox.command",
 			"assertion.evaluate",
 			"verdict.fold",
-			"attempt.teardown"
+			"attempt.after"
 		]),
 		kind: Schema.Literals(["advisory", "execution-error"]),
 		code: Schema.String,
@@ -6466,7 +6574,7 @@ var InspectionTraceResultSchema = Schema.Struct({
 		omittedLimitationCount: Schema.Number,
 		items: Schema.Array(Schema.Struct({
 			commandId: CommandIdSchema$1,
-			phase: Schema.Literals(SANDBOX_COMMAND_PHASES),
+			phase: Schema.Literals(SANDBOX_COMMAND_PHASES.map(projectLifecyclePhase)),
 			outcome: CommandOutcomeSchema
 		})),
 		hasMore: Schema.Boolean,
@@ -6645,12 +6753,12 @@ var InspectionTraceDetailResultSchema = Schema.Union([
 		outcome: CommandOutcomeSchema,
 		turnId: Schema.NullOr(Schema.String),
 		phase: Schema.Literals([
-			"attempt.setup",
+			"attempt.before",
 			"sandbox.prepare",
 			"agent.ensure",
 			"eval.run",
 			"sandbox.command",
-			"attempt.teardown"
+			"attempt.after"
 		]),
 		sequence: Schema.Number,
 		stdout: CommandStreamSchema,
@@ -6711,8 +6819,10 @@ var EffectiveCostSchema = Schema.Struct({
 	}),
 	state: Schema.Literals(["complete", "partial"])
 });
+var { modelSlot: persistedModelSlot, ...publicAdapterCallFields } = AdapterUsageCallSchema.fields;
 var AdapterUsageInspectionCallSchema = Schema.Struct({
-	...AdapterUsageCallSchema.fields,
+	...publicAdapterCallFields,
+	modelRole: persistedModelSlot,
 	effectiveCost: Schema.NullOr(EffectiveCostSchema)
 });
 var EffectiveCostTotalSchema = Schema.Struct({
@@ -6753,14 +6863,14 @@ var ModelGroupTokenTotalSchema = Schema.Struct({
 var ConfiguredModelsSchema = Schema.Union([Schema.Struct({ state: Schema.Literal("not-recorded") }), Schema.Struct({
 	state: Schema.Literal("available"),
 	bindings: Schema.Array(Schema.Struct({
-		modelSlot: AdapterUsageModelSlotSchema,
+		modelRole: AdapterUsageModelSlotSchema,
 		model: Schema.NullOr(Schema.String.check(Schema.isPattern(/\S/u))),
 		reasoningEffort: Schema.NullOr(Schema.String.check(Schema.isPattern(/\S/u))),
 		recordedCalls: Schema.NullOr(RecordedCallCountSchema)
-	})).check(Schema.makeFilter((bindings) => bindings.length <= 64 && bindings.every((entry, index) => index === 0 || bindings[index - 1].modelSlot < entry.modelSlot)))
+	})).check(Schema.makeFilter((bindings) => bindings.length <= 64 && bindings.every((entry, index) => index === 0 || bindings[index - 1].modelRole < entry.modelRole)))
 })]);
 var ModelGroupSchema = Schema.Struct({
-	modelSlot: Schema.NullOr(AdapterUsageModelSlotSchema),
+	modelRole: Schema.NullOr(AdapterUsageModelSlotSchema),
 	provider: AdapterUsageCallSchema.fields.provider,
 	model: AdapterUsageCallSchema.fields.model,
 	recordedCalls: RecordedCallCountSchema.check(Schema.isGreaterThan(0)),
@@ -6813,17 +6923,17 @@ var TotalUsageCostsSchema = Schema.Struct({
 	missingSources: Schema.Array(Schema.Literals(["application", "judge"])).check(Schema.makeFilter((sources) => sources.length <= 2 && sources.every((source, index) => index === 0 || sources[index - 1] === "application" && source === "judge")))
 }).check(Schema.makeFilter((value) => value.state === (value.missingSources.length === 0 ? "complete" : value.values.length === 0 ? "unavailable" : "partial") && value.values.every((entry, index) => index === 0 || value.values[index - 1].currency < entry.currency)));
 var ExperimentCostSummarySchema = Schema.Struct({
-	scope: Schema.Literal("latest-recorded-slots"),
+	scope: Schema.Literal("latest-recorded-attempts"),
 	totalCosts: TotalUsageCostsSchema,
 	coverage: Schema.Struct({
-		selectedSlotCount: NonNegativeSafeIntegerSchema$2,
-		resolvedSlotCount: NonNegativeSafeIntegerSchema$2,
+		plannedAttemptCount: NonNegativeSafeIntegerSchema$2,
+		recordedAttemptCount: NonNegativeSafeIntegerSchema$2,
 		originAttemptCount: NonNegativeSafeIntegerSchema$2,
 		completeAttemptCount: NonNegativeSafeIntegerSchema$2,
 		partialAttemptCount: NonNegativeSafeIntegerSchema$2,
 		unavailableAttemptCount: NonNegativeSafeIntegerSchema$2,
-		unresolvedSlotCount: NonNegativeSafeIntegerSchema$2
-	}).check(Schema.makeFilter((value) => value.selectedSlotCount === value.resolvedSlotCount + value.unresolvedSlotCount && value.originAttemptCount <= value.resolvedSlotCount && value.originAttemptCount === value.completeAttemptCount + value.partialAttemptCount + value.unavailableAttemptCount))
+		missingAttemptCount: NonNegativeSafeIntegerSchema$2
+	}).check(Schema.makeFilter((value) => value.plannedAttemptCount === value.recordedAttemptCount + value.missingAttemptCount && value.originAttemptCount <= value.recordedAttemptCount && value.originAttemptCount === value.completeAttemptCount + value.partialAttemptCount + value.unavailableAttemptCount))
 });
 var JudgeCostTotalsSchema = EffectiveCostTotalSchema.check(Schema.makeFilter((costs) => costs.totalCalls <= 4e3 && costs.source === null === (costs.values.length === 0) && costs.values.every((entry, index) => entry.coveredCalls <= entry.reportedCalls + entry.estimatedCalls && entry.reportedCalls + entry.estimatedCalls <= costs.totalCalls && entry.source === (entry.reportedCalls === 0 ? "estimated" : entry.estimatedCalls === 0 ? "reported" : "mixed") && (index === 0 || costs.values[index - 1].currency < entry.currency)) && costs.values.reduce((count, entry) => count + entry.reportedCalls + entry.estimatedCalls, 0) <= costs.totalCalls && (costs.state !== "unavailable" || costs.values.length === 0) && (costs.state !== "complete" || costs.values.reduce((count, entry) => count + entry.coveredCalls, 0) === costs.totalCalls)));
 var JudgeUsageUnavailableSchema = Schema.Union([Schema.Struct({
@@ -6967,8 +7077,8 @@ var RunOverviewMemberLimitationSchema = Schema.Union([
 	})
 ]);
 var RunOverviewMemberSchema = Schema.Struct({
-	...RecordSlotIdentitySchema.fields,
-	publication: RunSlotPublicationSchema,
+	...PlannedAttemptSchema.fields,
+	publication: PlannedAttemptPublicationSchema,
 	state: Schema.Literals([
 		"executed",
 		"carried",
@@ -6992,20 +7102,20 @@ var RunOverviewMemberSchema = Schema.Struct({
 	limitations: Schema.Array(RunOverviewMemberLimitationSchema)
 });
 var RunOverviewLocatedLimitationSchema = Schema.Struct({
-	slotId: Schema.String,
+	plannedAttemptId: Schema.String,
 	locator: Schema.NullOr(Schema.String),
 	limitation: RunOverviewMemberLimitationSchema
 });
 var InspectionRunResultSchema = Schema.Struct({
 	value: InspectionRunValueSchema,
-	members: Schema.Array(MemberDocumentSchema),
-	attempts: Schema.Array(AttemptDocumentSchema),
+	members: Schema.Array(InspectionMemberSchema),
+	attempts: Schema.Array(InspectionAttemptCoreSchema),
 	denominator: Schema.Struct({
 		expected: Schema.Number,
 		published: Schema.Number,
 		missing: Schema.Number
 	}),
-	slots: Schema.Array(RunOverviewMemberSchema),
+	plannedAttempts: Schema.Array(RunOverviewMemberSchema),
 	coverage: Schema.Struct({
 		state: RunOverviewStateSchema,
 		expectedMemberCount: Schema.Number,
@@ -7130,7 +7240,7 @@ var InspectionProjectGapReasonSchema = Schema.Literals([
 	"timeout-exceeded",
 	"adoption-unproven"
 ]);
-var InspectionProjectSlotSchema = Schema.Union([Schema.Struct({
+var InspectionProjectPlannedAttemptSchema = Schema.Union([Schema.Struct({
 	...PositionFields,
 	state: Schema.Literal("reuse"),
 	sourceRunId: RunIdSchema,
@@ -7157,14 +7267,14 @@ var InspectionProjectResultSchema = Schema.Struct({
 		covered: NonNegativeSafeIntegerSchema$2,
 		gaps: NonNegativeSafeIntegerSchema$2
 	}),
-	slots: Schema.Array(InspectionProjectSlotSchema),
+	plannedAttempts: Schema.Array(InspectionProjectPlannedAttemptSchema),
 	...InspectionOverviewResultSchema.fields,
 	history: Schema.Array(Schema.Struct({
 		...PositionFields,
 		sourceRunId: RunIdSchema,
 		locator: Schema.NullOr(LocatorSchema)
 	}))
-}).check(Schema.makeFilter((value) => value.coverage.covered + value.coverage.gaps === value.coverage.expected && value.slots.length === value.coverage.expected && value.slots.filter((slot) => slot.state === "reuse").length === value.coverage.covered));
+}).check(Schema.makeFilter((value) => value.coverage.covered + value.coverage.gaps === value.coverage.expected && value.plannedAttempts.length === value.coverage.expected && value.plannedAttempts.filter((slot) => slot.state === "reuse").length === value.coverage.covered));
 //#endregion
 //#region src/inspection/protocol.ts
 var AttemptLocatorSchema = Schema.String.pipe(Schema.refine((value) => ATTEMPT_LOCATOR_PATTERN.test(value), {
@@ -7245,7 +7355,7 @@ var ArtifactsResultSchema = Schema.Union([Schema.Struct({ state: Schema.Literal(
 var ComparisonMemberSchema = Schema.Struct({
 	key: Schema.String,
 	runId: Schema.String,
-	slotId: Schema.String,
+	plannedAttemptId: Schema.String,
 	evalId: Schema.String,
 	attemptOrdinal: Schema.Number,
 	locator: Schema.String
@@ -7257,11 +7367,11 @@ var RunsCompareResultSchema = Schema.Struct({
 		"paired"
 	]),
 	left: Schema.Struct({
-		runs: Schema.Array(RunDocumentSchema),
+		runs: Schema.Array(InspectionRunCoreSchema),
 		members: Schema.Array(ComparisonMemberSchema)
 	}),
 	right: Schema.Struct({
-		runs: Schema.Array(RunDocumentSchema),
+		runs: Schema.Array(InspectionRunCoreSchema),
 		members: Schema.Array(ComparisonMemberSchema)
 	}),
 	exactMemberSet: Schema.Boolean,
@@ -7298,6 +7408,7 @@ var inspectionProtocolRegistry = Object.freeze({
 		result: { experiment: InspectionExperimentResultSchema },
 		factKinds: [
 			"core",
+			"attempt-note",
 			"assertions",
 			"attempt-cost",
 			"agent-turns",
@@ -7327,7 +7438,11 @@ var inspectionProtocolRegistry = Object.freeze({
 	"attempt.get": spec({
 		request: operation("attempt.get", { locator: AttemptLocatorSchema }),
 		result: { attempt: InspectionAttemptResultSchema },
-		factKinds: ["core", "assertions"]
+		factKinds: [
+			"core",
+			"assertions",
+			"attempt-note"
+		]
 	}),
 	"attempt.assertion.detail": spec({
 		request: operation("attempt.assertion.detail", {
@@ -7577,7 +7692,6 @@ function decodeInspectionOperation(input) {
 }
 //#endregion
 //#region src/inspection/codec.ts
-var inspectionCodecErrors = /* @__PURE__ */ new WeakSet();
 var strictDecodeRequest = Schema.decodeUnknownResult(InspectionRequestSchema, { onExcessProperty: "error" });
 function decodeInspectionRequest(input) {
 	const decoded = strictDecodeRequest(input);
@@ -7586,83 +7700,6 @@ function decodeInspectionRequest(input) {
 		reason: String(decoded.failure)
 	})) : Result.succeed(decoded.success);
 }
-function closeInspectionJson(value) {
-	const seen = /* @__PURE__ */ new Set();
-	const close = (current, path) => {
-		if (current === null || typeof current === "string" || typeof current === "boolean") return current;
-		if (typeof current === "number") return Number.isFinite(current) ? current : invalidResult(path, "numbers must be finite");
-		if (Array.isArray(current)) {
-			if (seen.has(current)) return invalidResult(path, "cyclic arrays are not encodable");
-			seen.add(current);
-			const output = [];
-			for (const [index, entry] of current.entries()) {
-				const closed = close(entry, [...path, String(index)]);
-				if (isCodecError(closed)) return closed;
-				output.push(closed);
-			}
-			seen.delete(current);
-			return Object.freeze(output);
-		}
-		if (typeof current !== "object" || current === null) return invalidResult(path, `unsupported ${typeof current} value`);
-		if (seen.has(current)) return invalidResult(path, "cyclic objects are not encodable");
-		const prototype = Object.getPrototypeOf(current);
-		if (prototype !== Object.prototype && prototype !== null) return invalidResult(path, "only plain objects may cross Inspection delivery");
-		seen.add(current);
-		const output = Object.create(null);
-		for (const key of Object.keys(current).sort(compareCodeUnits$1)) {
-			const closed = close(Reflect.get(current, key), [...path, key]);
-			if (isCodecError(closed)) return closed;
-			output[key] = closed;
-		}
-		seen.delete(current);
-		return Object.freeze(output);
-	};
-	return close(value, []);
-}
-function invalidResult(path, reason) {
-	const error = Object.freeze({
-		code: "inspection-result-invalid",
-		reason: `${path.length === 0 ? "$" : path.join(".")}: ${reason}`
-	});
-	inspectionCodecErrors.add(error);
-	return error;
-}
-function isInspectionCodecError(value) {
-	return isObject(value) && inspectionCodecErrors.has(value);
-}
-var isCodecError = isInspectionCodecError;
-function isObject(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value);
-}
-function compareCodeUnits$1(left, right) {
-	return left < right ? -1 : left > right ? 1 : 0;
-}
-//#endregion
-//#region src/inspection/canonical.ts
-function canonicalInspectionJson(value) {
-	const decoded = decodeInspectionDocument(value);
-	if (!decoded.success) return Result.fail({
-		code: "inspection-result-invalid",
-		reason: decoded.reason
-	});
-	const closed = closeInspectionJson(decoded.value);
-	if (isInspectionCodecError(closed)) return Result.fail(closed);
-	return Result.succeed(`${encode(closed)}\n`);
-}
-function canonicalJsonValue(value) {
-	const closed = closeInspectionJson(value);
-	if (isInspectionCodecError(closed)) return Result.fail(closed);
-	return Result.succeed(`${encode(closed)}\n`);
-}
-function encode(value) {
-	if (value === null || typeof value === "boolean" || typeof value === "number" || typeof value === "string") return JSON.stringify(value);
-	if (Array.isArray(value)) return `[${value.map(encode).join(",")}]`;
-	const record = value;
-	return `{${Object.keys(record).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${encode(record[key])}`).join(",")}}`;
-}
-function compareCodeUnits(left, right) {
-	return left < right ? -1 : left > right ? 1 : 0;
-}
 //#endregion
 //#region src/assertions/score-match-audit.ts
 var protocol = "niceeval.score-match-audit/v1";
@@ -8416,4 +8453,4 @@ function validEvidenceIds(value, empty = false) {
 	return Array.isArray(value) && value.length <= 16384 && (empty || value.length > 0) && value.every((id) => text(id) && id.trim() !== "" && new TextEncoder().encode(id).length <= 128 && !/[\u0000-\u001f\u007f]/u.test(id)) && new Set(value).size === value.length;
 }
 //#endregion
-export { INSPECTION_BEHAVIOR_VERSION, INSPECTION_OPERATION_IDS, InspectionDiscoveryDocumentSchema, InspectionDocumentSchema, InspectionExplanationDocumentSchema, InspectionFailureDocumentSchema, InspectionOperationIdSchema, InspectionRequestSchema, InspectionSuccessDocumentSchema, QUERY_PROTOCOL, canonicalInspectionJson, canonicalJsonValue, closeInspectionJson, decodeInspectionDocument, decodeInspectionOperation, decodeInspectionRequest, inspectionProtocolRegistry, narrowInspectionExplanation, narrowInspectionSuccess, readScoreMatchAudit };
+export { INSPECTION_BEHAVIOR_VERSION, INSPECTION_OPERATION_IDS, InspectionDiscoveryDocumentSchema, InspectionDocumentSchema, InspectionExplanationDocumentSchema, InspectionFailureDocumentSchema, InspectionOperationIdSchema, InspectionRequestSchema, InspectionSuccessDocumentSchema, QUERY_PROTOCOL, decodeInspectionDocument, decodeInspectionOperation, decodeInspectionRequest, inspectionProtocolRegistry, narrowInspectionExplanation, narrowInspectionSuccess, readScoreMatchAudit };

package/dist/inspection/query.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.decodeInspectionOperation = decodeInspectionOperation;
-// @concord-file ne-eval-inspection-query
+// @concord-file
 // @concord-implements docs/feature/inspection/README.md
 const effect_1 = require("effect");
 const protocol_ts_1 = require("./protocol.cjs");

package/dist/inspection/query.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"query.js","sourceRoot":"","sources":["query.ts"],"names":[],"mappings":";;AAgCA,8DASC;AAzCD,yCAAyC;AACzC,wDAAwD;AACxD,mCAAwC;AACxC,+CAA+K;AAC/K,6DAAsD;AAwBtD,MAAM,aAAa,GAAG,eAAM,CAAC,mBAAmB,CAAC,qCAAuB,EAAE;IACtE,gBAAgB,EAAE,OAAO;CAC5B,CAAC,CAAC;AACH,mEAAmE;AACnE,SAAgB,yBAAyB,CAAC,KAAc;IACpD,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,QAAQ,EAAE,mCAAc,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,OAAO,eAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QAC5B,CAAC,CAAC,eAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YACxB,cAAc,EAAE,gBAAyB;YACzC,IAAI,EAAE,4BAAqC;YAC3C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAClC,CAAC,CAAC;QACH,CAAC,CAAC,eAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AACpD,CAAC","sourcesContent":["// @concord-file ne-eval-inspection-query\n// @concord-implements docs/feature/inspection/README.md\nimport { Result, Schema } from \"effect\";\nimport { InspectionRequestSchema, type InspectionOperation, type InspectionOperationFor, type InspectionOperationId, type InspectionSuccessDocumentFor, } from \"./protocol.ts\";\nimport { QUERY_PROTOCOL } from \"./protocol-values.ts\";\n/** Presentation-neutral query boundary implemented by local or remote Inspection adapters. */\nexport interface InspectionQuery {\n    inspect<Kind extends InspectionOperationId>(operation: InspectionOperationFor<Kind>): Promise<InspectionSuccessDocumentFor<Kind>>;\n}\nexport interface InspectionQueryProtocolError {\n    readonly classification: \"protocol/query\";\n    readonly code: \"inspection-request-invalid\" | \"inspection-source-invalid\" | \"inspection-record-integrity-failure\" | \"inspection-operation-failed\" | \"inspection-result-invalid\" | \"evidence-budget-exceeded\" | \"restart-required\";\n    readonly reason: string;\n    readonly identity?: {\n        readonly runId: string;\n    };\n    readonly cause?: unknown;\n}\nexport interface InspectionQuerySelectionError {\n    readonly classification: \"selection\";\n    readonly code: \"inspection-selection-missing\";\n    readonly reason: string;\n    readonly cause?: unknown;\n}\nexport type InspectionQueryError = InspectionQueryProtocolError | InspectionQuerySelectionError;\nexport interface InspectionOperationDecodeError extends InspectionQueryProtocolError {\n    readonly code: \"inspection-request-invalid\";\n}\nconst decodeRequest = Schema.decodeUnknownResult(InspectionRequestSchema, {\n    onExcessProperty: \"error\",\n});\n/** Runtime-safe operation builder for route and adapter inputs. */\nexport function decodeInspectionOperation(input: unknown): Result.Result<InspectionOperation, InspectionOperationDecodeError> {\n    const decoded = decodeRequest({ protocol: QUERY_PROTOCOL, operation: input });\n    return Result.isFailure(decoded)\n        ? Result.fail(Object.freeze({\n            classification: \"protocol/query\" as const,\n            code: \"inspection-request-invalid\" as const,\n            reason: String(decoded.failure),\n        }))\n        : Result.succeed(decoded.success.operation);\n}\n"]}
+{"version":3,"file":"query.js","sourceRoot":"","sources":["query.ts"],"names":[],"mappings":";;AAgCA,8DASC;AAzCD,gBAAgB;AAChB,wDAAwD;AACxD,mCAAwC;AACxC,+CAA+K;AAC/K,6DAAsD;AAwBtD,MAAM,aAAa,GAAG,eAAM,CAAC,mBAAmB,CAAC,qCAAuB,EAAE;IACtE,gBAAgB,EAAE,OAAO;CAC5B,CAAC,CAAC;AACH,mEAAmE;AACnE,SAAgB,yBAAyB,CAAC,KAAc;IACpD,MAAM,OAAO,GAAG,aAAa,CAAC,EAAE,QAAQ,EAAE,mCAAc,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,OAAO,eAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QAC5B,CAAC,CAAC,eAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YACxB,cAAc,EAAE,gBAAyB;YACzC,IAAI,EAAE,4BAAqC;YAC3C,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAClC,CAAC,CAAC;QACH,CAAC,CAAC,eAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AACpD,CAAC","sourcesContent":["// @concord-file\n// @concord-implements docs/feature/inspection/README.md\nimport { Result, Schema } from \"effect\";\nimport { InspectionRequestSchema, type InspectionOperation, type InspectionOperationFor, type InspectionOperationId, type InspectionSuccessDocumentFor, } from \"./protocol.ts\";\nimport { QUERY_PROTOCOL } from \"./protocol-values.ts\";\n/** Presentation-neutral query boundary implemented by local or remote Inspection adapters. */\nexport interface InspectionQuery {\n    inspect<Kind extends InspectionOperationId>(operation: InspectionOperationFor<Kind>): Promise<InspectionSuccessDocumentFor<Kind>>;\n}\nexport interface InspectionQueryProtocolError {\n    readonly classification: \"protocol/query\";\n    readonly code: \"inspection-request-invalid\" | \"inspection-source-invalid\" | \"inspection-record-integrity-failure\" | \"inspection-operation-failed\" | \"inspection-result-invalid\" | \"evidence-budget-exceeded\" | \"restart-required\";\n    readonly reason: string;\n    readonly identity?: {\n        readonly runId: string;\n    };\n    readonly cause?: unknown;\n}\nexport interface InspectionQuerySelectionError {\n    readonly classification: \"selection\";\n    readonly code: \"inspection-selection-missing\";\n    readonly reason: string;\n    readonly cause?: unknown;\n}\nexport type InspectionQueryError = InspectionQueryProtocolError | InspectionQuerySelectionError;\nexport interface InspectionOperationDecodeError extends InspectionQueryProtocolError {\n    readonly code: \"inspection-request-invalid\";\n}\nconst decodeRequest = Schema.decodeUnknownResult(InspectionRequestSchema, {\n    onExcessProperty: \"error\",\n});\n/** Runtime-safe operation builder for route and adapter inputs. */\nexport function decodeInspectionOperation(input: unknown): Result.Result<InspectionOperation, InspectionOperationDecodeError> {\n    const decoded = decodeRequest({ protocol: QUERY_PROTOCOL, operation: input });\n    return Result.isFailure(decoded)\n        ? Result.fail(Object.freeze({\n            classification: \"protocol/query\" as const,\n            code: \"inspection-request-invalid\" as const,\n            reason: String(decoded.failure),\n        }))\n        : Result.succeed(decoded.success.operation);\n}\n"]}