niceeval 0.19.0 → 0.19.1-canary.102

package/dist/view/function-runtime.mjs

@@ -650,40 +650,45 @@ var AssertionCollectionReceiptSchema = Schema.Struct({
 	identifier: "AssertionCollectionReceipt",
 	description: "a collection receipt whose counts and completion boundary agree"
 }));
-var AssertionFactValueSchema = Schema.suspend(() => Schema.Union([
-	Schema.Struct({
-		kind: Schema.Literal("unavailable"),
-		reason: Schema.Literals([
-			"not-recorded",
-			"not-declared",
-			"source-unavailable"
-		])
-	}),
-	Schema.Struct({
-		kind: Schema.Literal("value"),
-		value: Schema.Union([
-			Schema.Null,
-			Schema.Boolean,
-			Schema.Finite,
-			BoundedJsonStringSchema
-		])
-	}),
-	Schema.Struct({
-		kind: Schema.Literal("text"),
-		text: BoundedJsonStringSchema
-	}),
-	Schema.Struct({
-		kind: Schema.Literal("list"),
-		items: Schema.Array(AssertionFactValueSchema)
-	}),
-	Schema.Struct({
-		kind: Schema.Literal("fields"),
-		fields: Schema.Array(Schema.Struct({
-			label: BoundedJsonStringSchema,
-			value: AssertionFactValueSchema
-		}))
-	})
-]));
+/** Share the fact shape while letting each evidence owner apply its string budget. */
+function createAssertionFactValueSchema(text) {
+	const fact = Schema.suspend(() => Schema.Union([
+		Schema.Struct({
+			kind: Schema.Literal("unavailable"),
+			reason: Schema.Literals([
+				"not-recorded",
+				"not-declared",
+				"source-unavailable"
+			])
+		}),
+		Schema.Struct({
+			kind: Schema.Literal("value"),
+			value: Schema.Union([
+				Schema.Null,
+				Schema.Boolean,
+				Schema.Finite,
+				text
+			])
+		}),
+		Schema.Struct({
+			kind: Schema.Literal("text"),
+			text
+		}),
+		Schema.Struct({
+			kind: Schema.Literal("list"),
+			items: Schema.Array(fact)
+		}),
+		Schema.Struct({
+			kind: Schema.Literal("fields"),
+			fields: Schema.Array(Schema.Struct({
+				label: text,
+				value: fact
+			}))
+		})
+	]));
+	return fact;
+}
+var AssertionFactValueSchema = createAssertionFactValueSchema(BoundedJsonStringSchema);
 var MatcherRelationStatusSchema = Schema.Union([Schema.Struct({ state: Schema.Literal("exact") }), Schema.Struct({
 	state: Schema.Literal("unavailable"),
 	reason: Schema.Literals([
@@ -4023,7 +4028,7 @@ function validateObservabilityCollection(collection) {
 }
 //#endregion
 //#region src/inspection/protocol-values.ts
-var QUERY_PROTOCOL = "niceeval.query/v2";
+var QUERY_PROTOCOL = "niceeval.query/v3";
 /** Selector semantics revision; independent from the wire protocol major. */
 var INSPECTION_BEHAVIOR_VERSION = "2.0.0";
 //#endregion
@@ -4400,6 +4405,7 @@ var ProjectedLocatorSchema = Schema.Union([Schema.Struct({
 	kind: Schema.Literal("event"),
 	eventId: Schema.String
 })]);
+var SourceRowFactValueSchema = createAssertionFactValueSchema(Schema.String);
 var MatcherRowSchema = Schema.Struct({
 	kind: Schema.Literals([
 		"tool",
@@ -4414,7 +4420,7 @@ var MatcherRowSchema = Schema.Struct({
 		"historical"
 	]),
 	summary: Schema.String,
-	detail: AssertionFactValueSchema,
+	detail: SourceRowFactValueSchema,
 	locator: Schema.optional(ProjectedLocatorSchema),
 	evaluation: Schema.Struct({
 		result: Schema.Literals([
@@ -7218,8 +7224,7 @@ var AggregateFields = {
 	coverage: Schema.Array(OverviewCoverageSchema),
 	issues: Schema.Array(OverviewIssueSchema)
 };
-var OverviewPublishedMemberSchema = Schema.Struct({
-	...InspectionPublishedPublicationSchema.fields,
+var PlannedAttemptMetricsSchema = Schema.Struct({
 	verdict: Schema.NullOr(Schema.Literals([
 		"passed",
 		"failed",
@@ -7231,6 +7236,13 @@ var OverviewPublishedMemberSchema = Schema.Struct({
 	durationMs: MetricSchema,
 	tokens: MetricSchema
 });
+var PlannedAttemptMemberSchema = Schema.Struct({
+	plannedAttemptId: PlannedAttemptSchema.fields.plannedAttemptId,
+	evalId: PlannedAttemptSchema.fields.evalId,
+	attemptOrdinal: PlannedAttemptSchema.fields.attemptOrdinal,
+	publication: PlannedAttemptPublicationSchema,
+	...PlannedAttemptMetricsSchema.fields
+});
 var InspectionExecutionValueSchema = Schema.Union([
 	Schema.Struct({
 		state: Schema.Literal("available"),
@@ -7241,16 +7253,9 @@ var InspectionExecutionValueSchema = Schema.Union([
 ]);
 var InspectionLabelsSchema = Schema.Record(Schema.String, InspectionExecutionValueSchema);
 var OverviewMemberSchema = Schema.Struct({
+	...PlannedAttemptMemberSchema.fields,
 	runId: Schema.String,
-	plannedAttemptId: Schema.String,
-	evalId: Schema.String,
-	attemptOrdinal: Schema.Number,
-	labels: InspectionLabelsSchema,
-	publication: Schema.Union([
-		RunPendingPublicationSchema,
-		OverviewPublishedMemberSchema,
-		RunAbsentPublicationSchema
-	])
+	labels: InspectionLabelsSchema
 });
 var OverviewGroupSchema = Schema.Struct({
 	groupPath: Schema.Array(Schema.String),
@@ -8398,7 +8403,7 @@ var RunOverviewMemberLimitationSchema = Schema.Union([
 ]);
 var RunOverviewMemberSchema = Schema.Struct({
 	...PlannedAttemptSchema.fields,
-	publication: PlannedAttemptPublicationSchema,
+	...PlannedAttemptMemberSchema.fields,
 	state: Schema.Literals([
 		"executed",
 		"carried",
@@ -8407,7 +8412,6 @@ var RunOverviewMemberSchema = Schema.Struct({
 		"interrupted",
 		"pending"
 	]),
-	locator: Schema.NullOr(Schema.String),
 	relation: Schema.NullOr(Schema.Literals(["origin", "reference"])),
 	outcome: Schema.NullOr(Schema.Literals([
 		"completed",
@@ -8415,8 +8419,6 @@ var RunOverviewMemberSchema = Schema.Struct({
 		"cancelled",
 		"interrupted"
 	])),
-	verdict: VerdictSchema,
-	score: Schema.NullOr(InspectionScoredValueSchema),
 	coverage: RunOverviewCoverageSchema,
 	usage: RunOverviewUsageSchema,
 	limitations: Schema.Array(RunOverviewMemberLimitationSchema)
@@ -14676,6 +14678,45 @@ function foldRecordedAttemptVerdict(input) {
 	return foldVerdict(recordedAttemptFacts(input));
 }
 //#endregion
+//#region src/inspection/verdict.ts
+function attemptVerdict(resolved, assertions = readInspectionAssertions(resolved)) {
+	if (assertions.state === "available") return foldRecordedAttemptVerdict({
+		outcome: resolved.attempt.outcome,
+		assertions: assertions.value
+	});
+	if (assertions.state !== "failed" || assertions.attachment === void 0) return null;
+	if (resolved.attempt.outcome === "errored" || resolved.attempt.outcome === "interrupted") return "errored";
+	if (containsRequiredAssertionError(assertions.attachment.value)) return "errored";
+	return containsFailedDecision(assertions.attachment.value) ? "failed" : null;
+}
+function containsFailedDecision(value) {
+	return assertionWireEntries(value)?.some((entry) => {
+		if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return false;
+		const decision = entry.decision;
+		if (typeof decision !== "object" || decision === null || Array.isArray(decision)) return false;
+		const record = decision;
+		return record.result === "mismatched" && record.gate === "failed";
+	}) ?? false;
+}
+function containsRequiredAssertionError(value) {
+	return assertionWireEntries(value)?.some((entry) => {
+		if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return false;
+		const record = entry;
+		const decision = record.decision;
+		const policy = record.policy;
+		const requirement = typeof policy === "object" && policy !== null && !Array.isArray(policy) ? policy.requirement : void 0;
+		if (typeof decision !== "object" || decision === null || Array.isArray(decision) || typeof requirement !== "object" || requirement === null || Array.isArray(requirement)) return false;
+		const decisionRecord = decision;
+		const requirementRecord = requirement;
+		return (decisionRecord.result === "unavailable" || decisionRecord.result === "errored") && decisionRecord.gate === "unavailable" && requirementRecord.state === "available" && requirementRecord.value === "required";
+	}) ?? false;
+}
+function assertionWireEntries(value) {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) return void 0;
+	const entries = value["entries-data"];
+	return Array.isArray(entries) ? entries : void 0;
+}
+//#endregion
 //#region src/inspection/limits.ts
 /** Maximum UTF-8 JSON bytes for any one fixed operation result field. */
 var INSPECTION_RESULT_BYTE_LIMIT = 524288;
@@ -16689,7 +16730,7 @@ function aggregateInspectionCurrentSlots(inputs) {
 				coverage: Object.freeze([]),
 				issues: Object.freeze([]),
 				ref: null
-			}) : analyzeAttempt(recorded.resolved, recorded.target, recorded.member, recorded.slot)
+			}) : analyzeAttempt(recorded.resolved, recorded.target.run.runId, recorded.member, recorded.slot)
 		});
 	}));
 }
@@ -16721,14 +16762,14 @@ function selectLatestSlots(runs, resolutionRuns = runs) {
 			slot,
 			member,
 			resolved,
-			analysis: analyzeAttempt(resolved, target, member, slot)
+			analysis: analyzeAttempt(resolved, target.run.runId, member, slot)
 		});
 	}));
 }
 function compareOccurrence(left, right) {
 	return left.run.completedAt - right.run.completedAt || left.run.startedAt - right.run.startedAt || compareText(left.run.runId, right.run.runId);
 }
-function analyzeAttempt(resolved, target, member, slot) {
+function analyzeAttempt(resolved, runId, member, slot) {
 	if (resolved === void 0) {
 		const hasReference = member?.attempt !== void 0 && member.attempt !== null;
 		return Object.freeze({
@@ -16740,7 +16781,7 @@ function analyzeAttempt(resolved, target, member, slot) {
 			durationMs: unavailableOperationalMetric(),
 			tokens: unavailableOperationalMetric(),
 			coverage: Object.freeze([]),
-			issues: Object.freeze([overviewIssue(hasReference ? "attempt-origin-missing" : "attempt-not-observed", target.run.runId, slot)]),
+			issues: Object.freeze([overviewIssue(hasReference ? "attempt-origin-missing" : "attempt-not-observed", runId, slot)]),
 			ref: null
 		});
 	}
@@ -16750,7 +16791,7 @@ function analyzeAttempt(resolved, target, member, slot) {
 	if (assertions.state !== "available") return Object.freeze({
 		assertionsState: assertions.state,
 		evaluationKind: null,
-		verdict: null,
+		verdict: attemptVerdict(resolved, assertions),
 		score: emptyAttemptScore(),
 		costUSD: costUSDOf(resolved),
 		...operational,
@@ -16798,10 +16839,7 @@ function analyzeAttempt(resolved, target, member, slot) {
 	return Object.freeze({
 		assertionsState: "available",
 		evaluationKind: hasPoints ? "points" : "pass",
-		verdict: foldRecordedAttemptVerdict({
-			outcome: resolved.attempt.outcome,
-			assertions: assertions.value
-		}),
+		verdict: attemptVerdict(resolved, assertions),
 		score: Object.freeze({
 			hasPoints,
 			earned,
@@ -16826,14 +16864,14 @@ function makeCell(slots, labelKeys) {
 		...aggregate(slots, scoreForCell(slots)),
 		members: Object.freeze(slots.flatMap((selected) => {
 			if (selected.target === void 0 || selected.slot === void 0) return [];
-			selected.resolved;
 			return [Object.freeze({
 				runId: selected.target.run.runId,
 				plannedAttemptId: selected.slot.slotId,
 				evalId: selected.slot.evalId,
 				attemptOrdinal: selected.slot.attemptOrdinal,
 				labels: inspectionLabels([selected], labelKeys),
-				publication: overviewPublication(selected)
+				publication: overviewPublication(selected),
+				...memberMetrics(selected)
 			})];
 		}))
 	});
@@ -16958,13 +16996,22 @@ function overviewPublication(selected) {
 		attemptId: member.attempt.attemptId,
 		attemptLocator: resolved.locator,
 		originRunId: member.attempt.originRunId,
-		originPlannedAttemptId: resolved.attempt.slotId,
+		originPlannedAttemptId: resolved.attempt.slotId
+	});
+}
+/** Shared metrics for one planned position, including missing and unresolved publications. */
+function projectPlannedAttemptMetrics(runId, slot, member, resolved) {
+	return memberMetrics({ analysis: analyzeAttempt(resolved, runId, member, slot) });
+}
+var decodeMemberMetrics = Schema.decodeUnknownSync(PlannedAttemptMetricsSchema);
+function memberMetrics(selected) {
+	return Object.freeze(decodeMemberMetrics({
 		verdict: selected.analysis.verdict,
 		score: scoreForMember(selected),
-		costUSD: costForSlots([selected]),
+		costUSD: costForAnalyses([selected]),
 		durationMs: metricForSlots([selected], "durationMs", "ms", "average"),
 		tokens: metricForSlots([selected], "tokens", "tokens", "sum")
-	});
+	}));
 }
 function executionValue(values) {
 	if (new Set(values).size !== 1) return Object.freeze({ state: "mixed" });
@@ -17036,7 +17083,9 @@ function costForSlots(slots) {
 		const key = slot.resolved?.locator ?? (slot.target === void 0 ? `current:${slot.experimentId}:${slot.evalId}:${slot.attemptOrdinal}` : `slot:${slot.target.run.runId}:${slot.slot.slotId}`);
 		if (!unique.has(key)) unique.set(key, slot);
 	}
-	const subjects = [...unique.values()];
+	return costForAnalyses([...unique.values()]);
+}
+function costForAnalyses(subjects) {
 	const valued = subjects.filter(({ analysis }) => analysis.costUSD.value !== null);
 	const sources = new Set(valued.flatMap(({ analysis }) => analysis.costUSD.source === null ? [] : [analysis.costUSD.source]));
 	const hasReported = sources.has("reported") || sources.has("mixed");
@@ -20213,6 +20262,11 @@ function runDetail(_all, selectedResource) {
 			reason: binding.reason
 		}) : Object.freeze({ state: "pending" });
 		const targetMember = bindings.get(slot.plannedAttemptId);
+		const resolved = targetMember?.attempt == null ? void 0 : resolveInspectionResourceMemberAttempt(selectedResource, targetMember);
+		const metrics = projectPlannedAttemptMetrics(value.runId, {
+			...slot,
+			slotId: slot.plannedAttemptId
+		}, targetMember, resolved);
 		if (targetMember === void 0) {
 			if (publication.state !== "pending" && publication.state !== "absent") throw new Error(`Terminal Run ${value.runId} has no binding or absence for Planned attempt ${slot.plannedAttemptId}`);
 			const limitation = Object.freeze({
@@ -20222,12 +20276,10 @@ function runDetail(_all, selectedResource) {
 			return Object.freeze({ member: Object.freeze({
 				...slot,
 				publication,
+				...metrics,
 				state: limitation.state,
-				locator: null,
 				relation: null,
 				outcome: null,
-				verdict: null,
-				score: null,
 				coverage: unavailableCoverage(),
 				usage: unavailableUsage(),
 				limitations: Object.freeze([limitation])
@@ -20241,18 +20293,15 @@ function runDetail(_all, selectedResource) {
 			return Object.freeze({ member: Object.freeze({
 				...slot,
 				publication,
+				...metrics,
 				state: targetMember.action,
-				locator: null,
 				relation: null,
 				outcome: null,
-				verdict: null,
-				score: null,
 				coverage: unavailableCoverage(),
 				usage: unavailableUsage(),
 				limitations: Object.freeze([limitation])
 			}) });
 		}
-		const resolved = resolveInspectionResourceMemberAttempt(selectedResource, targetMember);
 		if (resolved === void 0) {
 			const limitation = Object.freeze({
 				kind: "attempt-unresolved",
@@ -20262,12 +20311,10 @@ function runDetail(_all, selectedResource) {
 			return Object.freeze({ member: Object.freeze({
 				...slot,
 				publication,
+				...metrics,
 				state: targetMember.action,
-				locator: null,
 				relation: null,
 				outcome: null,
-				verdict: null,
-				score: null,
 				coverage: unavailableCoverage(),
 				usage: unavailableUsage(),
 				limitations: Object.freeze([limitation])
@@ -20290,12 +20337,10 @@ function runDetail(_all, selectedResource) {
 			member: Object.freeze({
 				...slot,
 				publication,
+				...metrics,
 				state: targetMember.action,
-				locator: resolved.locator,
 				relation: resolved.attempt.originRunId === value.runId ? "origin" : "reference",
 				outcome: resolved.attempt.outcome,
-				verdict: attemptVerdict(resolved, assertions),
-				score: assertionScore(assertions.state === "available" ? assertions.value.entries : []),
 				coverage: Object.freeze({
 					state: coverageState,
 					facts: coverageFacts,
@@ -20319,11 +20364,11 @@ function runDetail(_all, selectedResource) {
 	const members = Object.freeze(projected.map(({ member }) => member));
 	const locatedLimitations = Object.freeze(members.flatMap((member) => member.limitations.map((limitation) => Object.freeze({
 		plannedAttemptId: member.plannedAttemptId,
-		locator: member.locator,
+		locator: member.publication.state === "published" ? member.publication.attemptLocator : null,
 		limitation
 	}))));
 	const usageResults = projected.flatMap(({ usage }) => usage === void 0 ? [] : [usage]);
-	const observed = members.filter(({ locator }) => locator !== null).length;
+	const observed = members.filter(({ relation }) => relation !== null).length;
 	const incompleteUsage = observed !== members.length || usageResults.some(({ state }) => state !== "complete");
 	const coverageLimitations = Object.freeze(locatedLimitations.filter(({ limitation }) => limitation.kind !== "usage"));
 	const usageLimitations = Object.freeze(locatedLimitations.filter(({ limitation }) => limitation.kind !== "coverage"));
@@ -20363,16 +20408,6 @@ function aggregateRunOverviewState(states) {
 	if (states.every((state) => state === "invalid")) return "invalid";
 	return "partial";
 }
-function attemptVerdict(resolved, assertions = readInspectionAssertions(resolved)) {
-	if (assertions.state === "available") return foldRecordedAttemptVerdict({
-		outcome: resolved.attempt.outcome,
-		assertions: assertions.value
-	});
-	if (assertions.state !== "failed" || assertions.attachment === void 0) return null;
-	if (resolved.attempt.outcome === "errored" || resolved.attempt.outcome === "interrupted") return "errored";
-	if (containsRequiredAssertionError(assertions.attachment.value)) return "errored";
-	return containsFailedDecision(assertions.attachment.value) ? "failed" : null;
-}
 function assertionScore(entries) {
 	let possible = 0;
 	let earned = 0;
@@ -20469,33 +20504,6 @@ function attemptLimitations(resolved, entries) {
 function isAttemptCoverageChannel(value) {
 	return value === "events" || value === "actions" || value === "messages" || value === "usage" || value === "status" || value === "data";
 }
-function containsFailedDecision(value) {
-	return assertionWireEntries(value)?.some((entry) => {
-		if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return false;
-		const decision = entry.decision;
-		if (typeof decision !== "object" || decision === null || Array.isArray(decision)) return false;
-		const record = decision;
-		return record.result === "mismatched" && record.gate === "failed";
-	}) ?? false;
-}
-function containsRequiredAssertionError(value) {
-	return assertionWireEntries(value)?.some((entry) => {
-		if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return false;
-		const record = entry;
-		const decision = record.decision;
-		const policy = record.policy;
-		const requirement = typeof policy === "object" && policy !== null && !Array.isArray(policy) ? policy.requirement : void 0;
-		if (typeof decision !== "object" || decision === null || Array.isArray(decision) || typeof requirement !== "object" || requirement === null || Array.isArray(requirement)) return false;
-		const decisionRecord = decision;
-		const requirementRecord = requirement;
-		return (decisionRecord.result === "unavailable" || decisionRecord.result === "errored") && decisionRecord.gate === "unavailable" && requirementRecord.state === "available" && requirementRecord.value === "required";
-	}) ?? false;
-}
-function assertionWireEntries(value) {
-	if (typeof value !== "object" || value === null || Array.isArray(value)) return void 0;
-	const entries = value["entries-data"];
-	return Array.isArray(entries) ? entries : void 0;
-}
 function comparisonDocument(source, all, operation) {
 	const left = selectRuns(all, operation.leftRunIds);
 	const right = selectRuns(all, operation.rightRunIds);

package/dist/view/http-protocol.d.cts

@@ -18,7 +18,7 @@ export type ViewGenerationCommitRequest = Schema.Schema.Type<typeof ViewGenerati
 export declare const ViewInspectionHttpRequestSchema: Schema.Struct<{
     readonly generationId: Schema.String;
     readonly request: Schema.Struct<{
-        readonly protocol: Schema.Literal<"niceeval.query/v2">;
+        readonly protocol: Schema.Literal<"niceeval.query/v3">;
         readonly operation: Schema.Union<(Schema.Struct<{
             readonly kind: Schema.Literal<"project.get">;
         } & {

package/dist/view/http-protocol.d.mts

@@ -18,7 +18,7 @@ export type ViewGenerationCommitRequest = Schema.Schema.Type<typeof ViewGenerati
 export declare const ViewInspectionHttpRequestSchema: Schema.Struct<{
     readonly generationId: Schema.String;
     readonly request: Schema.Struct<{
-        readonly protocol: Schema.Literal<"niceeval.query/v2">;
+        readonly protocol: Schema.Literal<"niceeval.query/v3">;
         readonly operation: Schema.Union<(Schema.Struct<{
             readonly kind: Schema.Literal<"project.get">;
         } & {

package/dist/view/http-protocol.d.ts

@@ -18,7 +18,7 @@ export type ViewGenerationCommitRequest = Schema.Schema.Type<typeof ViewGenerati
 export declare const ViewInspectionHttpRequestSchema: Schema.Struct<{
     readonly generationId: Schema.String;
     readonly request: Schema.Struct<{
-        readonly protocol: Schema.Literal<"niceeval.query/v2">;
+        readonly protocol: Schema.Literal<"niceeval.query/v3">;
         readonly operation: Schema.Union<(Schema.Struct<{
             readonly kind: Schema.Literal<"project.get">;
         } & {