niceeval 0.15.1-canary.76 → 0.16.1

package/dist/inspection/results.js

@@ -1,38 +1,40 @@
 import { createRequire as __niceevalCreateRequire } from "node:module";
 const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
 const __niceevalCanonical = __niceevalRequire("./results.cjs");
-const __niceevalExport0 = __niceevalCanonical["InspectionAttemptDiffResultSchema"];
-export { __niceevalExport0 as InspectionAttemptDiffResultSchema };
-const __niceevalExport1 = __niceevalCanonical["InspectionAttemptResultSchema"];
-export { __niceevalExport1 as InspectionAttemptResultSchema };
-const __niceevalExport2 = __niceevalCanonical["InspectionAttemptTimingResultSchema"];
-export { __niceevalExport2 as InspectionAttemptTimingResultSchema };
-const __niceevalExport3 = __niceevalCanonical["InspectionAttemptUsageResultSchema"];
-export { __niceevalExport3 as InspectionAttemptUsageResultSchema };
-const __niceevalExport4 = __niceevalCanonical["InspectionExperimentResultSchema"];
-export { __niceevalExport4 as InspectionExperimentResultSchema };
-const __niceevalExport5 = __niceevalCanonical["InspectionOverviewResultSchema"];
-export { __niceevalExport5 as InspectionOverviewResultSchema };
-const __niceevalExport6 = __niceevalCanonical["InspectionRunListResultSchema"];
-export { __niceevalExport6 as InspectionRunListResultSchema };
-const __niceevalExport7 = __niceevalCanonical["InspectionRunOverviewResultSchema"];
-export { __niceevalExport7 as InspectionRunOverviewResultSchema };
-const __niceevalExport8 = __niceevalCanonical["InspectionRunResultSchema"];
-export { __niceevalExport8 as InspectionRunResultSchema };
-const __niceevalExport9 = __niceevalCanonical["InspectionRunSummaryResultSchema"];
-export { __niceevalExport9 as InspectionRunSummaryResultSchema };
-const __niceevalExport10 = __niceevalCanonical["InspectionRunValueSchema"];
-export { __niceevalExport10 as InspectionRunValueSchema };
-const __niceevalExport11 = __niceevalCanonical["InspectionScoredValueSchema"];
-export { __niceevalExport11 as InspectionScoredValueSchema };
-const __niceevalExport12 = __niceevalCanonical["InspectionSourcesResultSchema"];
-export { __niceevalExport12 as InspectionSourcesResultSchema };
-const __niceevalExport13 = __niceevalCanonical["InspectionTraceDetailItemSchema"];
-export { __niceevalExport13 as InspectionTraceDetailItemSchema };
-const __niceevalExport14 = __niceevalCanonical["InspectionTraceDetailResultSchema"];
-export { __niceevalExport14 as InspectionTraceDetailResultSchema };
-const __niceevalExport15 = __niceevalCanonical["InspectionTraceItemSchema"];
-export { __niceevalExport15 as InspectionTraceItemSchema };
-const __niceevalExport16 = __niceevalCanonical["InspectionTraceResultSchema"];
-export { __niceevalExport16 as InspectionTraceResultSchema };
+const __niceevalExport0 = __niceevalCanonical["ExecutionDisplayPreviewSchema"];
+export { __niceevalExport0 as ExecutionDisplayPreviewSchema };
+const __niceevalExport1 = __niceevalCanonical["InspectionAttemptDiffResultSchema"];
+export { __niceevalExport1 as InspectionAttemptDiffResultSchema };
+const __niceevalExport2 = __niceevalCanonical["InspectionAttemptResultSchema"];
+export { __niceevalExport2 as InspectionAttemptResultSchema };
+const __niceevalExport3 = __niceevalCanonical["InspectionAttemptTimingResultSchema"];
+export { __niceevalExport3 as InspectionAttemptTimingResultSchema };
+const __niceevalExport4 = __niceevalCanonical["InspectionAttemptUsageResultSchema"];
+export { __niceevalExport4 as InspectionAttemptUsageResultSchema };
+const __niceevalExport5 = __niceevalCanonical["InspectionExperimentResultSchema"];
+export { __niceevalExport5 as InspectionExperimentResultSchema };
+const __niceevalExport6 = __niceevalCanonical["InspectionOverviewResultSchema"];
+export { __niceevalExport6 as InspectionOverviewResultSchema };
+const __niceevalExport7 = __niceevalCanonical["InspectionRunListResultSchema"];
+export { __niceevalExport7 as InspectionRunListResultSchema };
+const __niceevalExport8 = __niceevalCanonical["InspectionRunOverviewResultSchema"];
+export { __niceevalExport8 as InspectionRunOverviewResultSchema };
+const __niceevalExport9 = __niceevalCanonical["InspectionRunResultSchema"];
+export { __niceevalExport9 as InspectionRunResultSchema };
+const __niceevalExport10 = __niceevalCanonical["InspectionRunSummaryResultSchema"];
+export { __niceevalExport10 as InspectionRunSummaryResultSchema };
+const __niceevalExport11 = __niceevalCanonical["InspectionRunValueSchema"];
+export { __niceevalExport11 as InspectionRunValueSchema };
+const __niceevalExport12 = __niceevalCanonical["InspectionScoredValueSchema"];
+export { __niceevalExport12 as InspectionScoredValueSchema };
+const __niceevalExport13 = __niceevalCanonical["InspectionSourcesResultSchema"];
+export { __niceevalExport13 as InspectionSourcesResultSchema };
+const __niceevalExport14 = __niceevalCanonical["InspectionTraceDetailItemSchema"];
+export { __niceevalExport14 as InspectionTraceDetailItemSchema };
+const __niceevalExport15 = __niceevalCanonical["InspectionTraceDetailResultSchema"];
+export { __niceevalExport15 as InspectionTraceDetailResultSchema };
+const __niceevalExport16 = __niceevalCanonical["InspectionTraceItemSchema"];
+export { __niceevalExport16 as InspectionTraceItemSchema };
+const __niceevalExport17 = __niceevalCanonical["InspectionTraceResultSchema"];
+export { __niceevalExport17 as InspectionTraceResultSchema };
 //# sourceMappingURL=results.js.map

package/dist/inspection/select.cjs

@@ -3,9 +3,18 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.InspectionOperationError = void 0;
 exports.selectInspectionOperation = selectInspectionOperation;
 exports.explainInspectionOperation = explainInspectionOperation;
+// @concord-file ne-eval-inspection-selector
+// @concord-implements docs/feature/inspection/README.md
+// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
+// @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
+// @concord-implements docs/feature/insight/README.md
 const effect_1 = require("effect");
 const artifacts_ts_1 = require("./artifacts.cjs");
+const project_input_ts_1 = require("./project-input.cjs");
+const project_ts_1 = require("./project.cjs");
+const project_result_ts_1 = require("./project-result.cjs");
 const artifact_list_ts_1 = require("./artifact-list.cjs");
+const judge_usage_projection_ts_1 = require("../o11y/judge-usage-projection.cjs");
 const attempt_locator_ts_1 = require("../attempt-locator.cjs");
 const verdict_ts_1 = require("../eval/record/verdict.cjs");
 const current_ts_1 = require("../record/family/current.cjs");
@@ -30,10 +39,17 @@ const RUN_LIST_PAGE_SIZE = 100;
 const ATTACHMENT_COLLECTION_PAGE_SIZE = 64;
 const ATTACHMENT_CONTENT_METADATA_LIMIT = 64;
 const RUN_SELECTION_LIMIT = 64;
+// @concord-code ne-eval-inspection-select-operation
+// @concord-implements docs/feature/inspection/README.md
+// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
+// @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
 function selectInspectionOperation(facts, operation) {
     return evaluateInspectionSuccess(operation.kind, () => selectOperation(facts, operation));
 }
 /** Internal CLI explanation over the same selected facts and cutoff. */
+// @concord-code ne-eval-inspection-explain-operation
+// @concord-implements docs/feature/inspection/README.md
+// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
 function explainInspectionOperation(facts, operation) {
     return evaluateInspectionOperation(operation.kind, () => {
         if (operation.kind === "runs.list") {
@@ -58,6 +74,13 @@ function explainInspectionOperation(facts, operation) {
 }
 function selectOperation(source, operation) {
     switch (operation.kind) {
+        case "project.get": {
+            const input = requireCurrentProject(source, operation.experimentIds);
+            return Object.freeze({
+                ...resultMetadata(source, operation.kind, input.runs, [], [], locators(input.runs)),
+                project: decodeRequiredResult(operation.kind, project_result_ts_1.InspectionProjectResultSchema, (0, project_ts_1.selectInspectionProject)(input, operation.experimentIds)),
+            });
+        }
         case "overview.get": {
             const requested = operation.runIds ?? [];
             const selection = requested.length === 0
@@ -76,13 +99,69 @@ function selectOperation(source, operation) {
             if (experiment === undefined) {
                 throw selectionMissing(operation.kind, `Experiment ${operation.experimentId} was not found`);
             }
-            return Object.freeze({
-                ...resultMetadata(source, operation.kind, selected, [], [], locators(selected)),
-                experiment: decodeRequiredResult(operation.kind, results_ts_1.InspectionExperimentResultSchema, Object.freeze({
-                    experiment,
-                    cells: Object.freeze(overview.cells.filter(({ experimentId }) => experimentId === operation.experimentId)),
-                })),
+            const experimentCells = Object.freeze(overview.cells.filter(({ experimentId }) => experimentId === operation.experimentId));
+            const usageAttempts = new Map();
+            let unresolvedAttemptCount = 0;
+            let selectedSlotCount = 0;
+            let resolvedSlotCount = 0;
+            for (const cell of experimentCells) {
+                for (const member of cell.members) {
+                    selectedSlotCount++;
+                    if (member.publication.state !== "published") {
+                        unresolvedAttemptCount++;
+                        continue;
+                    }
+                    const identity = member.publication;
+                    const origin = selected.find(({ run }) => run.runId === identity.originRunId);
+                    const attempt = origin?.attempts.find((candidate) => candidate.attemptId === identity.attemptId &&
+                        candidate.originRunId === identity.originRunId);
+                    if (origin === undefined || attempt === undefined) {
+                        unresolvedAttemptCount++;
+                        continue;
+                    }
+                    const resolved = Object.freeze({ origin, attempt,
+                        locator: (0, attempt_locator_ts_1.encodeAttemptLocator)(attempt.attemptId), targets: Object.freeze([]) });
+                    resolvedSlotCount++;
+                    usageAttempts.set(`${resolved.attempt.originRunId}\u0000${resolved.attempt.attemptId}`, resolved);
+                }
+            }
+            const orderedUsageAttempts = [...usageAttempts.values()].sort((left, right) => left.attempt.originRunId.localeCompare(right.attempt.originRunId) || left.locator.localeCompare(right.locator));
+            const metadata = resultMetadata(source, operation.kind, selected, [], [], locators(selected));
+            const allAttempts = Object.freeze(orderedUsageAttempts.map((resolved) => {
+                const { state, configuredModels, modelGroups, judgeUsage, totalCosts, totals } = (0, trace_ts_1.projectAttemptUsage)(traceAttachments(resolved));
+                return Object.freeze({
+                    locator: resolved.locator,
+                    originRunId: resolved.attempt.originRunId,
+                    usage: Object.freeze({ state, configuredModels, modelGroups, judgeUsage: (0, judge_usage_projection_ts_1.summarizeJudgeUsage)(judgeUsage), totalCosts, totals }),
+                });
+            }));
+            const costSummary = Object.freeze({
+                scope: "latest-recorded-slots",
+                totalCosts: (0, judge_usage_projection_ts_1.combineTotalUsageCosts)(allAttempts.map(({ usage }) => usage.totalCosts), unresolvedAttemptCount > 0),
+                coverage: Object.freeze({ selectedSlotCount, resolvedSlotCount,
+                    originAttemptCount: allAttempts.length,
+                    completeAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "complete").length,
+                    partialAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "partial").length,
+                    unavailableAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "unavailable").length,
+                    unresolvedSlotCount: unresolvedAttemptCount }),
+            });
+            let attempts = Object.freeze(allAttempts.slice(0, 64));
+            const experimentResult = () => Object.freeze({
+                experiment, cells: experimentCells, costSummary,
+                modelUsage: Object.freeze({
+                    attempts, totalAttemptCount: orderedUsageAttempts.length,
+                    omittedAttemptCount: orderedUsageAttempts.length - attempts.length,
+                    unresolvedAttemptCount,
+                }),
             });
+            while ((0, bytes_ts_1.utf8ByteLength)(JSON.stringify({ ...metadata, experiment: experimentResult() })) > limits_ts_1.INSPECTION_EXPERIMENT_RESULT_BYTE_LIMIT &&
+                attempts.length > 0) {
+                attempts = Object.freeze(attempts.slice(0, -1));
+            }
+            return requireInspectionResultBudget(operation.kind, Object.freeze({
+                ...metadata,
+                experiment: decodeRequiredResult(operation.kind, results_ts_1.InspectionExperimentResultSchema, experimentResult()),
+            }));
         }
         case "runs.list": return runsListDocument(source, operation);
         case "run.get": {
@@ -130,7 +209,7 @@ function selectOperation(source, operation) {
             }
             return Object.freeze({
                 ...baseDocument(source, operation.kind, attemptRuns(resolved), [], [], [operation.locator]),
-                assertion: boundedJson(assertion),
+                assertion: boundedJson(assertion, limits_ts_1.INSPECTION_ASSERTION_DETAIL_BYTE_LIMIT),
             });
         }
         case "attempt.assertion.image": {
@@ -148,11 +227,13 @@ function selectOperation(source, operation) {
                     ...(operation.traceId === undefined ? {} : { traceId: operation.traceId }),
                     ...(operation.sourceId === undefined ? {} : { sourceId: operation.sourceId }),
                     ...(operation.actorId === undefined ? {} : { actorId: operation.actorId }),
+                    ...(operation.eventType === undefined ? {} : { eventType: operation.eventType }),
                     ...(operation.continuation === undefined ? {} : { continuation: operation.continuation }),
                 }), {
                     ...(operation.traceId === undefined ? {} : { traceId: operation.traceId }),
                     ...(operation.sourceId === undefined ? {} : { sourceId: operation.sourceId }),
                     ...(operation.actorId === undefined ? {} : { actorId: operation.actorId }),
+                    ...(operation.eventType === undefined ? {} : { eventType: operation.eventType }),
                     ...(operation.continuation === undefined ? {} : { continuation: operation.continuation }),
                 })),
             });
@@ -182,10 +263,17 @@ function selectOperation(source, operation) {
         }
         case "attempt.usage": {
             const resolved = requireAttemptFromSource(source, operation.kind, operation.locator);
-            return Object.freeze({
-                ...resultMetadata(source, operation.kind, attemptRuns(resolved), [], [], [operation.locator]),
-                usage: decodeRequiredResult(operation.kind, results_ts_1.InspectionAttemptUsageResultSchema, (0, trace_ts_1.projectAttemptUsage)(traceAttachments(resolved))),
-            });
+            const metadata = resultMetadata(source, operation.kind, attemptRuns(resolved), [], [], [operation.locator]);
+            let usage = (0, trace_ts_1.projectAttemptUsage)(traceAttachments(resolved));
+            while ((0, bytes_ts_1.utf8ByteLength)(JSON.stringify({ ...metadata, usage })) > limits_ts_1.INSPECTION_RESULT_BYTE_LIMIT &&
+                (usage.judgeUsage.state === "complete" || usage.judgeUsage.state === "partial") && usage.judgeUsage.calls.length > 0) {
+                usage = Object.freeze({ ...usage,
+                    judgeUsage: (0, judge_usage_projection_ts_1.takeJudgeUsagePreview)(usage.judgeUsage, usage.judgeUsage.calls.length - 1) });
+            }
+            return requireInspectionResultBudget(operation.kind, Object.freeze({
+                ...metadata,
+                usage: decodeRequiredResult(operation.kind, results_ts_1.InspectionAttemptUsageResultSchema, usage),
+            }));
         }
         case "attempt.diff": {
             const resolved = requireAttemptFromSource(source, operation.kind, operation.locator);
@@ -357,6 +445,8 @@ function decodeContinuation(token) {
     }
 }
 function loadForOperation(source, operation) {
+    if (operation.kind === "project.get")
+        return { selected: requireCurrentProject(source, operation.experimentIds).runs, requestedRunIds: [], missingRunIds: [] };
     if (operation.kind === "overview.get") {
         const requested = operation.runIds ?? [];
         if (requested.length === 0)
@@ -400,6 +490,18 @@ function loadForOperation(source, operation) {
         missingRunIds: Object.freeze([]),
     };
 }
+function requireCurrentProject(source, experimentIds) {
+    const input = (0, project_input_ts_1.projectInput)(source);
+    if (source.kind !== "project-record" || input === undefined || input.cutoffIdentity !== source.cutoff().identity)
+        throw new InspectionOperationError({
+            code: "current-target-unavailable", operation: "project.get",
+            reason: "Current project declarations are unavailable. Use overview.get or an exact run.get to read recorded results.",
+        });
+    for (const id of experimentIds ?? [])
+        if (!input.target.experiments.some((experiment) => experiment.experimentId === id))
+            throw selectionMissing("project.get", `Current Experiment ${JSON.stringify(id)} was not found.`);
+    return input;
+}
 function loadRuns(source, runIds) {
     const uniqueRunIds = [...new Set(runIds)];
     if (uniqueRunIds.length > RUN_SELECTION_LIMIT) {
@@ -498,7 +600,9 @@ function attemptRuns(resolved) {
 }
 function traceAttachments(resolved) {
     return Object.freeze({
+        models: resolved.origin.run.context.execution.models,
         adapterUsage: (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.adapterUsage.family),
+        judgeUsage: (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.judgeUsage.family),
         agentTurns: (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.agentTurns.family),
         turnContexts: (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.turnContexts.family),
         sandboxCommands: (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.sandboxCommands.family),
@@ -1177,6 +1281,14 @@ function decodeRequiredResult(operation, schema, input) {
     }
     return decoded.success;
 }
+function requireInspectionResultBudget(operation, value) {
+    const byteLimit = operation === "experiment.get" ? limits_ts_1.INSPECTION_EXPERIMENT_RESULT_BYTE_LIMIT : limits_ts_1.INSPECTION_RESULT_BYTE_LIMIT;
+    if ((0, bytes_ts_1.utf8ByteLength)(JSON.stringify(value)) > byteLimit) {
+        throw new InspectionOperationError({ code: "evidence-budget-exceeded", operation,
+            reason: "Inspection result cannot fit in the result byte limit." });
+    }
+    return value;
+}
 function locators(runs) {
     return Object.freeze(runs.flatMap(({ attempts }) => attempts.map(({ attemptId }) => (0, attempt_locator_ts_1.encodeAttemptLocator)(attemptId))));
 }
@@ -1189,15 +1301,15 @@ function closeJson(value) {
         throw closed;
     return closed;
 }
-function boundedJson(value) {
+function boundedJson(value, byteLimit = limits_ts_1.INSPECTION_RESULT_BYTE_LIMIT) {
     const closed = closeJson(value);
     const byteLength = (0, bytes_ts_1.utf8ByteLength)(JSON.stringify(closed));
-    if (byteLength <= limits_ts_1.INSPECTION_RESULT_BYTE_LIMIT)
+    if (byteLength <= byteLimit)
         return closed;
     return closeJson(Object.freeze({
         state: "omitted",
         reason: "inspection-result-byte-limit",
-        byteLimit: limits_ts_1.INSPECTION_RESULT_BYTE_LIMIT,
+        byteLimit,
     }));
 }
 function arraysEqual(left, right) {