niceeval 0.18.0 → 0.19.0

package/dist/inspection/select.cjs

@@ -3,11 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.InspectionOperationError = void 0;
 exports.selectInspectionOperation = selectInspectionOperation;
 exports.explainInspectionOperation = explainInspectionOperation;
-// @concord-file ne-eval-inspection-selector
+// @concord-file
 // @concord-implements docs/feature/inspection/README.md
 // @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
 // @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
 // @concord-implements docs/feature/insight/README.md
+const planned_attempt_ts_1 = require("./planned-attempt.cjs");
+const notes_ts_1 = require("./notes.cjs");
 const effect_1 = require("effect");
 const artifacts_ts_1 = require("./artifacts.cjs");
 const project_input_ts_1 = require("./project-input.cjs");
@@ -39,7 +41,7 @@ const RUN_LIST_PAGE_SIZE = 100;
 const ATTACHMENT_COLLECTION_PAGE_SIZE = 64;
 const ATTACHMENT_CONTENT_METADATA_LIMIT = 64;
 const RUN_SELECTION_LIMIT = 64;
-// @concord-code ne-eval-inspection-select-operation
+// @concord-code
 // @concord-implements docs/feature/inspection/README.md
 // @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
 // @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
@@ -47,7 +49,7 @@ function selectInspectionOperation(facts, operation) {
     return evaluateInspectionSuccess(operation.kind, () => selectOperation(facts, operation));
 }
 /** Internal CLI explanation over the same selected facts and cutoff. */
-// @concord-code ne-eval-inspection-explain-operation
+// @concord-code
 // @concord-implements docs/feature/inspection/README.md
 // @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
 function explainInspectionOperation(facts, operation) {
@@ -103,14 +105,14 @@ function selectOperation(source, operation) {
             const experimentCells = Object.freeze(overview.cells.filter(({ experimentId }) => experimentId === operation.experimentId));
             const usageAttempts = new Map();
             let unresolvedAttemptCount = 0;
-            let selectedSlotCount = 0;
-            let resolvedSlotCount = 0;
+            let plannedAttemptCount = 0;
+            let recordedAttemptCount = 0;
             const origins = new Map(selected.map((origin) => [origin.run.runId, {
                     origin, attempts: new Map(origin.attempts.map((attempt) => [attempt.attemptId, attempt])),
                 }]));
             for (const cell of experimentCells) {
                 for (const member of cell.members) {
-                    selectedSlotCount++;
+                    plannedAttemptCount++;
                     if (member.publication.state !== "published") {
                         unresolvedAttemptCount++;
                         continue;
@@ -125,7 +127,7 @@ function selectOperation(source, operation) {
                     }
                     const resolved = Object.freeze({ origin, attempt,
                         locator: (0, attempt_locator_ts_1.encodeAttemptLocator)(attempt.attemptId), targets: Object.freeze([]) });
-                    resolvedSlotCount++;
+                    recordedAttemptCount++;
                     usageAttempts.set(`${resolved.attempt.originRunId}\u0000${resolved.attempt.attemptId}`, resolved);
                 }
             }
@@ -140,18 +142,20 @@ function selectOperation(source, operation) {
                 });
             }));
             const costSummary = Object.freeze({
-                scope: "latest-recorded-slots",
+                scope: "latest-recorded-attempts",
                 totalCosts: (0, judge_usage_projection_ts_1.combineTotalUsageCosts)(allAttempts.map(({ usage }) => usage.totalCosts), unresolvedAttemptCount > 0),
-                coverage: Object.freeze({ selectedSlotCount, resolvedSlotCount,
+                coverage: Object.freeze({ plannedAttemptCount, recordedAttemptCount,
                     originAttemptCount: allAttempts.length,
                     completeAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "complete").length,
                     partialAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "partial").length,
                     unavailableAttemptCount: allAttempts.filter(({ usage }) => usage.totalCosts.state === "unavailable").length,
-                    unresolvedSlotCount: unresolvedAttemptCount }),
+                    missingAttemptCount: unresolvedAttemptCount }),
             });
             let attempts = Object.freeze(allAttempts.slice(0, 64));
+            const notes = (0, notes_ts_1.aggregateAttemptNotes)(orderedUsageAttempts);
             const experimentResult = () => Object.freeze({
                 experiment, cells: experimentCells, costSummary,
+                notes,
                 modelUsage: Object.freeze({
                     attempts, totalAttemptCount: orderedUsageAttempts.length,
                     omittedAttemptCount: orderedUsageAttempts.length - attempts.length,
@@ -518,12 +522,12 @@ function inspectionRunValue(run) {
         ...(resource.completedAt === undefined
             ? {}
             : { completedAt: Date.parse(resource.completedAt) }),
-        expectedSlots: published?.run.expectedSlots ?? resource.slots.map((slot) => Object.freeze({
+        plannedAttempts: (published?.run.expectedSlots ?? resource.slots.map((slot) => Object.freeze({
             slotId: slot.slotId,
             evalId: slot.evalId,
             attemptOrdinal: slot.attemptOrdinal,
             executionIdentityDigest: slot.executionIdentityDigest,
-        })),
+        }))).map(planned_attempt_ts_1.projectPlannedAttempt),
     }));
 }
 function supportingRunFacts(_source, run) {
@@ -655,10 +659,10 @@ function compactAssertionInspection(value) {
 function attemptDetail(resolved) {
     const assertions = (0, facts_ts_1.readInspectionAssertions)(resolved);
     return Object.freeze({
-        core: resolved.attempt,
+        core: (0, planned_attempt_ts_1.projectAttemptCore)(resolved.attempt),
         locator: resolved.locator,
-        originRun: resolved.origin.run,
-        targets: resolved.targets.map(({ run, member }) => Object.freeze({ runId: run.run.runId, member })),
+        originRun: (0, planned_attempt_ts_1.projectRunCore)(resolved.origin.run),
+        targets: resolved.targets.map(({ run, member }) => Object.freeze({ runId: run.run.runId, member: (0, planned_attempt_ts_1.projectMember)(member) })),
         evidence: assertions.state === "available"
             ? Object.freeze({
                 state: "available",
@@ -669,6 +673,7 @@ function attemptDetail(resolved) {
                 state: assertions.state,
                 entryCount: 0,
             }),
+        notes: (0, notes_ts_1.readAttemptNotes)(resolved),
         assertions: assertionIndex(assertions),
         sections: attemptSections(resolved),
         verdict: attemptVerdict(resolved, assertions),
@@ -763,22 +768,22 @@ function runDetail(_all, selectedResource) {
         facts: Object.freeze([]),
         limitations: Object.freeze([]),
     });
-    const projected = value.expectedSlots.map((slot) => {
-        const binding = publications.get(slot.slotId);
+    const projected = value.plannedAttempts.map((slot) => {
+        const binding = publications.get(slot.plannedAttemptId);
         if (binding === undefined)
-            throw new Error(`Run ${value.runId} has no Slot ${slot.slotId}`);
+            throw new Error(`Run ${value.runId} has no Planned attempt ${slot.plannedAttemptId}`);
         const publication = binding.state === "published"
             ? Object.freeze({
                 state: "published", action: binding.action, attemptId: binding.attemptId,
-                attemptLocator: binding.attemptLocator, originRunId: binding.originRunId, originSlotId: binding.originSlotId,
+                attemptLocator: binding.attemptLocator, originRunId: binding.originRunId, originPlannedAttemptId: binding.originSlotId,
             })
             : binding.state === "absent"
                 ? Object.freeze({ state: "absent", reason: binding.reason })
                 : Object.freeze({ state: "pending" });
-        const targetMember = bindings.get(slot.slotId);
+        const targetMember = bindings.get(slot.plannedAttemptId);
         if (targetMember === undefined) {
             if (publication.state !== "pending" && publication.state !== "absent") {
-                throw new Error(`Terminal Run ${value.runId} has no binding or absence for Slot ${slot.slotId}`);
+                throw new Error(`Terminal Run ${value.runId} has no binding or absence for Planned attempt ${slot.plannedAttemptId}`);
             }
             const limitation = Object.freeze({
                 kind: "member-not-observed",
@@ -898,7 +903,7 @@ function runDetail(_all, selectedResource) {
     });
     const members = Object.freeze(projected.map(({ member }) => member));
     const locatedLimitations = Object.freeze(members.flatMap((member) => member.limitations.map((limitation) => Object.freeze({
-        slotId: member.slotId,
+        plannedAttemptId: member.plannedAttemptId,
         locator: member.locator,
         limitation,
     }))));
@@ -910,14 +915,14 @@ function runDetail(_all, selectedResource) {
     const usageLimitations = Object.freeze(locatedLimitations.filter(({ limitation }) => limitation.kind !== "coverage"));
     return Object.freeze({
         value,
-        members: selectedResource.members,
-        attempts: selected?.attempts ?? Object.freeze([]),
+        members: selectedResource.members.map(planned_attempt_ts_1.projectMember),
+        attempts: selected?.attempts.map(planned_attempt_ts_1.projectAttemptCore) ?? Object.freeze([]),
         denominator: Object.freeze({
             expected: selectedResource.resource.expected,
             published: selectedResource.resource.published,
             missing: selectedResource.resource.missing,
         }),
-        slots: members,
+        plannedAttempts: members,
         coverage: Object.freeze({
             state: aggregateRunOverviewState(members.map(({ coverage }) => coverage.state)),
             expectedMemberCount: members.length,
@@ -1167,8 +1172,8 @@ function comparisonDocument(source, all, operation) {
         ], issues),
         comparison: boundedJson(Object.freeze({
             mode: operation.mode,
-            left: Object.freeze({ runs: left.selected.map(({ run }) => run), members: leftMembers }),
-            right: Object.freeze({ runs: right.selected.map(({ run }) => run), members: rightMembers }),
+            left: Object.freeze({ runs: left.selected.map(({ run }) => (0, planned_attempt_ts_1.projectRunCore)(run)), members: leftMembers }),
+            right: Object.freeze({ runs: right.selected.map(({ run }) => (0, planned_attempt_ts_1.projectRunCore)(run)), members: rightMembers }),
             exactMemberSet: exact,
             pairs: operation.mode === "side-by-side" ? Object.freeze([]) : Object.freeze(pairs),
         })),
@@ -1184,7 +1189,7 @@ function comparisonMembers(all, runs) {
             return [];
         return [Object.freeze({
                 key: `${run.run.experimentId}\u0000${slot.evalId}\u0000${slot.attemptOrdinal}`,
-                runId: run.run.runId, slotId: slot.slotId, evalId: slot.evalId, attemptOrdinal: slot.attemptOrdinal,
+                runId: run.run.runId, plannedAttemptId: slot.slotId, evalId: slot.evalId, attemptOrdinal: slot.attemptOrdinal,
                 locator: resolved.locator,
             })];
     }));