niceeval 0.17.1 → 0.19.0

package/dist/inspection/lifecycle.d.cts

@@ -0,0 +1,4 @@
+/** Public projections use current lifecycle names while Record keeps its stable encoding. */
+type PublicLifecyclePhase<Phase extends string> = Phase extends `${infer Owner}.setup` ? `${Owner}.before` : Phase extends `${infer Owner}.teardown` ? `${Owner}.after` : Phase;
+export declare function projectLifecyclePhase<Phase extends string>(phase: Phase): PublicLifecyclePhase<Phase>;
+export {};

package/dist/inspection/lifecycle.d.mts

@@ -0,0 +1,4 @@
+/** Public projections use current lifecycle names while Record keeps its stable encoding. */
+type PublicLifecyclePhase<Phase extends string> = Phase extends `${infer Owner}.setup` ? `${Owner}.before` : Phase extends `${infer Owner}.teardown` ? `${Owner}.after` : Phase;
+export declare function projectLifecyclePhase<Phase extends string>(phase: Phase): PublicLifecyclePhase<Phase>;
+export {};

package/dist/inspection/lifecycle.d.ts

@@ -0,0 +1,4 @@
+/** Public projections use current lifecycle names while Record keeps its stable encoding. */
+type PublicLifecyclePhase<Phase extends string> = Phase extends `${infer Owner}.setup` ? `${Owner}.before` : Phase extends `${infer Owner}.teardown` ? `${Owner}.after` : Phase;
+export declare function projectLifecyclePhase<Phase extends string>(phase: Phase): PublicLifecyclePhase<Phase>;
+export {};

package/dist/inspection/lifecycle.js

@@ -0,0 +1,6 @@
+import { createRequire as __niceevalCreateRequire } from "node:module";
+const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
+const __niceevalCanonical = __niceevalRequire("./lifecycle.cjs");
+const __niceevalExport0 = __niceevalCanonical["projectLifecyclePhase"];
+export { __niceevalExport0 as projectLifecyclePhase };
+//# sourceMappingURL=lifecycle.js.map

package/dist/inspection/lifecycle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lifecycle.js","sources":[],"names":[],"mappings":""}

package/dist/inspection/notes.cjs

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readAttemptNotes = readAttemptNotes;
+exports.aggregateAttemptNotes = aggregateAttemptNotes;
+const effect_1 = require("effect");
+const schema_ts_1 = require("../record/family/attempt-note/schema.cjs");
+const current_ts_1 = require("../record/family/current.cjs");
+const facts_ts_1 = require("./facts.cjs");
+function readAttemptNotes(resolved) {
+    const attachment = (0, facts_ts_1.attemptAttachment)(resolved, current_ts_1.NiceEvalRecordAttachments.attemptNotes.family);
+    return attachment === undefined ? [] : effect_1.Schema.decodeUnknownSync(schema_ts_1.AttemptNotesSchema)(attachment.value);
+}
+function aggregateAttemptNotes(attempts) {
+    const groups = new Map();
+    for (const attempt of attempts)
+        for (const note of readAttemptNotes(attempt)) {
+            if (typeof note.value !== "number")
+                continue;
+            const key = JSON.stringify([note.label, note.unit ?? null]);
+            let group = groups.get(key);
+            if (group === undefined) {
+                group = { label: note.label, ...(note.unit === undefined ? {} : { unit: note.unit }), values: [] };
+                groups.set(key, group);
+            }
+            group.values.push(note.value);
+        }
+    return [...groups.values()].map(({ label, unit, values }) => {
+        values.sort((a, b) => a - b);
+        const quantile = (p) => {
+            const position = (values.length - 1) * p;
+            const lower = values[Math.floor(position)];
+            const upper = values[Math.ceil(position)];
+            const fraction = position - Math.floor(position);
+            return lower * (1 - fraction) + upper * fraction;
+        };
+        return Object.freeze({ label, ...(unit === undefined ? {} : { unit }), count: values.length,
+            min: values[0], median: quantile(0.5), p90: quantile(0.9), max: values[values.length - 1] });
+    });
+}
+//# sourceMappingURL=notes.cjs.map

package/dist/inspection/notes.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"notes.js","sourceRoot":"","sources":["notes.ts"],"names":[],"mappings":";;AAIA,4CAGC;AACD,sDA8BC;AAtCD,mCAAgC;AAChC,uEAA+F;AAC/F,4DAAwE;AACxE,yCAA+E;AAC/E,SAAgB,gBAAgB,CAAC,QAAmC;IAChE,MAAM,UAAU,GAAG,IAAA,4BAAiB,EAAC,QAAQ,EAAE,sCAAyB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9F,OAAO,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAM,CAAC,iBAAiB,CAAC,8BAAkB,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAC1G,CAAC;AACD,SAAgB,qBAAqB,CAAC,QAA8C;IAChF,MAAM,MAAM,GAAG,IAAI,GAAG,EAIlB,CAAC;IACL,KAAK,MAAM,OAAO,IAAI,QAAQ;QAC1B,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;gBAC9B,SAAS;YACb,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;YAC5D,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,KAAK,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;gBACnG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3B,CAAC;YACD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;IACL,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE;QACxD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE;YACnC,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjD,OAAO,KAAK,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,GAAG,KAAK,GAAG,QAAQ,CAAC;QACrD,CAAC,CAAC;QACF,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM;YACvF,GAAG,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC;IACvG,CAAC,CAAC,CAAC;AACP,CAAC","sourcesContent":["import { Schema } from \"effect\";\nimport { AttemptNotesSchema, type AttemptNote } from \"../record/family/attempt-note/schema.ts\";\nimport { NiceEvalRecordAttachments } from \"../record/family/current.ts\";\nimport { attemptAttachment, type ResolvedInspectionAttempt } from \"./facts.ts\";\nexport function readAttemptNotes(resolved: ResolvedInspectionAttempt): readonly AttemptNote[] {\n    const attachment = attemptAttachment(resolved, NiceEvalRecordAttachments.attemptNotes.family);\n    return attachment === undefined ? [] : Schema.decodeUnknownSync(AttemptNotesSchema)(attachment.value);\n}\nexport function aggregateAttemptNotes(attempts: readonly ResolvedInspectionAttempt[]) {\n    const groups = new Map<string, {\n        label: string;\n        unit?: AttemptNote[\"unit\"];\n        values: number[];\n    }>();\n    for (const attempt of attempts)\n        for (const note of readAttemptNotes(attempt)) {\n            if (typeof note.value !== \"number\")\n                continue;\n            const key = JSON.stringify([note.label, note.unit ?? null]);\n            let group = groups.get(key);\n            if (group === undefined) {\n                group = { label: note.label, ...(note.unit === undefined ? {} : { unit: note.unit }), values: [] };\n                groups.set(key, group);\n            }\n            group.values.push(note.value);\n        }\n    return [...groups.values()].map(({ label, unit, values }) => {\n        values.sort((a, b) => a - b);\n        const quantile = (p: number): number => {\n            const position = (values.length - 1) * p;\n            const lower = values[Math.floor(position)]!;\n            const upper = values[Math.ceil(position)]!;\n            const fraction = position - Math.floor(position);\n            return lower * (1 - fraction) + upper * fraction;\n        };\n        return Object.freeze({ label, ...(unit === undefined ? {} : { unit }), count: values.length,\n            min: values[0]!, median: quantile(0.5), p90: quantile(0.9), max: values[values.length - 1]! });\n    });\n}\n"]}

package/dist/inspection/notes.d.cts

@@ -0,0 +1,12 @@
+import { type AttemptNote } from "../record/family/attempt-note/schema.cjs";
+import { type ResolvedInspectionAttempt } from "./facts.cjs";
+export declare function readAttemptNotes(resolved: ResolvedInspectionAttempt): readonly AttemptNote[];
+export declare function aggregateAttemptNotes(attempts: readonly ResolvedInspectionAttempt[]): Readonly<{
+    count: number;
+    min: number;
+    median: number;
+    p90: number;
+    max: number;
+    unit?: "ms" | "seconds" | "tokens" | "usd" | "ratio" | "count" | "bytes" | undefined;
+    label: string;
+}>[];

package/dist/inspection/notes.d.mts

@@ -0,0 +1,12 @@
+import { type AttemptNote } from "../record/family/attempt-note/schema.mjs";
+import { type ResolvedInspectionAttempt } from "./facts.mjs";
+export declare function readAttemptNotes(resolved: ResolvedInspectionAttempt): readonly AttemptNote[];
+export declare function aggregateAttemptNotes(attempts: readonly ResolvedInspectionAttempt[]): Readonly<{
+    count: number;
+    min: number;
+    median: number;
+    p90: number;
+    max: number;
+    unit?: "ms" | "seconds" | "tokens" | "usd" | "ratio" | "count" | "bytes" | undefined;
+    label: string;
+}>[];

package/dist/inspection/notes.d.ts

@@ -0,0 +1,12 @@
+import { type AttemptNote } from "../record/family/attempt-note/schema.js";
+import { type ResolvedInspectionAttempt } from "./facts.js";
+export declare function readAttemptNotes(resolved: ResolvedInspectionAttempt): readonly AttemptNote[];
+export declare function aggregateAttemptNotes(attempts: readonly ResolvedInspectionAttempt[]): Readonly<{
+    count: number;
+    min: number;
+    median: number;
+    p90: number;
+    max: number;
+    unit?: "ms" | "seconds" | "tokens" | "usd" | "ratio" | "count" | "bytes" | undefined;
+    label: string;
+}>[];

package/dist/inspection/notes.js

@@ -0,0 +1,8 @@
+import { createRequire as __niceevalCreateRequire } from "node:module";
+const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
+const __niceevalCanonical = __niceevalRequire("./notes.cjs");
+const __niceevalExport0 = __niceevalCanonical["aggregateAttemptNotes"];
+export { __niceevalExport0 as aggregateAttemptNotes };
+const __niceevalExport1 = __niceevalCanonical["readAttemptNotes"];
+export { __niceevalExport1 as readAttemptNotes };
+//# sourceMappingURL=notes.js.map

package/dist/inspection/notes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"notes.js","sources":[],"names":[],"mappings":""}

package/dist/inspection/overview.cjs

@@ -2,10 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.selectInspectionOverview = selectInspectionOverview;
 exports.aggregateInspectionCurrentSlots = aggregateInspectionCurrentSlots;
-// @concord-file ne-eval-inspection-overview
-// @concord-implements docs/feature/inspection/README.md
-// @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
-// @concord-implements docs/feature/insight/README.md
 const verdict_ts_1 = require("../eval/record/verdict.cjs");
 const effect_1 = require("effect");
 const current_ts_1 = require("../record/family/current.cjs");
@@ -16,7 +12,7 @@ const trace_ts_1 = require("./trace.cjs");
  * Selects the latest sealed occurrence of every Experiment/Eval/ordinal Slot
  * and closes the shared machine/Insight Overview without platform APIs.
  */
-// @concord-code ne-eval-inspection-select-overview
+// @concord-code
 // @concord-implements docs/feature/inspection/use-case/inspection-compare-quality-cost.md
 // @concord-implements docs/feature/insight/README.md
 function selectInspectionOverview(facts, supportingRuns) {
@@ -212,7 +208,7 @@ function makeCell(slots, labelKeys) {
             const resolved = selected.resolved;
             return [Object.freeze({
                     runId: selected.target.run.runId,
-                    slotId: selected.slot.slotId,
+                    plannedAttemptId: selected.slot.slotId,
                     evalId: selected.slot.evalId,
                     attemptOrdinal: selected.slot.attemptOrdinal,
                     labels: inspectionLabels([selected], labelKeys),
@@ -246,6 +242,7 @@ function makeExperiment(slots, cells, labelKeys) {
     }));
     return Object.freeze({
         experimentId: first.experimentId,
+        plugins: Object.freeze([...new Map(slots.flatMap(({ target }) => target?.run.context.execution.plugins ?? []).map((plugin) => [JSON.stringify(plugin), plugin])).values()]),
         adapter: adapterValue(slots.map(({ adapter }) => adapter)),
         model: executionValue(slots.map(({ model }) => model)),
         labels: inspectionLabels(slots, labelKeys),
@@ -340,7 +337,7 @@ function metricForSlots(slots, key, unit, fold) {
         state: failed ? "failed" : valued.length === 0 ? "unavailable" : partial ? "partial" : "available",
         samples: valued.length,
         total: slots.length,
-        basis: "slot",
+        basis: "planned-attempt",
         issues: Object.freeze([]),
         refs: refsOf(valued),
         unit,
@@ -366,7 +363,7 @@ function overviewPublication(selected) {
         attemptId: member.attempt.attemptId,
         attemptLocator: resolved.locator,
         originRunId: member.attempt.originRunId,
-        originSlotId: resolved.attempt.slotId,
+        originPlannedAttemptId: resolved.attempt.slotId,
         verdict: selected.analysis.verdict,
         score: scoreForMember(selected),
         costUSD: costForSlots([selected]),
@@ -470,7 +467,7 @@ function costForSlots(slots) {
                 : "available",
         samples: valued.length,
         total: subjects.length,
-        basis: "slot",
+        basis: "planned-attempt",
         issues: Object.freeze([]),
         refs: refsOf(valued),
         unit: "USD",
@@ -524,7 +521,7 @@ function passRateOf(slots, denominator, tally) {
         }),
         samples: denominator.classified,
         total: denominator.expected,
-        basis: "slot",
+        basis: "planned-attempt",
         issues,
         refs: refsOf(classified),
     });
@@ -537,7 +534,7 @@ function scoreForMember(selected) {
             state: "unsupported",
             samples: 0,
             total: 0,
-            basis: "slot",
+            basis: "planned-attempt",
             issues: Object.freeze([]),
             refs: analysis.ref === null ? Object.freeze([]) : Object.freeze([analysis.ref]),
             unit: "points",
@@ -560,7 +557,7 @@ function scoreForMember(selected) {
                 }),
         samples: complete ? 1 : 0,
         total: 1,
-        basis: "slot",
+        basis: "planned-attempt",
         issues: uniqueJson([
             ...analysis.issues,
             ...(analysis.score.hasPoints && !analysis.score.complete
@@ -583,7 +580,7 @@ function scoreForCell(slots) {
             state: slots.length === 0 ? "empty" : "unsupported",
             samples: 0,
             total: 0,
-            basis: "slot",
+            basis: "planned-attempt",
             issues: Object.freeze([]),
             refs: Object.freeze([]),
             unit: "points",
@@ -619,7 +616,7 @@ function scoreForCell(slots) {
         }),
         samples: complete.length,
         total: eligible.length,
-        basis: "slot",
+        basis: "planned-attempt",
         issues,
         refs: refsOf(eligible),
         unit: "points",
@@ -776,7 +773,7 @@ function overviewIssue(code, runId, slot) {
     return Object.freeze({
         code,
         runId,
-        slotId: slot.slotId,
+        plannedAttemptId: slot.slotId,
         evalId: slot.evalId,
         attemptOrdinal: slot.attemptOrdinal,
     });