niceeval 0.19.0 → 0.19.1-canary.102

package/dist/inspection/overview.d.cts

@@ -1,6 +1,4 @@
-import { PlannedAttemptPublicationSchema } from "./planned-attempt.cjs";
-import type { VerdictState } from "../eval/record/verdict.cjs";
-import { Schema } from "effect";
+import { type PlannedAttemptMember } from "./results.cjs";
 import type { MemberDocument, RecordSlotIdentity } from "../record/model/core.cjs";
 import type { RunPluginIdentity, AdapterIdentity } from "../record/model/run-context.cjs";
 import { type InspectionJson } from "./codec.cjs";
@@ -57,23 +55,9 @@ export interface InspectionOverviewAggregate {
     readonly coverage: readonly InspectionJson[];
     readonly issues: readonly InspectionJson[];
 }
-export interface InspectionOverviewMember {
+export interface InspectionOverviewMember extends PlannedAttemptMember {
     readonly runId: string;
-    readonly plannedAttemptId: string;
-    readonly evalId: string;
-    readonly attemptOrdinal: number;
     readonly labels: InspectionLabels;
-    readonly publication: Exclude<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> | (Extract<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> & {
-        readonly verdict: VerdictState | null;
-        readonly score: InspectionMetricValue;
-        readonly costUSD: InspectionCostMetricValue;
-        readonly durationMs: InspectionMetricValue;
-        readonly tokens: InspectionMetricValue;
-    });
 }
 export interface InspectionOverviewCell extends InspectionOverviewAggregate {
     readonly experimentId: string;
@@ -134,3 +118,152 @@ export interface InspectionCurrentSlotInput {
 }
 /** Aggregate exactly the Host-assessed current positions; gaps add no value. */
 export declare function aggregateInspectionCurrentSlots(inputs: readonly InspectionCurrentSlotInput[]): InspectionOverview;
+/** Shared metrics for one planned position, including missing and unresolved publications. */
+export declare function projectPlannedAttemptMetrics(runId: string, slot: RecordSlotIdentity, member: MemberDocument | undefined, resolved: ResolvedInspectionAttempt | undefined): Readonly<{
+    readonly verdict: "passed" | "failed" | "errored" | "skipped" | null;
+    readonly score: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly costUSD: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly source: "reported" | "estimated" | "mixed" | null;
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly durationMs: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly tokens: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+}>;

package/dist/inspection/overview.d.mts

@@ -1,6 +1,4 @@
-import { PlannedAttemptPublicationSchema } from "./planned-attempt.mjs";
-import type { VerdictState } from "../eval/record/verdict.mjs";
-import { Schema } from "effect";
+import { type PlannedAttemptMember } from "./results.mjs";
 import type { MemberDocument, RecordSlotIdentity } from "../record/model/core.mjs";
 import type { RunPluginIdentity, AdapterIdentity } from "../record/model/run-context.mjs";
 import { type InspectionJson } from "./codec.mjs";
@@ -57,23 +55,9 @@ export interface InspectionOverviewAggregate {
     readonly coverage: readonly InspectionJson[];
     readonly issues: readonly InspectionJson[];
 }
-export interface InspectionOverviewMember {
+export interface InspectionOverviewMember extends PlannedAttemptMember {
     readonly runId: string;
-    readonly plannedAttemptId: string;
-    readonly evalId: string;
-    readonly attemptOrdinal: number;
     readonly labels: InspectionLabels;
-    readonly publication: Exclude<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> | (Extract<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> & {
-        readonly verdict: VerdictState | null;
-        readonly score: InspectionMetricValue;
-        readonly costUSD: InspectionCostMetricValue;
-        readonly durationMs: InspectionMetricValue;
-        readonly tokens: InspectionMetricValue;
-    });
 }
 export interface InspectionOverviewCell extends InspectionOverviewAggregate {
     readonly experimentId: string;
@@ -134,3 +118,152 @@ export interface InspectionCurrentSlotInput {
 }
 /** Aggregate exactly the Host-assessed current positions; gaps add no value. */
 export declare function aggregateInspectionCurrentSlots(inputs: readonly InspectionCurrentSlotInput[]): InspectionOverview;
+/** Shared metrics for one planned position, including missing and unresolved publications. */
+export declare function projectPlannedAttemptMetrics(runId: string, slot: RecordSlotIdentity, member: MemberDocument | undefined, resolved: ResolvedInspectionAttempt | undefined): Readonly<{
+    readonly verdict: "passed" | "failed" | "errored" | "skipped" | null;
+    readonly score: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly costUSD: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly source: "reported" | "estimated" | "mixed" | null;
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly durationMs: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly tokens: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+}>;

package/dist/inspection/overview.d.ts

@@ -1,6 +1,4 @@
-import { PlannedAttemptPublicationSchema } from "./planned-attempt.js";
-import type { VerdictState } from "../eval/record/verdict.js";
-import { Schema } from "effect";
+import { type PlannedAttemptMember } from "./results.js";
 import type { MemberDocument, RecordSlotIdentity } from "../record/model/core.js";
 import type { RunPluginIdentity, AdapterIdentity } from "../record/model/run-context.js";
 import { type InspectionJson } from "./codec.js";
@@ -57,23 +55,9 @@ export interface InspectionOverviewAggregate {
     readonly coverage: readonly InspectionJson[];
     readonly issues: readonly InspectionJson[];
 }
-export interface InspectionOverviewMember {
+export interface InspectionOverviewMember extends PlannedAttemptMember {
     readonly runId: string;
-    readonly plannedAttemptId: string;
-    readonly evalId: string;
-    readonly attemptOrdinal: number;
     readonly labels: InspectionLabels;
-    readonly publication: Exclude<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> | (Extract<Schema.Schema.Type<typeof PlannedAttemptPublicationSchema>, {
-        readonly state: "published";
-    }> & {
-        readonly verdict: VerdictState | null;
-        readonly score: InspectionMetricValue;
-        readonly costUSD: InspectionCostMetricValue;
-        readonly durationMs: InspectionMetricValue;
-        readonly tokens: InspectionMetricValue;
-    });
 }
 export interface InspectionOverviewCell extends InspectionOverviewAggregate {
     readonly experimentId: string;
@@ -134,3 +118,152 @@ export interface InspectionCurrentSlotInput {
 }
 /** Aggregate exactly the Host-assessed current positions; gaps add no value. */
 export declare function aggregateInspectionCurrentSlots(inputs: readonly InspectionCurrentSlotInput[]): InspectionOverview;
+/** Shared metrics for one planned position, including missing and unresolved publications. */
+export declare function projectPlannedAttemptMetrics(runId: string, slot: RecordSlotIdentity, member: MemberDocument | undefined, resolved: ResolvedInspectionAttempt | undefined): Readonly<{
+    readonly verdict: "passed" | "failed" | "errored" | "skipped" | null;
+    readonly score: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly costUSD: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly source: "reported" | "estimated" | "mixed" | null;
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly durationMs: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+    readonly tokens: {
+        readonly value: number | null;
+        readonly state: "partial" | "failed" | "available" | "unavailable" | "empty" | "unsupported";
+        readonly issues: readonly ({
+            readonly code: "assertions-not-recorded" | "assertions-revision-unsupported" | "assertions-current-invalid";
+            readonly locator: string;
+        } | {
+            readonly code: "attempt-origin-missing" | "attempt-not-observed";
+            readonly runId: string;
+            readonly plannedAttemptId: string;
+            readonly evalId: string;
+            readonly attemptOrdinal: number;
+        } | {
+            readonly code: "score-contribution-unavailable";
+            readonly locator: string;
+            readonly entryId: string;
+            readonly reason: "source-unavailable" | "not-applicable" | "evaluation-errored";
+        } | {
+            readonly code: "score-partial";
+            readonly locator: string | null;
+        })[];
+        readonly basis: "eval" | "planned-attempt";
+        readonly total: number;
+        readonly refs: readonly {
+            readonly identity: {
+                readonly kind: "attempt";
+                readonly locator: string;
+            };
+        }[];
+        readonly samples: number;
+        readonly unit?: "ms" | "tokens" | "USD" | "points" | undefined;
+        readonly bounds?: {
+            readonly min: number;
+            readonly max: number;
+        } | undefined;
+    };
+}>;

package/dist/inspection/overview.js

@@ -3,6 +3,8 @@ const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
 const __niceevalCanonical = __niceevalRequire("./overview.cjs");
 const __niceevalExport0 = __niceevalCanonical["aggregateInspectionCurrentSlots"];
 export { __niceevalExport0 as aggregateInspectionCurrentSlots };
-const __niceevalExport1 = __niceevalCanonical["selectInspectionOverview"];
-export { __niceevalExport1 as selectInspectionOverview };
+const __niceevalExport1 = __niceevalCanonical["projectPlannedAttemptMetrics"];
+export { __niceevalExport1 as projectPlannedAttemptMetrics };
+const __niceevalExport2 = __niceevalCanonical["selectInspectionOverview"];
+export { __niceevalExport2 as selectInspectionOverview };
 //# sourceMappingURL=overview.js.map