niceeval 0.13.1 → 0.13.2-canary.35

package/dist/report/built-in/result-components.d.cts

@@ -12,6 +12,7 @@ import type { BuiltInAttemptRows, BuiltInExperimentRows, BuiltInSummaryRows } fr
 export interface OverviewCounts {
     readonly experiments: number;
     readonly attempts: number;
+    readonly expectedResults: number;
 }
 /** Display-safe membership facts prepared by the Page loader from Snapshot. */
 export interface MembershipRow {
@@ -24,6 +25,19 @@ export interface MembershipRow {
     readonly state: string;
     readonly relation: string | null;
     readonly locator: string | null;
+    readonly outcome?: string | null;
+    readonly phase?: string | null;
+    readonly error?: string | null;
+    readonly sharedFailure?: string | null;
+}
+export interface RunErrorRow {
+    readonly key: string;
+    readonly failure: string;
+    readonly phase: string;
+    readonly affected: string;
+    readonly error: string;
+    readonly message: string;
+    readonly fix: string | null;
 }
 export interface StandardOverviewResult {
     readonly hero: HeroData;
@@ -55,6 +69,7 @@ export interface RunMembershipResult {
     readonly hero: HeroData;
     readonly summary: BuiltInSummaryRows;
     readonly members: readonly MembershipRow[];
+    readonly errors: readonly RunErrorRow[];
     readonly evidence: AttemptEvidenceDomainView;
 }
 export interface AttemptTraceProps {

package/dist/report/built-in/result-components.d.mts

@@ -12,6 +12,7 @@ import type { BuiltInAttemptRows, BuiltInExperimentRows, BuiltInSummaryRows } fr
 export interface OverviewCounts {
     readonly experiments: number;
     readonly attempts: number;
+    readonly expectedResults: number;
 }
 /** Display-safe membership facts prepared by the Page loader from Snapshot. */
 export interface MembershipRow {
@@ -24,6 +25,19 @@ export interface MembershipRow {
     readonly state: string;
     readonly relation: string | null;
     readonly locator: string | null;
+    readonly outcome?: string | null;
+    readonly phase?: string | null;
+    readonly error?: string | null;
+    readonly sharedFailure?: string | null;
+}
+export interface RunErrorRow {
+    readonly key: string;
+    readonly failure: string;
+    readonly phase: string;
+    readonly affected: string;
+    readonly error: string;
+    readonly message: string;
+    readonly fix: string | null;
 }
 export interface StandardOverviewResult {
     readonly hero: HeroData;
@@ -55,6 +69,7 @@ export interface RunMembershipResult {
     readonly hero: HeroData;
     readonly summary: BuiltInSummaryRows;
     readonly members: readonly MembershipRow[];
+    readonly errors: readonly RunErrorRow[];
     readonly evidence: AttemptEvidenceDomainView;
 }
 export interface AttemptTraceProps {

package/dist/report/built-in/result-components.d.ts

@@ -12,6 +12,7 @@ import type { BuiltInAttemptRows, BuiltInExperimentRows, BuiltInSummaryRows } fr
 export interface OverviewCounts {
     readonly experiments: number;
     readonly attempts: number;
+    readonly expectedResults: number;
 }
 /** Display-safe membership facts prepared by the Page loader from Snapshot. */
 export interface MembershipRow {
@@ -24,6 +25,19 @@ export interface MembershipRow {
     readonly state: string;
     readonly relation: string | null;
     readonly locator: string | null;
+    readonly outcome?: string | null;
+    readonly phase?: string | null;
+    readonly error?: string | null;
+    readonly sharedFailure?: string | null;
+}
+export interface RunErrorRow {
+    readonly key: string;
+    readonly failure: string;
+    readonly phase: string;
+    readonly affected: string;
+    readonly error: string;
+    readonly message: string;
+    readonly fix: string | null;
 }
 export interface StandardOverviewResult {
     readonly hero: HeroData;
@@ -55,6 +69,7 @@ export interface RunMembershipResult {
     readonly hero: HeroData;
     readonly summary: BuiltInSummaryRows;
     readonly members: readonly MembershipRow[];
+    readonly errors: readonly RunErrorRow[];
     readonly evidence: AttemptEvidenceDomainView;
 }
 export interface AttemptTraceProps {

package/dist/report/built-in/run-membership-overview.cjs

@@ -2,6 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.defaultRunMembershipOverviewReport = void 0;
 exports.runMembershipOverviewReport = runMembershipOverviewReport;
+const index_ts_1 = require("../../analysis/index.cjs");
+const api_ts_1 = require("../../analysis/api.cjs");
 const jsx_runtime_1 = require("react/jsx-runtime");
 const conversions_ts_1 = require("../model/conversions.cjs");
 const analysis_values_ts_1 = require("./analysis-values.cjs");
@@ -14,7 +16,86 @@ function heroData(snapshot) {
         runs: snapshot.runs.length,
     });
 }
-function membershipRows(snapshot) {
+function sharedBuildFailure(summary) {
+    let value;
+    try {
+        value = JSON.parse(summary);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof value !== "object" || value === null)
+        return undefined;
+    const item = value;
+    return item.schema === "niceeval.shared-build-failure/v1"
+        && typeof item.failureId === "string"
+        && typeof item.evalId === "string"
+        && Number.isSafeInteger(item.attemptOrdinal)
+        && item.phase === "sandbox.image.build"
+        && typeof item.errorCode === "string"
+        && typeof item.message === "string"
+        && (item.remediation === undefined || item.remediation === "pnpm-allow-builds")
+        ? item
+        : undefined;
+}
+function failureBySlot(view) {
+    const failures = new Map();
+    for (const entry of view.entries) {
+        if (entry.state !== "available")
+            continue;
+        for (const diagnostic of entry.detail.diagnostics) {
+            if (diagnostic.code !== "sandbox-build-failed")
+                continue;
+            const failure = sharedBuildFailure(diagnostic.summary);
+            if (failure !== undefined) {
+                failures.set(`${entry.runId}\u0000${failure.evalId}\u0000${failure.attemptOrdinal}`, failure);
+            }
+        }
+    }
+    return failures;
+}
+function runErrorRows(view) {
+    const groups = new Map();
+    for (const entry of view.entries) {
+        if (entry.state !== "available")
+            continue;
+        for (const diagnostic of entry.detail.diagnostics) {
+            if (diagnostic.code !== "sandbox-build-failed")
+                continue;
+            const failure = sharedBuildFailure(diagnostic.summary);
+            if (failure === undefined)
+                continue;
+            const key = `${entry.runId}\u0000${failure.failureId}`;
+            const group = groups.get(key);
+            const affected = `${failure.evalId}#${failure.attemptOrdinal + 1}`;
+            if (group === undefined) {
+                groups.set(key, {
+                    runId: entry.runId,
+                    experimentId: entry.experimentId,
+                    failure,
+                    affected: [affected],
+                });
+            }
+            else {
+                group.affected.push(affected);
+            }
+        }
+    }
+    return Object.freeze([...groups.entries()].map(([key, group]) => Object.freeze({
+        key,
+        failure: group.failure.failureId,
+        phase: group.failure.phase,
+        affected: `${group.experimentId} · ${group.affected.join(", ")}`,
+        error: group.failure.errorCode,
+        message: group.failure.message,
+        fix: group.failure.remediation === "pnpm-allow-builds"
+            ? "Review these dependencies and configure pnpm allowBuilds."
+            : null,
+    })));
+}
+function membershipRows(snapshot, diagnostics, evidence) {
+    const failures = failureBySlot(diagnostics);
+    const outcomes = new Map(evidence.entries.flatMap((entry) => entry.state === "available" ? [[entry.attempt.locator, entry.detail.outcome]] : []));
     return Object.freeze(snapshot.slots.map((slot) => Object.freeze({
         key: `${slot.runId}:${slot.slotId}`,
         experiment: String(slot.experimentId),
@@ -22,9 +103,22 @@ function membershipRows(snapshot) {
         attempt: slot.attemptOrdinal,
         selectedRun: String(slot.runId),
         slot: String(slot.slotId),
-        state: slot.state,
+        state: slot.state === "included" ? slot.action : slot.state === "not-recorded" ? slot.action ?? slot.state : slot.state,
         relation: slot.state === "included" ? slot.relation : null,
         locator: slot.state === "included" ? slot.attempt.locator : null,
+        outcome: slot.state === "included" ? outcomes.get(slot.attempt.locator) ?? null : null,
+        phase: null,
+        error: null,
+        sharedFailure: null,
+        ...(() => {
+            const failure = failures.get(`${slot.runId}\u0000${slot.evalId}\u0000${slot.attemptOrdinal}`);
+            return failure === undefined ? {} : {
+                outcome: "errored",
+                phase: failure.phase,
+                error: failure.errorCode,
+                sharedFailure: failure.failureId,
+            };
+        })(),
     })));
 }
 const runMembershipPage = {
@@ -32,14 +126,16 @@ const runMembershipPage = {
     path: "/",
     title: "Run membership overview",
     load: async (sample) => {
-        const [summary, evidence] = await Promise.all([
+        const [summary, evidence, diagnostics] = await Promise.all([
             (0, analysis_values_ts_1.loadBuiltInSummaryRows)(sample),
             (0, conversions_ts_1.toAttemptEvidence)(sample),
+            (0, index_ts_1.query)(sample, { kind: "domain-view", view: api_ts_1.runDiagnosticsView }),
         ]);
         return Object.freeze({
             hero: heroData(sample.snapshot),
             summary,
-            members: membershipRows(sample.snapshot),
+            members: membershipRows(sample.snapshot, diagnostics, evidence),
+            errors: runErrorRows(diagnostics),
             evidence,
         });
     },

package/dist/report/built-in/run-membership-overview.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"run-membership-overview.js","sourceRoot":"","sources":["run-membership-overview.ts"],"names":[],"mappings":";;;AAsDA,kEAKC;AAzDD,mDAAwC;AAGxC,4DAA4D;AAC5D,6DAAwF;AACxF,kEAAuF;AACvF,6CAA+E;AAO/E,SAAS,QAAQ,CAAC,QAAwB;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAClK,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;QACxE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;KAC7B,CAAC,CAAC;AACP,CAAC;AACD,SAAS,cAAc,CAAC,QAAwB;IAC5C,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5D,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;QACnC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,EAAE,IAAI,CAAC,cAAc;QAC5B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAC1D,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAC,CAAC;AACT,CAAC;AACD,MAAM,iBAAiB,GAAG;IACtB,EAAE,EAAE,gBAAgB;IACpB,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,yBAAyB;IAChC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAmC,EAAE;QAC5D,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,IAAA,2CAAsB,EAAC,MAAM,CAAC;YAC9B,IAAA,kCAAiB,EAAC,MAAM,CAAC;SAC5B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,OAAO;YACP,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC;YACxC,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAA6B,EAAE,EAAE,CAAC,IAAA,iBAAG,EAAC,+CAAuB,EAAE,KAAK,CAAC;CACpC,CAAC;AAC/C,iEAAiE;AACjE,SAAgB,2BAA2B;IACvC,OAAO,IAAA,gCAAmB,EAAC,sCAAyB,CAAC,qBAAqB,EAAE;QACxE,KAAK,EAAE,yBAAyB;QAChC,KAAK,EAAE,CAAC,iBAAiB,CAAC;KAC7B,CAAC,CAAC;AACP,CAAC;AACD,yEAAyE;AAC5D,QAAA,kCAAkC,GAAG,2BAA2B,EAAE,CAAC;AAChF,kBAAe,0CAAkC,CAAC","sourcesContent":["/** The stable explicit-Run membership selector. */\nimport type { Sample, SampleSnapshot, } from \"../../analysis/index.ts\";\nimport { jsx } from \"react/jsx-runtime\";\nimport type { Page } from \"../definition/report.ts\";\nimport { type HeroData } from \"../components/site-components/index.tsx\";\nimport { toAttemptEvidence } from \"../model/conversions.ts\";\nimport { loadBuiltInSummaryRows, type BuiltInSummaryRows, } from \"./analysis-values.ts\";\nimport { RunMembershipResultView, type MembershipRow, } from \"./result-components.tsx\";\nimport { builtInMachineProducerIds, defineBuiltInReport, } from \"./machine.ts\";\ninterface RunMembershipPageInput {\n    readonly hero: HeroData;\n    readonly summary: BuiltInSummaryRows;\n    readonly members: readonly MembershipRow[];\n    readonly evidence: Awaited<ReturnType<typeof toAttemptEvidence>>;\n}\nfunction heroData(snapshot: SampleSnapshot): HeroData {\n    const latest = snapshot.runs.reduce<number | null>((current, run) => current === null || Number(run.startedAt) > current ? Number(run.startedAt) : current, null);\n    return Object.freeze({\n        latestStartedAt: latest === null ? null : new Date(latest).toISOString(),\n        runs: snapshot.runs.length,\n    });\n}\nfunction membershipRows(snapshot: SampleSnapshot): readonly MembershipRow[] {\n    return Object.freeze(snapshot.slots.map((slot) => Object.freeze({\n        key: `${slot.runId}:${slot.slotId}`,\n        experiment: String(slot.experimentId),\n        eval: String(slot.evalId),\n        attempt: slot.attemptOrdinal,\n        selectedRun: String(slot.runId),\n        slot: String(slot.slotId),\n        state: slot.state,\n        relation: slot.state === \"included\" ? slot.relation : null,\n        locator: slot.state === \"included\" ? slot.attempt.locator : null,\n    })));\n}\nconst runMembershipPage = {\n    id: \"run-membership\",\n    path: \"/\",\n    title: \"Run membership overview\",\n    load: async (sample: Sample): Promise<RunMembershipPageInput> => {\n        const [summary, evidence] = await Promise.all([\n            loadBuiltInSummaryRows(sample),\n            toAttemptEvidence(sample),\n        ]);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            summary,\n            members: membershipRows(sample.snapshot),\n            evidence,\n        });\n    },\n    render: (input: RunMembershipPageInput) => jsx(RunMembershipResultView, input),\n} satisfies Page<void, RunMembershipPageInput>;\n/** The bounded default for explicit historical Run selectors. */\nexport function runMembershipOverviewReport() {\n    return defineBuiltInReport(builtInMachineProducerIds.runMembershipOverview, {\n        title: \"Run membership overview\",\n        pages: [runMembershipPage],\n    });\n}\n/** The CLI default Report for one or more explicit `--run` selectors. */\nexport const defaultRunMembershipOverviewReport = runMembershipOverviewReport();\nexport default defaultRunMembershipOverviewReport;\n"]}
+{"version":3,"file":"run-membership-overview.js","sourceRoot":"","sources":["run-membership-overview.ts"],"names":[],"mappings":";;;AAsKA,kEAKC;AAzKD,sDAAgD;AAChD,kDAA2F;AAC3F,mDAAwC;AAGxC,4DAA4D;AAC5D,6DAAwF;AACxF,kEAAyG;AACzG,6CAA+E;AAQ/E,SAAS,QAAQ,CAAC,QAAwB;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAClK,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;QACxE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;KAC7B,CAAC,CAAC;AACP,CAAC;AAWD,SAAS,kBAAkB,CAAC,OAAe;IACvC,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,CAAC;QACH,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAC3C,OAAO,SAAS,CAAC;IACrB,MAAM,IAAI,GAAG,KAA8C,CAAC;IAC5D,OAAO,IAAI,CAAC,MAAM,KAAK,kCAAkC;WAClD,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;WAClC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;WAC/B,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC;WACzC,IAAI,CAAC,KAAK,KAAK,qBAAqB;WACpC,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;WAClC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;WAChC,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,mBAAmB,CAAC;QAC/E,CAAC,CAAC,IAAoC;QACtC,CAAC,CAAC,SAAS,CAAC;AACpB,CAAC;AACD,SAAS,aAAa,CAAC,IAA8B;IACjD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwC,CAAC;IACjE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW;YAC3B,SAAS;QACb,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChD,IAAI,UAAU,CAAC,IAAI,KAAK,sBAAsB;gBAC1C,SAAS;YACb,MAAM,OAAO,GAAG,kBAAkB,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,SAAS,OAAO,CAAC,MAAM,SAAS,OAAO,CAAC,cAAc,EAAE,EAAE,OAAO,CAAC,CAAC;YAClG,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC;AACD,SAAS,YAAY,CAAC,IAA8B;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAKlB,CAAC;IACL,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW;YAC3B,SAAS;QACb,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChD,IAAI,UAAU,CAAC,IAAI,KAAK,sBAAsB;gBAC1C,SAAS;YACb,MAAM,OAAO,GAAG,kBAAkB,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,OAAO,KAAK,SAAS;gBACrB,SAAS;YACb,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,KAAK,SAAS,OAAO,CAAC,SAAS,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YACnE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;oBACZ,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,OAAO;oBACP,QAAQ,EAAE,CAAC,QAAQ,CAAC;iBACvB,CAAC,CAAC;YACP,CAAC;iBACI,CAAC;gBACF,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC3E,GAAG;QACH,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;QAChC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK;QAC1B,QAAQ,EAAE,GAAG,KAAK,CAAC,YAAY,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAChE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;QAC9B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,OAAO;QAC9B,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,WAAW,KAAK,mBAAmB;YAClD,CAAC,CAAC,2DAA2D;YAC7D,CAAC,CAAC,IAAI;KACb,CAAC,CAAC,CAAC,CAAC;AACT,CAAC;AACD,SAAS,cAAc,CAAC,QAAwB,EAAE,WAAqC,EAAE,QAAuD;IAC5I,MAAM,QAAQ,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3J,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5D,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;QACnC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,EAAE,IAAI,CAAC,cAAc;QAC5B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;QACvH,QAAQ,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAC1D,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QAChE,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI;QACtF,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,IAAI;QACX,aAAa,EAAE,IAAI;QACnB,GAAG,CAAC,GAAG,EAAE;YACL,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,MAAM,SAAS,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;YAC9F,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAChC,OAAO,EAAE,SAAS;gBAClB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,KAAK,EAAE,OAAO,CAAC,SAAS;gBACxB,aAAa,EAAE,OAAO,CAAC,SAAS;aACnC,CAAC;QACN,CAAC,CAAC,EAAE;KACP,CAAC,CAAC,CAAC,CAAC;AACT,CAAC;AACD,MAAM,iBAAiB,GAAG;IACtB,EAAE,EAAE,gBAAgB;IACpB,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,yBAAyB;IAChC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAmC,EAAE;QAC5D,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACvD,IAAA,2CAAsB,EAAC,MAAM,CAAC;YAC9B,IAAA,kCAAiB,EAAC,MAAM,CAAC;YACzB,IAAA,gBAAK,EAAC,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,2BAAkB,EAAE,CAAC;SACnE,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,OAAO;YACP,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC;YAC/D,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC;YACjC,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAA6B,EAAE,EAAE,CAAC,IAAA,iBAAG,EAAC,+CAAuB,EAAE,KAAK,CAAC;CACpC,CAAC;AAC/C,iEAAiE;AACjE,SAAgB,2BAA2B;IACvC,OAAO,IAAA,gCAAmB,EAAC,sCAAyB,CAAC,qBAAqB,EAAE;QACxE,KAAK,EAAE,yBAAyB;QAChC,KAAK,EAAE,CAAC,iBAAiB,CAAC;KAC7B,CAAC,CAAC;AACP,CAAC;AACD,yEAAyE;AAC5D,QAAA,kCAAkC,GAAG,2BAA2B,EAAE,CAAC;AAChF,kBAAe,0CAAkC,CAAC","sourcesContent":["/** The stable explicit-Run membership selector. */\nimport type { Sample, SampleSnapshot, } from \"../../analysis/index.ts\";\nimport { query } from \"../../analysis/index.ts\";\nimport { runDiagnosticsView, type RunDiagnosticsDomainView, } from \"../../analysis/api.ts\";\nimport { jsx } from \"react/jsx-runtime\";\nimport type { Page } from \"../definition/report.ts\";\nimport { type HeroData } from \"../components/site-components/index.tsx\";\nimport { toAttemptEvidence } from \"../model/conversions.ts\";\nimport { loadBuiltInSummaryRows, type BuiltInSummaryRows, } from \"./analysis-values.ts\";\nimport { RunMembershipResultView, type MembershipRow, type RunErrorRow, } from \"./result-components.tsx\";\nimport { builtInMachineProducerIds, defineBuiltInReport, } from \"./machine.ts\";\ninterface RunMembershipPageInput {\n    readonly hero: HeroData;\n    readonly summary: BuiltInSummaryRows;\n    readonly members: readonly MembershipRow[];\n    readonly errors: readonly RunErrorRow[];\n    readonly evidence: Awaited<ReturnType<typeof toAttemptEvidence>>;\n}\nfunction heroData(snapshot: SampleSnapshot): HeroData {\n    const latest = snapshot.runs.reduce<number | null>((current, run) => current === null || Number(run.startedAt) > current ? Number(run.startedAt) : current, null);\n    return Object.freeze({\n        latestStartedAt: latest === null ? null : new Date(latest).toISOString(),\n        runs: snapshot.runs.length,\n    });\n}\ninterface SharedBuildFailureDiagnostic {\n    readonly schema: \"niceeval.shared-build-failure/v1\";\n    readonly failureId: string;\n    readonly evalId: string;\n    readonly attemptOrdinal: number;\n    readonly phase: \"sandbox.image.build\";\n    readonly errorCode: string;\n    readonly message: string;\n    readonly remediation?: \"pnpm-allow-builds\";\n}\nfunction sharedBuildFailure(summary: string): SharedBuildFailureDiagnostic | undefined {\n    let value: unknown;\n    try {\n        value = JSON.parse(summary);\n    }\n    catch {\n        return undefined;\n    }\n    if (typeof value !== \"object\" || value === null)\n        return undefined;\n    const item = value as Partial<SharedBuildFailureDiagnostic>;\n    return item.schema === \"niceeval.shared-build-failure/v1\"\n        && typeof item.failureId === \"string\"\n        && typeof item.evalId === \"string\"\n        && Number.isSafeInteger(item.attemptOrdinal)\n        && item.phase === \"sandbox.image.build\"\n        && typeof item.errorCode === \"string\"\n        && typeof item.message === \"string\"\n        && (item.remediation === undefined || item.remediation === \"pnpm-allow-builds\")\n        ? item as SharedBuildFailureDiagnostic\n        : undefined;\n}\nfunction failureBySlot(view: RunDiagnosticsDomainView): ReadonlyMap<string, SharedBuildFailureDiagnostic> {\n    const failures = new Map<string, SharedBuildFailureDiagnostic>();\n    for (const entry of view.entries) {\n        if (entry.state !== \"available\")\n            continue;\n        for (const diagnostic of entry.detail.diagnostics) {\n            if (diagnostic.code !== \"sandbox-build-failed\")\n                continue;\n            const failure = sharedBuildFailure(diagnostic.summary);\n            if (failure !== undefined) {\n                failures.set(`${entry.runId}\\u0000${failure.evalId}\\u0000${failure.attemptOrdinal}`, failure);\n            }\n        }\n    }\n    return failures;\n}\nfunction runErrorRows(view: RunDiagnosticsDomainView): readonly RunErrorRow[] {\n    const groups = new Map<string, {\n        readonly runId: string;\n        readonly experimentId: string;\n        readonly failure: SharedBuildFailureDiagnostic;\n        readonly affected: string[];\n    }>();\n    for (const entry of view.entries) {\n        if (entry.state !== \"available\")\n            continue;\n        for (const diagnostic of entry.detail.diagnostics) {\n            if (diagnostic.code !== \"sandbox-build-failed\")\n                continue;\n            const failure = sharedBuildFailure(diagnostic.summary);\n            if (failure === undefined)\n                continue;\n            const key = `${entry.runId}\\u0000${failure.failureId}`;\n            const group = groups.get(key);\n            const affected = `${failure.evalId}#${failure.attemptOrdinal + 1}`;\n            if (group === undefined) {\n                groups.set(key, {\n                    runId: entry.runId,\n                    experimentId: entry.experimentId,\n                    failure,\n                    affected: [affected],\n                });\n            }\n            else {\n                group.affected.push(affected);\n            }\n        }\n    }\n    return Object.freeze([...groups.entries()].map(([key, group]) => Object.freeze({\n        key,\n        failure: group.failure.failureId,\n        phase: group.failure.phase,\n        affected: `${group.experimentId} · ${group.affected.join(\", \")}`,\n        error: group.failure.errorCode,\n        message: group.failure.message,\n        fix: group.failure.remediation === \"pnpm-allow-builds\"\n            ? \"Review these dependencies and configure pnpm allowBuilds.\"\n            : null,\n    })));\n}\nfunction membershipRows(snapshot: SampleSnapshot, diagnostics: RunDiagnosticsDomainView, evidence: Awaited<ReturnType<typeof toAttemptEvidence>>): readonly MembershipRow[] {\n    const failures = failureBySlot(diagnostics);\n    const outcomes = new Map(evidence.entries.flatMap((entry) => entry.state === \"available\" ? [[entry.attempt.locator, entry.detail.outcome] as const] : []));\n    return Object.freeze(snapshot.slots.map((slot) => Object.freeze({\n        key: `${slot.runId}:${slot.slotId}`,\n        experiment: String(slot.experimentId),\n        eval: String(slot.evalId),\n        attempt: slot.attemptOrdinal,\n        selectedRun: String(slot.runId),\n        slot: String(slot.slotId),\n        state: slot.state === \"included\" ? slot.action : slot.state === \"not-recorded\" ? slot.action ?? slot.state : slot.state,\n        relation: slot.state === \"included\" ? slot.relation : null,\n        locator: slot.state === \"included\" ? slot.attempt.locator : null,\n        outcome: slot.state === \"included\" ? outcomes.get(slot.attempt.locator) ?? null : null,\n        phase: null,\n        error: null,\n        sharedFailure: null,\n        ...(() => {\n            const failure = failures.get(`${slot.runId}\\u0000${slot.evalId}\\u0000${slot.attemptOrdinal}`);\n            return failure === undefined ? {} : {\n                outcome: \"errored\",\n                phase: failure.phase,\n                error: failure.errorCode,\n                sharedFailure: failure.failureId,\n            };\n        })(),\n    })));\n}\nconst runMembershipPage = {\n    id: \"run-membership\",\n    path: \"/\",\n    title: \"Run membership overview\",\n    load: async (sample: Sample): Promise<RunMembershipPageInput> => {\n        const [summary, evidence, diagnostics] = await Promise.all([\n            loadBuiltInSummaryRows(sample),\n            toAttemptEvidence(sample),\n            query(sample, { kind: \"domain-view\", view: runDiagnosticsView }),\n        ]);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            summary,\n            members: membershipRows(sample.snapshot, diagnostics, evidence),\n            errors: runErrorRows(diagnostics),\n            evidence,\n        });\n    },\n    render: (input: RunMembershipPageInput) => jsx(RunMembershipResultView, input),\n} satisfies Page<void, RunMembershipPageInput>;\n/** The bounded default for explicit historical Run selectors. */\nexport function runMembershipOverviewReport() {\n    return defineBuiltInReport(builtInMachineProducerIds.runMembershipOverview, {\n        title: \"Run membership overview\",\n        pages: [runMembershipPage],\n    });\n}\n/** The CLI default Report for one or more explicit `--run` selectors. */\nexport const defaultRunMembershipOverviewReport = runMembershipOverviewReport();\nexport default defaultRunMembershipOverviewReport;\n"]}

package/dist/report/built-in/standard.cjs

@@ -52,6 +52,7 @@ exports.standardOverviewPage = {
             counts: Object.freeze({
                 experiments: experiments.length,
                 attempts: includedAttemptCount(sample.snapshot),
+                expectedResults: sample.snapshot.slots.filter((slot) => slot.state !== "excluded").length,
             }),
         });
     },

package/dist/report/built-in/standard.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"standard.js","sourceRoot":"","sources":["standard.tsx"],"names":[],"mappings":";;;;AASA,gEAAmD;AACnD,qEAAwE;AACxE,sEAA8E;AAC9E,4DAAiE;AACjE,sDAA6G;AAE7G,6DAAgM;AAChM,kEAAgK;AAChK,6CAA+E;AA4B/E,SAAS,QAAQ,CAAC,QAAwB;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAClK,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;QACxE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;KAC7B,CAAC,CAAC;AACP,CAAC;AACD,SAAS,cAAc,CAAC,QAAwB,EAAE,SAG9C,EAAE;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK;SAC9B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC;SACpG,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC;SACxH,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7B,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;QACnC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,EAAE,IAAI,CAAC,cAAc;QAC5B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAC1D,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAC,CAAC;AACT,CAAC;AACD,SAAS,oBAAoB,CAAC,QAAwB;IAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;AAC7E,CAAC;AACD,yCAAyC;AAC5B,QAAA,oBAAoB,GAAG;IAChC,EAAE,EAAE,QAAQ;IACZ,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;IACtC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAsC,EAAE;QAC/D,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC7C,IAAA,2CAAsB,EAAC,MAAM,CAAC;YAC9B,IAAA,8CAAyB,EAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,OAAO;YACP,WAAW;YACX,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;gBAClB,WAAW,EAAE,WAAW,CAAC,MAAM;gBAC/B,QAAQ,EAAE,oBAAoB,CAAC,MAAM,CAAC,QAAQ,CAAC;aAClD,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAAgC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACzC,CAAC;AAClD,qFAAqF;AACxE,QAAA,oBAAoB,GAAG;IAChC,EAAE,EAAE,UAAU;IACd,KAAK,EAAE,UAAU;IACjB,IAAI,EAAE,KAAK,EAAE,MAAc,EAAsC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC9E,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC/B,QAAQ,EAAE,MAAM,IAAA,2CAAsB,EAAC,MAAM,CAAC;KACjD,CAAC;IACF,MAAM,EAAE,CAAC,KAAgC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACzC,CAAC;AAClD,oFAAoF;AACvE,QAAA,kBAAkB,GAAG;IAC9B,EAAE,EAAE,QAAQ;IACZ,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;IACtC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAoC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5E,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC/B,aAAa,EAAE,MAAM,IAAA,uCAAsB,EAAC,MAAM,CAAC;KACtD,CAAC;IACF,MAAM,EAAE,CAAC,KAA8B,EAAE,EAAE,CAAC,CAAC,wBAAC,oBAAG,eAC/C,uBAAC,gBAAI,IAAC,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,EACzB,uBAAC,oCAAY,IAAC,IAAI,EAAE,KAAK,CAAC,aAAa,EAAE,IAAI,EAAC,WAAW,EAAC,UAAU,EAAC,MAAM,GAAE,IACzE,CAAC;CACoC,CAAC;AAChD,iEAAiE;AACpD,QAAA,mBAAmB,GAAG;IAC/B,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,SAAS;IAChB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,gCAAmB;IAC3B,IAAI,EAAE,KAAK,EAAE,OAAe,EAAE,MAA2B,EAAE,OAAO,EAAqC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QACpH,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;KACnD,CAAC;IACF,MAAM,EAAE,CAAC,KAA+B,EAAE,EAAE,CAAC,uBAAC,0BAAc,IAAC,OAAO,EAAE,KAAK,GAAG;CACnB,CAAC;AAChE,SAAS,mBAAmB,CAAC,MAAc,EAAE,MAA8B;IACvE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI;QAClC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;IAC/D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK;QACpC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACjE,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,IAAA,mCAAsB,EAAC,SAAS,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG;YACpD,OAAO,SAAS,CAAC;IACzB,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,uDAAuD,CAAC,CAAC;AACjF,CAAC;AACD,yEAAyE;AAC5D,QAAA,sBAAsB,GAAG;IAClC,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,aAAa;IACnB,KAAK,EAAE,YAAY;IACnB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,mCAAsB;IAC9B,IAAI,EAAE,KAAK,EAAE,MAAc,EAAE,MAA8B,EAAwC,EAAE;QACjG,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;YAC9B,IAAI,EAAE,MAAM,IAAA,8CAAyB,EAAC,MAAM,CAAC;YAC7C,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;SAC/E,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAAkC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACvB,CAAC;AACtE,qEAAqE;AACxD,QAAA,QAAQ,GAAG,IAAA,gCAAmB,EAAC,sCAAyB,CAAC,QAAQ,EAAE;IAC5E,KAAK,EAAE,0BAA0B;IACjC,KAAK,EAAE;QACH,4BAAoB;QACpB,4BAAoB;QACpB,0BAAkB;QAClB,2BAAmB;QACnB,8BAAsB;KACzB;CACJ,CAAC,CAAC;AACH,kBAAe,gBAAQ,CAAC","sourcesContent":["/**\n * The standard information architecture.\n *\n * Detail Pages are deliberately listed here. `defineReport()` receives no\n * library page collection, so a consumer's arbitrary custom Report never\n * gains attempt or experiment routes as an implicit side effect.\n */\nimport type { ExperimentId, Sample, SampleSnapshot, } from \"../../analysis/index.ts\";\nimport type { Page, PageEvidence, } from \"../definition/report.ts\";\nimport { Col } from \"../definition/primitives.tsx\";\nimport { AttemptDetails } from \"../components/attempt-detail/index.tsx\";\nimport { Hero, type HeroData } from \"../components/site-components/index.tsx\";\nimport { toAttemptObservability } from \"../model/conversions.ts\";\nimport { attemptDetailParams, experimentDetailParams, experimentDetailTarget, } from \"../library/details.ts\";\nimport type { AttemptDetailTarget, ExperimentDetailTarget, } from \"../library/details.ts\";\nimport { loadBuiltInAttemptRows, loadBuiltInExperimentRows, loadBuiltInSummaryRows, type BuiltInAttemptRows, type BuiltInExperimentRows, type BuiltInSummaryRows, } from \"./analysis-values.ts\";\nimport { AttemptTrace, ExperimentDetailResultView, StandardAttemptsResultView, StandardOverviewResultView, type MembershipRow, } from \"./result-components.tsx\";\nimport { builtInMachineProducerIds, defineBuiltInReport, } from \"./machine.ts\";\ninterface StandardOverviewPageInput {\n    readonly hero: HeroData;\n    readonly summary: BuiltInSummaryRows;\n    readonly experiments: BuiltInExperimentRows;\n    readonly counts: {\n        readonly experiments: number;\n        readonly attempts: number;\n    };\n}\ninterface StandardAttemptsPageInput {\n    readonly hero: HeroData;\n    readonly attempts: BuiltInAttemptRows;\n}\ninterface StandardTracesPageInput {\n    readonly hero: HeroData;\n    readonly observability: Awaited<ReturnType<typeof toAttemptObservability>>;\n}\ninterface StandardAttemptPageInput {\n    readonly target: AttemptDetailTarget;\n    readonly evidence: PageEvidence;\n}\ninterface StandardExperimentPageInput {\n    readonly hero: HeroData;\n    readonly experiment: string;\n    readonly rows: BuiltInExperimentRows;\n    readonly members: readonly MembershipRow[];\n}\nfunction heroData(snapshot: SampleSnapshot): HeroData {\n    const latest = snapshot.runs.reduce<number | null>((current, run) => current === null || Number(run.startedAt) > current ? Number(run.startedAt) : current, null);\n    return Object.freeze({\n        latestStartedAt: latest === null ? null : new Date(latest).toISOString(),\n        runs: snapshot.runs.length,\n    });\n}\nfunction membershipRows(snapshot: SampleSnapshot, filter: {\n    readonly locator?: AttemptDetailTarget[\"locator\"];\n    readonly experiment?: string;\n} = {}): readonly MembershipRow[] {\n    return Object.freeze(snapshot.slots\n        .filter((slot) => filter.experiment === undefined || String(slot.experimentId) === filter.experiment)\n        .filter((slot) => filter.locator === undefined || (slot.state === \"included\" && slot.attempt.locator === filter.locator))\n        .map((slot) => Object.freeze({\n        key: `${slot.runId}:${slot.slotId}`,\n        experiment: String(slot.experimentId),\n        eval: String(slot.evalId),\n        attempt: slot.attemptOrdinal,\n        selectedRun: String(slot.runId),\n        slot: String(slot.slotId),\n        state: slot.state,\n        relation: slot.state === \"included\" ? slot.relation : null,\n        locator: slot.state === \"included\" ? slot.attempt.locator : null,\n    })));\n}\nfunction includedAttemptCount(snapshot: SampleSnapshot): number {\n    return snapshot.slots.filter((slot) => slot.state === \"included\").length;\n}\n/** The default top-level report Page. */\nexport const standardOverviewPage = {\n    id: \"report\",\n    title: { en: \"Report\", \"zh-CN\": \"报告\" },\n    load: async (sample: Sample): Promise<StandardOverviewPageInput> => {\n        const [summary, experiments] = await Promise.all([\n            loadBuiltInSummaryRows(sample),\n            loadBuiltInExperimentRows(sample),\n        ]);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            summary,\n            experiments,\n            counts: Object.freeze({\n                experiments: experiments.length,\n                attempts: includedAttemptCount(sample.snapshot),\n            }),\n        });\n    },\n    render: (input: StandardOverviewPageInput) => <StandardOverviewResultView {...input}/>,\n} satisfies Page<void, StandardOverviewPageInput>;\n/** The standard attempt list does one aggregation and never queries detail views. */\nexport const standardAttemptsPage = {\n    id: \"attempts\",\n    title: \"Attempts\",\n    load: async (sample: Sample): Promise<StandardAttemptsPageInput> => Object.freeze({\n        hero: heroData(sample.snapshot),\n        attempts: await loadBuiltInAttemptRows(sample),\n    }),\n    render: (input: StandardAttemptsPageInput) => <StandardAttemptsResultView {...input}/>,\n} satisfies Page<void, StandardAttemptsPageInput>;\n/** Trace navigation reads only the observability DomainView needed by this Page. */\nexport const standardTracesPage = {\n    id: \"traces\",\n    title: { en: \"Traces\", \"zh-CN\": \"追踪\" },\n    load: async (sample: Sample): Promise<StandardTracesPageInput> => Object.freeze({\n        hero: heroData(sample.snapshot),\n        observability: await toAttemptObservability(sample),\n    }),\n    render: (input: StandardTracesPageInput) => (<Col>\n      <Hero data={input.hero}/>\n      <AttemptTrace view={input.observability} mode=\"execution\" timingMode=\"full\"/>\n    </Col>),\n} satisfies Page<void, StandardTracesPageInput>;\n/** One standard detail Page for one already-selected Attempt. */\nexport const standardAttemptPage = {\n    id: \"attempt\",\n    path: \"/attempt\",\n    title: \"Attempt\",\n    navigation: false,\n    params: attemptDetailParams,\n    load: async (_sample: Sample, params: AttemptDetailTarget, context): Promise<StandardAttemptPageInput> => Object.freeze({\n        target: params,\n        evidence: await context.evidence(params.locator),\n    }),\n    render: (input: StandardAttemptPageInput) => <AttemptDetails attempt={input}/>,\n} satisfies Page<AttemptDetailTarget, StandardAttemptPageInput>;\nfunction experimentForTarget(sample: Sample, target: ExperimentDetailTarget): ExperimentId {\n    const candidates = new Map<string, ExperimentId>();\n    for (const run of sample.snapshot.runs)\n        candidates.set(String(run.experimentId), run.experimentId);\n    for (const slot of sample.snapshot.slots)\n        candidates.set(String(slot.experimentId), slot.experimentId);\n    for (const candidate of candidates.values()) {\n        if (experimentDetailTarget(candidate).key === target.key)\n            return candidate;\n    }\n    throw new TypeError(\"Experiment Page target does not belong to this Sample\");\n}\n/** One standard detail Page for one fixed-Sample Experiment identity. */\nexport const standardExperimentPage = {\n    id: \"experiment\",\n    path: \"/experiment\",\n    title: \"Experiment\",\n    navigation: false,\n    params: experimentDetailParams,\n    load: async (sample: Sample, params: ExperimentDetailTarget): Promise<StandardExperimentPageInput> => {\n        const experiment = experimentForTarget(sample, params);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            experiment: String(experiment),\n            rows: await loadBuiltInExperimentRows(sample),\n            members: membershipRows(sample.snapshot, { experiment: String(experiment) }),\n        });\n    },\n    render: (input: StandardExperimentPageInput) => <ExperimentDetailResultView {...input}/>,\n} satisfies Page<ExperimentDetailTarget, StandardExperimentPageInput>;\n/** Explicit composition only: standard owns its two detail Pages. */\nexport const standard = defineBuiltInReport(builtInMachineProducerIds.standard, {\n    title: \"NiceEval standard report\",\n    pages: [\n        standardOverviewPage,\n        standardAttemptsPage,\n        standardTracesPage,\n        standardAttemptPage,\n        standardExperimentPage,\n    ],\n});\nexport default standard;\n"]}
+{"version":3,"file":"standard.js","sourceRoot":"","sources":["standard.tsx"],"names":[],"mappings":";;;;AASA,gEAAmD;AACnD,qEAAwE;AACxE,sEAA8E;AAC9E,4DAAiE;AACjE,sDAA6G;AAE7G,6DAAgM;AAChM,kEAAgK;AAChK,6CAA+E;AA6B/E,SAAS,QAAQ,CAAC,QAAwB;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAClK,OAAO,MAAM,CAAC,MAAM,CAAC;QACjB,eAAe,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE;QACxE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;KAC7B,CAAC,CAAC;AACP,CAAC;AACD,SAAS,cAAc,CAAC,QAAwB,EAAE,SAG9C,EAAE;IACF,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK;SAC9B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,MAAM,CAAC,UAAU,CAAC;SACpG,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC;SACxH,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC7B,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;QACnC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,EAAE,IAAI,CAAC,cAAc;QAC5B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QAC1D,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,CAAC,CAAC,CAAC;AACT,CAAC;AACD,SAAS,oBAAoB,CAAC,QAAwB;IAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;AAC7E,CAAC;AACD,yCAAyC;AAC5B,QAAA,oBAAoB,GAAG;IAChC,EAAE,EAAE,QAAQ;IACZ,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;IACtC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAsC,EAAE;QAC/D,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC7C,IAAA,2CAAsB,EAAC,MAAM,CAAC;YAC9B,IAAA,8CAAyB,EAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,OAAO;YACP,WAAW;YACX,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;gBAClB,WAAW,EAAE,WAAW,CAAC,MAAM;gBAC/B,QAAQ,EAAE,oBAAoB,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC/C,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,MAAM;aAC5F,CAAC;SACL,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAAgC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACzC,CAAC;AAClD,qFAAqF;AACxE,QAAA,oBAAoB,GAAG;IAChC,EAAE,EAAE,UAAU;IACd,KAAK,EAAE,UAAU;IACjB,IAAI,EAAE,KAAK,EAAE,MAAc,EAAsC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC9E,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC/B,QAAQ,EAAE,MAAM,IAAA,2CAAsB,EAAC,MAAM,CAAC;KACjD,CAAC;IACF,MAAM,EAAE,CAAC,KAAgC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACzC,CAAC;AAClD,oFAAoF;AACvE,QAAA,kBAAkB,GAAG;IAC9B,EAAE,EAAE,QAAQ;IACZ,KAAK,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;IACtC,IAAI,EAAE,KAAK,EAAE,MAAc,EAAoC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5E,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC/B,aAAa,EAAE,MAAM,IAAA,uCAAsB,EAAC,MAAM,CAAC;KACtD,CAAC;IACF,MAAM,EAAE,CAAC,KAA8B,EAAE,EAAE,CAAC,CAAC,wBAAC,oBAAG,eAC/C,uBAAC,gBAAI,IAAC,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,EACzB,uBAAC,oCAAY,IAAC,IAAI,EAAE,KAAK,CAAC,aAAa,EAAE,IAAI,EAAC,WAAW,EAAC,UAAU,EAAC,MAAM,GAAE,IACzE,CAAC;CACoC,CAAC;AAChD,iEAAiE;AACpD,QAAA,mBAAmB,GAAG;IAC/B,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,SAAS;IAChB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,gCAAmB;IAC3B,IAAI,EAAE,KAAK,EAAE,OAAe,EAAE,MAA2B,EAAE,OAAO,EAAqC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QACpH,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;KACnD,CAAC;IACF,MAAM,EAAE,CAAC,KAA+B,EAAE,EAAE,CAAC,uBAAC,0BAAc,IAAC,OAAO,EAAE,KAAK,GAAG;CACnB,CAAC;AAChE,SAAS,mBAAmB,CAAC,MAAc,EAAE,MAA8B;IACvE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI;QAClC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;IAC/D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK;QACpC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACjE,KAAK,MAAM,SAAS,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,IAAA,mCAAsB,EAAC,SAAS,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG;YACpD,OAAO,SAAS,CAAC;IACzB,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,uDAAuD,CAAC,CAAC;AACjF,CAAC;AACD,yEAAyE;AAC5D,QAAA,sBAAsB,GAAG;IAClC,EAAE,EAAE,YAAY;IAChB,IAAI,EAAE,aAAa;IACnB,KAAK,EAAE,YAAY;IACnB,UAAU,EAAE,KAAK;IACjB,MAAM,EAAE,mCAAsB;IAC9B,IAAI,EAAE,KAAK,EAAE,MAAc,EAAE,MAA8B,EAAwC,EAAE;QACjG,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvD,OAAO,MAAM,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/B,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;YAC9B,IAAI,EAAE,MAAM,IAAA,8CAAyB,EAAC,MAAM,CAAC;YAC7C,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;SAC/E,CAAC,CAAC;IACP,CAAC;IACD,MAAM,EAAE,CAAC,KAAkC,EAAE,EAAE,CAAC,uBAAC,kDAA0B,OAAK,KAAK,GAAG;CACvB,CAAC;AACtE,qEAAqE;AACxD,QAAA,QAAQ,GAAG,IAAA,gCAAmB,EAAC,sCAAyB,CAAC,QAAQ,EAAE;IAC5E,KAAK,EAAE,0BAA0B;IACjC,KAAK,EAAE;QACH,4BAAoB;QACpB,4BAAoB;QACpB,0BAAkB;QAClB,2BAAmB;QACnB,8BAAsB;KACzB;CACJ,CAAC,CAAC;AACH,kBAAe,gBAAQ,CAAC","sourcesContent":["/**\n * The standard information architecture.\n *\n * Detail Pages are deliberately listed here. `defineReport()` receives no\n * library page collection, so a consumer's arbitrary custom Report never\n * gains attempt or experiment routes as an implicit side effect.\n */\nimport type { ExperimentId, Sample, SampleSnapshot, } from \"../../analysis/index.ts\";\nimport type { Page, PageEvidence, } from \"../definition/report.ts\";\nimport { Col } from \"../definition/primitives.tsx\";\nimport { AttemptDetails } from \"../components/attempt-detail/index.tsx\";\nimport { Hero, type HeroData } from \"../components/site-components/index.tsx\";\nimport { toAttemptObservability } from \"../model/conversions.ts\";\nimport { attemptDetailParams, experimentDetailParams, experimentDetailTarget, } from \"../library/details.ts\";\nimport type { AttemptDetailTarget, ExperimentDetailTarget, } from \"../library/details.ts\";\nimport { loadBuiltInAttemptRows, loadBuiltInExperimentRows, loadBuiltInSummaryRows, type BuiltInAttemptRows, type BuiltInExperimentRows, type BuiltInSummaryRows, } from \"./analysis-values.ts\";\nimport { AttemptTrace, ExperimentDetailResultView, StandardAttemptsResultView, StandardOverviewResultView, type MembershipRow, } from \"./result-components.tsx\";\nimport { builtInMachineProducerIds, defineBuiltInReport, } from \"./machine.ts\";\ninterface StandardOverviewPageInput {\n    readonly hero: HeroData;\n    readonly summary: BuiltInSummaryRows;\n    readonly experiments: BuiltInExperimentRows;\n    readonly counts: {\n        readonly experiments: number;\n        readonly attempts: number;\n        readonly expectedResults: number;\n    };\n}\ninterface StandardAttemptsPageInput {\n    readonly hero: HeroData;\n    readonly attempts: BuiltInAttemptRows;\n}\ninterface StandardTracesPageInput {\n    readonly hero: HeroData;\n    readonly observability: Awaited<ReturnType<typeof toAttemptObservability>>;\n}\ninterface StandardAttemptPageInput {\n    readonly target: AttemptDetailTarget;\n    readonly evidence: PageEvidence;\n}\ninterface StandardExperimentPageInput {\n    readonly hero: HeroData;\n    readonly experiment: string;\n    readonly rows: BuiltInExperimentRows;\n    readonly members: readonly MembershipRow[];\n}\nfunction heroData(snapshot: SampleSnapshot): HeroData {\n    const latest = snapshot.runs.reduce<number | null>((current, run) => current === null || Number(run.startedAt) > current ? Number(run.startedAt) : current, null);\n    return Object.freeze({\n        latestStartedAt: latest === null ? null : new Date(latest).toISOString(),\n        runs: snapshot.runs.length,\n    });\n}\nfunction membershipRows(snapshot: SampleSnapshot, filter: {\n    readonly locator?: AttemptDetailTarget[\"locator\"];\n    readonly experiment?: string;\n} = {}): readonly MembershipRow[] {\n    return Object.freeze(snapshot.slots\n        .filter((slot) => filter.experiment === undefined || String(slot.experimentId) === filter.experiment)\n        .filter((slot) => filter.locator === undefined || (slot.state === \"included\" && slot.attempt.locator === filter.locator))\n        .map((slot) => Object.freeze({\n        key: `${slot.runId}:${slot.slotId}`,\n        experiment: String(slot.experimentId),\n        eval: String(slot.evalId),\n        attempt: slot.attemptOrdinal,\n        selectedRun: String(slot.runId),\n        slot: String(slot.slotId),\n        state: slot.state,\n        relation: slot.state === \"included\" ? slot.relation : null,\n        locator: slot.state === \"included\" ? slot.attempt.locator : null,\n    })));\n}\nfunction includedAttemptCount(snapshot: SampleSnapshot): number {\n    return snapshot.slots.filter((slot) => slot.state === \"included\").length;\n}\n/** The default top-level report Page. */\nexport const standardOverviewPage = {\n    id: \"report\",\n    title: { en: \"Report\", \"zh-CN\": \"报告\" },\n    load: async (sample: Sample): Promise<StandardOverviewPageInput> => {\n        const [summary, experiments] = await Promise.all([\n            loadBuiltInSummaryRows(sample),\n            loadBuiltInExperimentRows(sample),\n        ]);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            summary,\n            experiments,\n            counts: Object.freeze({\n                experiments: experiments.length,\n                attempts: includedAttemptCount(sample.snapshot),\n                expectedResults: sample.snapshot.slots.filter((slot) => slot.state !== \"excluded\").length,\n            }),\n        });\n    },\n    render: (input: StandardOverviewPageInput) => <StandardOverviewResultView {...input}/>,\n} satisfies Page<void, StandardOverviewPageInput>;\n/** The standard attempt list does one aggregation and never queries detail views. */\nexport const standardAttemptsPage = {\n    id: \"attempts\",\n    title: \"Attempts\",\n    load: async (sample: Sample): Promise<StandardAttemptsPageInput> => Object.freeze({\n        hero: heroData(sample.snapshot),\n        attempts: await loadBuiltInAttemptRows(sample),\n    }),\n    render: (input: StandardAttemptsPageInput) => <StandardAttemptsResultView {...input}/>,\n} satisfies Page<void, StandardAttemptsPageInput>;\n/** Trace navigation reads only the observability DomainView needed by this Page. */\nexport const standardTracesPage = {\n    id: \"traces\",\n    title: { en: \"Traces\", \"zh-CN\": \"追踪\" },\n    load: async (sample: Sample): Promise<StandardTracesPageInput> => Object.freeze({\n        hero: heroData(sample.snapshot),\n        observability: await toAttemptObservability(sample),\n    }),\n    render: (input: StandardTracesPageInput) => (<Col>\n      <Hero data={input.hero}/>\n      <AttemptTrace view={input.observability} mode=\"execution\" timingMode=\"full\"/>\n    </Col>),\n} satisfies Page<void, StandardTracesPageInput>;\n/** One standard detail Page for one already-selected Attempt. */\nexport const standardAttemptPage = {\n    id: \"attempt\",\n    path: \"/attempt\",\n    title: \"Attempt\",\n    navigation: false,\n    params: attemptDetailParams,\n    load: async (_sample: Sample, params: AttemptDetailTarget, context): Promise<StandardAttemptPageInput> => Object.freeze({\n        target: params,\n        evidence: await context.evidence(params.locator),\n    }),\n    render: (input: StandardAttemptPageInput) => <AttemptDetails attempt={input}/>,\n} satisfies Page<AttemptDetailTarget, StandardAttemptPageInput>;\nfunction experimentForTarget(sample: Sample, target: ExperimentDetailTarget): ExperimentId {\n    const candidates = new Map<string, ExperimentId>();\n    for (const run of sample.snapshot.runs)\n        candidates.set(String(run.experimentId), run.experimentId);\n    for (const slot of sample.snapshot.slots)\n        candidates.set(String(slot.experimentId), slot.experimentId);\n    for (const candidate of candidates.values()) {\n        if (experimentDetailTarget(candidate).key === target.key)\n            return candidate;\n    }\n    throw new TypeError(\"Experiment Page target does not belong to this Sample\");\n}\n/** One standard detail Page for one fixed-Sample Experiment identity. */\nexport const standardExperimentPage = {\n    id: \"experiment\",\n    path: \"/experiment\",\n    title: \"Experiment\",\n    navigation: false,\n    params: experimentDetailParams,\n    load: async (sample: Sample, params: ExperimentDetailTarget): Promise<StandardExperimentPageInput> => {\n        const experiment = experimentForTarget(sample, params);\n        return Object.freeze({\n            hero: heroData(sample.snapshot),\n            experiment: String(experiment),\n            rows: await loadBuiltInExperimentRows(sample),\n            members: membershipRows(sample.snapshot, { experiment: String(experiment) }),\n        });\n    },\n    render: (input: StandardExperimentPageInput) => <ExperimentDetailResultView {...input}/>,\n} satisfies Page<ExperimentDetailTarget, StandardExperimentPageInput>;\n/** Explicit composition only: standard owns its two detail Pages. */\nexport const standard = defineBuiltInReport(builtInMachineProducerIds.standard, {\n    title: \"NiceEval standard report\",\n    pages: [\n        standardOverviewPage,\n        standardAttemptsPage,\n        standardTracesPage,\n        standardAttemptPage,\n        standardExperimentPage,\n    ],\n});\nexport default standard;\n"]}

package/dist/report/built-in/standard.d.cts

@@ -19,6 +19,7 @@ interface StandardOverviewPageInput {
     readonly counts: {
         readonly experiments: number;
         readonly attempts: number;
+        readonly expectedResults: number;
     };
 }
 interface StandardAttemptsPageInput {

package/dist/report/built-in/standard.d.mts

@@ -19,6 +19,7 @@ interface StandardOverviewPageInput {
     readonly counts: {
         readonly experiments: number;
         readonly attempts: number;
+        readonly expectedResults: number;
     };
 }
 interface StandardAttemptsPageInput {

package/dist/report/built-in/standard.d.ts

@@ -19,6 +19,7 @@ interface StandardOverviewPageInput {
     readonly counts: {
         readonly experiments: number;
         readonly attempts: number;
+        readonly expectedResults: number;
     };
 }
 interface StandardAttemptsPageInput {

package/dist/report/components/entity-lists/compute.cjs

@@ -8,6 +8,7 @@ const aggregate_ts_1 = require("../../model/aggregate.cjs");
 const metrics_ts_1 = require("../../model/metrics.cjs");
 const conversions_ts_1 = require("../../model/conversions.cjs");
 const details_ts_1 = require("../../library/details.cjs");
+const compute_ts_1 = require("../attempt-detail/compute.cjs");
 /** 相对已表达前缀的剩余路径;无前缀时是完整 evalId。 */
 function relativeEvalLabel(evalIdValue, labelPrefix) {
     if (!labelPrefix)
@@ -100,9 +101,21 @@ async function evidenceFacts(sample) {
     const facts = new Map();
     for (const entry of view.entries) {
         const locator = entry.attempt.locator;
+        const assertions = entry.state === "available"
+            ? entry.detail.entries.map(compute_ts_1.assertionEntryViewOf).filter((item) => item !== undefined)
+            : [];
+        const scoreContributions = assertions
+            .map((item) => item.result.score)
+            .filter((score) => score.state !== "not-scored");
+        const completeScore = scoreContributions.length > 0 &&
+            scoreContributions.every((score) => score.state === "earned" && score.earned !== undefined)
+            ? scoreContributions.reduce((sum, score) => sum + (score.earned ?? 0), 0)
+            : undefined;
         facts.set(locator, Object.freeze({
             verdict: entry.state === "available" ? entry.detail.verdict : null,
             failureSummary: closedFailureSummary(entry),
+            evaluationKind: scoreContributions.length > 0 ? "points" : "pass",
+            ...(completeScore === undefined ? {} : { totalScore: completeScore }),
         }));
     }
     return facts;
@@ -279,7 +292,11 @@ async function attemptListData(sample, pricing = null) {
     ]);
     const selections = selectedSlots(sample);
     return Object.freeze(selections.map((selection) => {
-        const fact = facts.get(selection.locator) ?? Object.freeze({ verdict: null, failureSummary: null });
+        const fact = facts.get(selection.locator) ?? Object.freeze({
+            verdict: null,
+            failureSummary: null,
+            evaluationKind: "pass",
+        });
         const metric = requireMetrics(metrics.get(metricsKey(selection.experimentId, selection.evalId, selection.locator)), `attempt ${JSON.stringify(selection.locator)}`);
         const item = Object.freeze({
             locator: selection.locator,
@@ -288,6 +305,8 @@ async function attemptListData(sample, pricing = null) {
             attemptOrdinal: selection.attemptOrdinal,
             verdict: fact.verdict,
             failureSummary: fact.failureSummary,
+            evaluationKind: fact.evaluationKind,
+            ...(fact.totalScore === undefined ? {} : { totalScore: fact.totalScore }),
             passRate: metric.passRate,
             durationMs: metric.durationMs,
             ...(metric.costUSD === undefined ? {} : { costUSD: metric.costUSD }),
@@ -367,8 +386,14 @@ async function experimentListData(sample, pricing = null) {
             const list = attemptsByEval.get(metricsKey(experimentId, evalIdValue)) ?? [];
             const sorted = [...list].sort((left, right) => left.attemptOrdinal - right.attemptOrdinal || left.locator.localeCompare(right.locator));
             const metrics = requireMetrics(evalMetrics.get(metricsKey(experimentId, evalIdValue)), `experiment/eval ${JSON.stringify(experimentId)}/${JSON.stringify(evalIdValue)}`);
+            const evaluationKind = sorted.some((attempt) => attempt.evaluationKind === "points") ? "points" : "pass";
+            const completeScores = sorted.flatMap((attempt) => attempt.totalScore === undefined ? [] : [attempt.totalScore]);
             evalRows.push(Object.freeze({
                 evalId: evalIdValue,
+                evaluationKind,
+                ...(completeScores.length === 0
+                    ? {}
+                    : { totalScore: completeScores.reduce((sum, score) => sum + score, 0) / completeScores.length }),
                 endToEndPassRate: metrics.passRate,
                 durationMs: metrics.durationMs,
                 ...(metrics.costUSD === undefined ? {} : { costUSD: metrics.costUSD }),
@@ -385,12 +410,19 @@ async function experimentListData(sample, pricing = null) {
             if (metric !== undefined)
                 groupMetricsForItem.set(prefix, metric);
         }
+        const evaluationKind = evalRows.some((row) => row.evaluationKind === "points")
+            ? evalRows.some((row) => row.evaluationKind === "pass") ? "mixed" : "points"
+            : "pass";
+        const completeEvalScores = evalRows.flatMap((row) => row.totalScore === undefined ? [] : [row.totalScore]);
         items.push(Object.freeze({
             experimentId,
             agent: watermark?.agent ?? null,
             model: watermark?.model ?? null,
             flags: watermark?.flags ?? null,
-            evaluationKind: "pass",
+            evaluationKind,
+            ...(completeEvalScores.length === 0
+                ? {}
+                : { totalScore: completeEvalScores.reduce((sum, score) => sum + score, 0) }),
             endToEndPassRate: experimentMetric.passRate,
             durationMs: experimentMetric.durationMs,
             ...(experimentMetric.costUSD === undefined ? {} : { costUSD: experimentMetric.costUSD }),
@@ -401,7 +433,12 @@ async function experimentListData(sample, pricing = null) {
             href: experimentHref(experimentId),
         }));
     }
-    items.sort(byMetricDescThenId((item) => item.endToEndPassRate.value));
+    const composition = items.some((item) => item.evaluationKind !== "pass")
+        ? items.some((item) => item.evaluationKind !== "points") ? "mixed" : "points"
+        : "pass";
+    items.sort(byMetricDescThenId((item) => composition !== "pass"
+        ? item.totalScore ?? null
+        : item.endToEndPassRate.value));
     return Object.freeze(items);
 }
 function evalIdsUnder(node) {