@bolt-foundry/gambit 0.8.6-rc.1 → 0.8.6-rc.3

package/script/simulator-ui/isograph/components/Query/SimulatorVerifyPage.js

@@ -20,12 +20,13 @@ const PageGrid_js_1 = __importDefault(require("../../../src/gds/PageGrid.js"));
 const PageShell_js_1 = __importDefault(require("../../../src/gds/PageShell.js"));
 const Panel_js_1 = __importDefault(require("../../../src/gds/Panel.js"));
 const utils_js_1 = require("../../../src/utils.js");
-const verify_metrics_js_1 = require("../../../src/verify_metrics.js");
-const MAX_BATCH_SIZE = 24;
-const MAX_BATCH_CONCURRENCY = 6;
-const DEFAULT_BATCH_SIZE = 8;
-const DEFAULT_BATCH_CONCURRENCY = 3;
-const NO_SCENARIO_RUN_VALUE = "__workspace_context__";
+const verify_unified_js_1 = require("../../../src/verify_unified.js");
+const MAX_SCENARIO_RUNS = verify_unified_js_1.VERIFY_LIMITS.scenarioRunsMax;
+const MAX_GRADER_REPEATS = verify_unified_js_1.VERIFY_LIMITS.graderRepeatsMax;
+const MAX_BATCH_CONCURRENCY = verify_unified_js_1.VERIFY_LIMITS.concurrencyMax;
+const DEFAULT_SCENARIO_RUNS = verify_unified_js_1.VERIFY_DEFAULTS.scenarioRuns;
+const DEFAULT_GRADER_REPEATS = verify_unified_js_1.VERIFY_DEFAULTS.graderRepeatsPerScenario;
+const DEFAULT_BATCH_CONCURRENCY = verify_unified_js_1.VERIFY_DEFAULTS.concurrency;
 function getRoutePrefix(path) {
     return path === "/isograph" || path.startsWith("/isograph/")
         ? "/isograph"
@@ -55,19 +56,19 @@ function clampInt(value, min, max) {
     const rounded = Number.isFinite(value) ? Math.round(value) : min;
     return Math.max(min, Math.min(max, rounded));
 }
+function formatPercent(value) {
+    if (typeof value !== "number" || !Number.isFinite(value))
+        return "-";
+    return `${Math.round(value * 100)}%`;
+}
 exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
   field Workspace.VerifyTab @component {
     id
-    scenarioRuns(first: 50) {
-      edges {
-        node {
-          id
-          status
-          startedAt
-          finishedAt
-          error
-        }
-      }
+    scenarioDecks {
+      id
+      label
+      description
+      path
     }
     verification {
       graderDecks(first: 50) {
@@ -85,8 +86,10 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
           node {
             id
             workspaceId
+            scenarioDeckId
             graderId
-            scenarioRunId
+            scenarioRuns
+            graderRepeatsPerScenario
             status
             startedAt
             finishedAt
@@ -94,10 +97,13 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
             active
             completed
             failed
-            requests(first: 50) {
+            scenarioRunsCompleted
+            scenarioRunsFailed
+            requests(first: 200) {
               edges {
                 node {
                   id
+                  scenarioRunId
                   status
                   runId
                   error
@@ -105,31 +111,48 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
               }
             }
             metrics {
-              sampleSize
-              agreementRate
-              scoreSpreadMin
-              scoreSpreadMedian
-              scoreSpreadMax
-              instabilityCount
-              verdict
-              verdictReason
-              outliers(first: 25) {
+              scenarioRunCountRequested
+              scenarioRunCountCompleted
+              scenarioRunCountFailed
+              gradeSampleCountRequested
+              gradeSampleCountCompleted
+              gradeSampleCountFailed
+              executionFailureCount
+              gradingFailureCount
+              passRate
+              scoreMin
+              scoreMedian
+              scoreMax
+              scoreMean
+              outlierScenarioRuns(first: 25) {
                 edges {
                   node {
                     key
-                    label
-                    sampleSize
-                    agreementRate
-                    scoreDelta
-                    passFlip
-                    instability
+                    scenarioRunId
+                    gradeSampleCount
+                    completedSampleCount
+                    executionFailureCount
+                    gradingFailureCount
+                    averageScore
+                    minScore
+                    maxScore
+                    failed
                     minRunId
                     maxRunId
-                    turnIndex
                     messageRefId
                   }
                 }
               }
+              failureReasons(first: 25) {
+                edges {
+                  node {
+                    key
+                    kind
+                    reason
+                    count
+                  }
+                }
+              }
             }
           }
         }
@@ -145,6 +168,16 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
     const toPrefixedPath = (0, react_1.useCallback)((path) => `${routePrefix}${path}`, [routePrefix]);
     const runBatchMutation = (0, useGambitTypedMutation_js_1.useGambitTypedMutation)(GambitWorkspaceVerifyBatchRunCreateMutation_js_1.default);
     (0, useGambitTypedSubscription_js_1.useGambitTypedSubscription)(GambitWorkspaceVerifyLiveSubscription_js_1.default, workspaceId ? { workspaceId } : null);
+    const scenarioDecks = (0, react_1.useMemo)(() => data.scenarioDecks?.flatMap((deck) => {
+        if (!deck?.id || !deck.label)
+            return [];
+        return [{
+                id: deck.id,
+                label: deck.label,
+                description: deck.description ?? null,
+                path: deck.path ?? "",
+            }];
+    }) ?? [], [data.scenarioDecks]);
     const graders = (0, react_1.useMemo)(() => (data.verification?.graderDecks?.edges ?? []).flatMap((edge) => {
         const grader = edge?.node;
         if (!grader?.id || !grader.label)
@@ -156,30 +189,16 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                 path: grader.path ?? "",
             }];
     }), [data.verification?.graderDecks?.edges]);
-    const scenarioRuns = (0, react_1.useMemo)(() => (data.scenarioRuns?.edges ?? []).flatMap((edge) => {
-        const run = edge?.node;
-        if (!run?.id)
-            return [];
-        return [{
-                id: run.id,
-                status: toBatchStatus(run.status),
-                startedAt: run.startedAt ?? null,
-                finishedAt: run.finishedAt ?? null,
-                error: run.error ?? null,
-            }];
-    }).sort((left, right) => {
-        const leftKey = left.finishedAt ?? left.startedAt ?? left.id;
-        const rightKey = right.finishedAt ?? right.startedAt ?? right.id;
-        return rightKey.localeCompare(leftKey);
-    }), [data.scenarioRuns?.edges]);
     const batches = (0, react_1.useMemo)(() => (data.verification?.batches?.edges ?? []).flatMap((edge) => {
         const batch = edge?.node;
         if (!batch?.id || !batch.graderId)
             return [];
         return [{
                 id: batch.id,
+                scenarioDeckId: batch.scenarioDeckId ?? null,
                 graderId: batch.graderId,
-                scenarioRunId: batch.scenarioRunId ?? null,
+                scenarioRuns: batch.scenarioRuns ?? 0,
+                graderRepeatsPerScenario: batch.graderRepeatsPerScenario ?? 0,
                 status: toBatchStatus(batch.status),
                 startedAt: batch.startedAt ?? null,
                 finishedAt: batch.finishedAt ?? null,
@@ -187,12 +206,15 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                 active: batch.active ?? 0,
                 completed: batch.completed ?? 0,
                 failed: batch.failed ?? 0,
+                scenarioRunsCompleted: batch.scenarioRunsCompleted ?? 0,
+                scenarioRunsFailed: batch.scenarioRunsFailed ?? 0,
                 requests: (batch.requests?.edges ?? []).flatMap((requestEdge) => {
                     const request = requestEdge?.node;
                     if (!request?.id)
                         return [];
                     return [{
                             id: request.id,
+                            scenarioRunId: request.scenarioRunId ?? undefined,
                             status: toBatchRequestStatus(request.status),
                             runId: request.runId ?? undefined,
                             error: request.error ?? undefined,
@@ -200,59 +222,76 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                 }),
                 metrics: batch.metrics
                     ? {
-                        sampleSize: batch.metrics.sampleSize ?? 0,
-                        agreementRate: typeof batch.metrics.agreementRate === "number"
-                            ? batch.metrics.agreementRate
+                        scenarioRunCountRequested: batch.metrics.scenarioRunCountRequested ?? 0,
+                        scenarioRunCountCompleted: batch.metrics.scenarioRunCountCompleted ?? 0,
+                        scenarioRunCountFailed: batch.metrics.scenarioRunCountFailed ?? 0,
+                        gradeSampleCountRequested: batch.metrics.gradeSampleCountRequested ?? 0,
+                        gradeSampleCountCompleted: batch.metrics.gradeSampleCountCompleted ?? 0,
+                        gradeSampleCountFailed: batch.metrics.gradeSampleCountFailed ?? 0,
+                        executionFailureCount: batch.metrics.executionFailureCount ?? 0,
+                        gradingFailureCount: batch.metrics.gradingFailureCount ?? 0,
+                        passRate: typeof batch.metrics.passRate === "number"
+                            ? batch.metrics.passRate
                             : null,
-                        scoreSpreadMin: typeof batch.metrics.scoreSpreadMin === "number"
-                            ? batch.metrics.scoreSpreadMin
+                        scoreMin: typeof batch.metrics.scoreMin === "number"
+                            ? batch.metrics.scoreMin
                             : null,
-                        scoreSpreadMedian: typeof batch.metrics.scoreSpreadMedian === "number"
-                            ? batch.metrics.scoreSpreadMedian
+                        scoreMedian: typeof batch.metrics.scoreMedian === "number"
+                            ? batch.metrics.scoreMedian
                             : null,
-                        scoreSpreadMax: typeof batch.metrics.scoreSpreadMax === "number"
-                            ? batch.metrics.scoreSpreadMax
+                        scoreMax: typeof batch.metrics.scoreMax === "number"
+                            ? batch.metrics.scoreMax
                             : null,
-                        instabilityCount: batch.metrics.instabilityCount ?? 0,
-                        verdict: batch.metrics.verdict === "PASS" ||
-                            batch.metrics.verdict === "WARN" ||
-                            batch.metrics.verdict === "FAIL"
-                            ? batch.metrics.verdict
-                            : "WARN",
-                        verdictReason: batch.metrics.verdictReason ??
-                            "Verify batch completed.",
-                        outliers: (batch.metrics.outliers?.edges ?? []).flatMap((outlierEdge) => {
+                        scoreMean: typeof batch.metrics.scoreMean === "number"
+                            ? batch.metrics.scoreMean
+                            : null,
+                        outlierScenarioRuns: (batch.metrics.outlierScenarioRuns?.edges ?? []).flatMap((outlierEdge) => {
                             const outlier = outlierEdge?.node;
-                            if (!outlier?.key || !outlier.label)
+                            if (!outlier?.key || !outlier.scenarioRunId)
                                 return [];
                             return [{
                                     key: outlier.key,
-                                    label: outlier.label,
-                                    sampleSize: outlier.sampleSize ?? 0,
-                                    agreementRate: typeof outlier.agreementRate === "number"
-                                        ? outlier.agreementRate
+                                    scenarioRunId: outlier.scenarioRunId,
+                                    gradeSampleCount: outlier.gradeSampleCount ?? 0,
+                                    completedSampleCount: outlier.completedSampleCount ?? 0,
+                                    executionFailureCount: outlier.executionFailureCount ?? 0,
+                                    gradingFailureCount: outlier.gradingFailureCount ?? 0,
+                                    averageScore: typeof outlier.averageScore === "number"
+                                        ? outlier.averageScore
+                                        : null,
+                                    minScore: typeof outlier.minScore === "number"
+                                        ? outlier.minScore
                                         : null,
-                                    scoreDelta: typeof outlier.scoreDelta === "number"
-                                        ? outlier.scoreDelta
+                                    maxScore: typeof outlier.maxScore === "number"
+                                        ? outlier.maxScore
                                         : null,
-                                    passFlip: Boolean(outlier.passFlip),
-                                    instability: Boolean(outlier.instability),
+                                    failed: Boolean(outlier.failed),
                                     minRunId: outlier.minRunId ?? undefined,
                                     maxRunId: outlier.maxRunId ?? undefined,
-                                    turnIndex: typeof outlier.turnIndex === "number"
-                                        ? outlier.turnIndex
-                                        : undefined,
                                     messageRefId: outlier.messageRefId ?? undefined,
                                 }];
                         }),
+                        failureReasons: (batch.metrics.failureReasons?.edges ?? [])
+                            .flatMap((reasonEdge) => {
+                            const reason = reasonEdge?.node;
+                            if (!reason?.key || !reason.reason)
+                                return [];
+                            return [{
+                                    key: reason.key,
+                                    kind: reason.kind === "GRADING" ? "grading" : "execution",
+                                    reason: reason.reason,
+                                    count: reason.count ?? 0,
+                                }];
+                        }),
                     }
                     : null,
             }];
     }), [data.verification?.batches?.edges]);
-    const [selectedScenarioRunId, setSelectedScenarioRunId] = (0, react_1.useState)(null);
+    const [selectedScenarioDeckId, setSelectedScenarioDeckId] = (0, react_1.useState)(null);
     const [selectedGraderId, setSelectedGraderId] = (0, react_1.useState)(null);
     const [selectedBatchId, setSelectedBatchId] = (0, react_1.useState)(null);
-    const [batchSize, setBatchSize] = (0, react_1.useState)(DEFAULT_BATCH_SIZE);
+    const [scenarioRuns, setScenarioRuns] = (0, react_1.useState)(DEFAULT_SCENARIO_RUNS);
+    const [graderRepeatsPerScenario, setGraderRepeatsPerScenario] = (0, react_1.useState)(DEFAULT_GRADER_REPEATS);
     const [batchConcurrency, setBatchConcurrency] = (0, react_1.useState)(DEFAULT_BATCH_CONCURRENCY);
     const [mutationError, setMutationError] = (0, react_1.useState)(null);
     (0, react_1.useEffect)(() => {
@@ -263,22 +302,23 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
         setSelectedGraderId(graders[0]?.id ?? null);
     }, [graders, selectedGraderId]);
     (0, react_1.useEffect)(() => {
-        if (selectedScenarioRunId &&
-            scenarioRuns.some((scenarioRun) => scenarioRun.id === selectedScenarioRunId)) {
+        if (selectedScenarioDeckId &&
+            scenarioDecks.some((deck) => deck.id === selectedScenarioDeckId)) {
             return;
         }
-        setSelectedScenarioRunId(scenarioRuns[0]?.id ?? null);
-    }, [scenarioRuns, selectedScenarioRunId]);
+        setSelectedScenarioDeckId(scenarioDecks[0]?.id ?? null);
+    }, [scenarioDecks, selectedScenarioDeckId]);
     const filteredBatches = (0, react_1.useMemo)(() => {
         return batches.filter((batch) => {
             if (selectedGraderId && batch.graderId !== selectedGraderId)
                 return false;
-            if (selectedScenarioRunId && batch.scenarioRunId !== selectedScenarioRunId) {
+            if (selectedScenarioDeckId &&
+                batch.scenarioDeckId !== selectedScenarioDeckId) {
                 return false;
             }
             return true;
         });
-    }, [batches, selectedGraderId, selectedScenarioRunId]);
+    }, [batches, selectedGraderId, selectedScenarioDeckId]);
     const visibleBatches = filteredBatches.length > 0 ? filteredBatches : batches;
     (0, react_1.useEffect)(() => {
         if (selectedBatchId &&
@@ -296,10 +336,11 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
         return visibleBatches[0] ?? null;
     }, [selectedBatchId, visibleBatches]);
     const selectedGrader = (0, react_1.useMemo)(() => graders.find((grader) => grader.id === selectedGraderId) ?? null, [graders, selectedGraderId]);
-    const queuedCount = (0, react_1.useMemo)(() => (selectedBatch?.requests ?? []).filter((request) => request.status === "queued")
-        .length, [selectedBatch?.requests]);
+    const selectedScenarioDeck = (0, react_1.useMemo)(() => scenarioDecks.find((deck) => deck.id === selectedScenarioDeckId) ?? null, [scenarioDecks, selectedScenarioDeckId]);
+    const queuedCount = (0, react_1.useMemo)(() => (selectedBatch?.requests ?? []).filter((request) => request.status === "queued").length, [selectedBatch?.requests]);
     const metrics = selectedBatch?.metrics ?? null;
-    const topOutliers = (metrics?.outliers ?? []).slice(0, 8);
+    const topOutlierScenarioRuns = (0, react_1.useMemo)(() => (0, verify_unified_js_1.sortVerifyOutlierScenarioRuns)(metrics?.outlierScenarioRuns ?? [])
+        .slice(0, 8), [metrics?.outlierScenarioRuns]);
     const hasRunningBatch = visibleBatches.some((batch) => batch.status === "running");
     const canRun = Boolean(workspaceId && selectedGraderId && !runBatchMutation.inFlight &&
         !hasRunningBatch);
@@ -307,16 +348,18 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
         if (!workspaceId || !selectedGraderId)
             return;
         setMutationError(null);
-        const nextBatchSize = clampInt(batchSize, 1, MAX_BATCH_SIZE);
-        const nextConcurrency = clampInt(batchConcurrency, 1, Math.min(MAX_BATCH_CONCURRENCY, nextBatchSize));
+        const nextScenarioRuns = clampInt(scenarioRuns, 1, MAX_SCENARIO_RUNS);
+        const nextRepeats = clampInt(graderRepeatsPerScenario, 1, MAX_GRADER_REPEATS);
+        const nextConcurrency = clampInt(batchConcurrency, 1, MAX_BATCH_CONCURRENCY);
         runBatchMutation.commit({
             input: {
                 workspaceId,
-                graderId: selectedGraderId,
-                ...(selectedScenarioRunId
-                    ? { scenarioRunId: selectedScenarioRunId }
+                ...(selectedScenarioDeckId
+                    ? { scenarioDeckId: selectedScenarioDeckId }
                     : {}),
-                batchSize: nextBatchSize,
+                graderId: selectedGraderId,
+                scenarioRuns: nextScenarioRuns,
+                graderRepeatsPerScenario: nextRepeats,
                 concurrency: nextConcurrency,
             },
         }, {
@@ -333,40 +376,28 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
         });
     }, [
         batchConcurrency,
-        batchSize,
+        graderRepeatsPerScenario,
         runBatchMutation,
+        scenarioRuns,
         selectedGraderId,
-        selectedScenarioRunId,
+        selectedScenarioDeckId,
         workspaceId,
     ]);
     const navigateToGradeRun = (0, react_1.useCallback)((runId) => {
         const gradePath = (0, workspace_routes_js_1.buildWorkspacePath)("grade", workspaceId, { runId });
         navigate(toPrefixedPath(gradePath));
     }, [navigate, toPrefixedPath, workspaceId]);
-    const scenarioOptions = (0, react_1.useMemo)(() => [
-        {
-            value: NO_SCENARIO_RUN_VALUE,
-            label: "Current workspace context",
-            meta: "Run without a prior scenario run binding",
-        },
-        ...scenarioRuns.map((run) => ({
-            value: run.id,
-            label: (0, utils_js_1.scenarioNameFromValue)(run.id) ?? run.id,
-            meta: [
-                run.status,
-                run.finishedAt ?? run.startedAt,
-            ].filter(Boolean).join(" · "),
-        })),
-    ], [scenarioRuns]);
     return (React.createElement(PageShell_js_1.default, { className: "verify-shell" },
         React.createElement(PageGrid_js_1.default, { as: "main", className: "verify-layout" },
             React.createElement(Panel_js_1.default, { className: "verify-controls", "data-testid": "verify-tab-scaffold" },
                 React.createElement("div", { className: "verify-controls-header" },
-                    React.createElement("strong", null, "Verify consistency"),
-                    React.createElement("span", { className: "secondary-note" }, "Run repeated grading checks against one grader and scenario.")),
-                scenarioRuns.length > 0 && (React.createElement(Listbox_js_1.default, { label: "Scenario run", value: selectedScenarioRunId ?? NO_SCENARIO_RUN_VALUE, onChange: (runId) => {
-                        setSelectedScenarioRunId(runId === NO_SCENARIO_RUN_VALUE ? null : runId);
-                    }, options: scenarioOptions, placeholder: "Select scenario run" })),
+                    React.createElement("strong", null, "Verify repeated evidence"),
+                    React.createElement("span", { className: "secondary-note" }, "Generate scenario runs, then grade each run repeatedly.")),
+                React.createElement(Listbox_js_1.default, { label: "Scenario deck", value: selectedScenarioDeckId ?? "", onChange: (value) => setSelectedScenarioDeckId(value.length ? value : null), options: scenarioDecks.map((deck) => ({
+                        value: deck.id,
+                        label: deck.label,
+                        meta: deck.path,
+                    })), placeholder: "Select scenario deck", disabled: scenarioDecks.length === 0 }),
                 React.createElement(Listbox_js_1.default, { label: "Grader", value: selectedGraderId ?? "", onChange: (value) => setSelectedGraderId(value.length ? value : null), options: graders.map((grader) => ({
                         value: grader.id,
                         label: grader.label,
@@ -374,19 +405,28 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                     })), placeholder: "Select grader", disabled: graders.length === 0 }),
                 React.createElement("div", { className: "verify-number-grid" },
                     React.createElement("label", { className: "verify-number-field" },
-                        "Batch size",
-                        React.createElement("input", { type: "number", min: 1, max: MAX_BATCH_SIZE, value: batchSize, onChange: (event) => setBatchSize(clampInt(Number(event.target.value), 1, MAX_BATCH_SIZE)) })),
+                        "Scenario runs",
+                        React.createElement("input", { type: "number", min: 1, max: MAX_SCENARIO_RUNS, value: scenarioRuns, onChange: (event) => setScenarioRuns(clampInt(Number(event.target.value), 1, MAX_SCENARIO_RUNS)) })),
+                    React.createElement("label", { className: "verify-number-field" },
+                        "Grader repeats per scenario",
+                        React.createElement("input", { type: "number", min: 1, max: MAX_GRADER_REPEATS, value: graderRepeatsPerScenario, onChange: (event) => setGraderRepeatsPerScenario(clampInt(Number(event.target.value), 1, MAX_GRADER_REPEATS)) })),
                     React.createElement("label", { className: "verify-number-field" },
                         "Concurrency",
                         React.createElement("input", { type: "number", min: 1, max: MAX_BATCH_CONCURRENCY, value: batchConcurrency, onChange: (event) => setBatchConcurrency(clampInt(Number(event.target.value), 1, MAX_BATCH_CONCURRENCY)) }))),
                 React.createElement(Button_js_1.default, { "data-testid": "verify-run-batch", variant: "primary", onClick: runBatch, disabled: !canRun }, hasRunningBatch || runBatchMutation.inFlight
-                    ? "Running consistency batch…"
-                    : "Run consistency batch"),
+                    ? "Running verify batch..."
+                    : "Run verify batch"),
+                scenarioDecks.length === 0 && (React.createElement(Callout_js_1.default, null,
+                    "No scenario decks are available. Add ",
+                    React.createElement("code", null, "[[testDecks]]"),
+                    " ",
+                    "entries to the active root deck.")),
                 graders.length === 0 && (React.createElement(Callout_js_1.default, null,
                     "No graders are available. Add ",
                     React.createElement("code", null, "[[graders]]"),
                     " ",
                     "entries to the active root deck.")),
+                selectedScenarioDeck?.description && (React.createElement(Callout_js_1.default, null, selectedScenarioDeck.description)),
                 selectedGrader?.description && (React.createElement(Callout_js_1.default, null, selectedGrader.description)),
                 React.createElement(Callout_js_1.default, { variant: "emphasis", title: "Build assistant stays available" }, "Use the chat drawer toggle in the top-right corner to investigate and iterate while this page remains open.")),
             React.createElement(Panel_js_1.default, { className: "verify-results", "data-testid": "verify-results" },
@@ -401,93 +441,93 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                                 : "",
                             selectedBatch?.finishedAt
                                 ? ` · finished ${(0, utils_js_1.formatTimestampShort)(selectedBatch.finishedAt)}`
-                                : "")),
-                    metrics && metrics.sampleSize > 0 && (React.createElement("span", { className: (0, utils_js_1.classNames)("verify-verdict-badge", `verify-verdict-badge--${metrics.verdict.toLowerCase()}`) }, metrics.verdict))),
-                selectedBatch && selectedBatch.requested > 0 && (React.createElement("div", { className: "verify-progress-row" },
-                    React.createElement("span", null,
-                        "Queued: ",
-                        queuedCount),
-                    React.createElement("span", null,
-                        "Running: ",
-                        selectedBatch.active),
-                    React.createElement("span", null,
-                        "Completed: ",
-                        selectedBatch.completed),
-                    React.createElement("span", null,
-                        "Failed: ",
-                        selectedBatch.failed))),
-                !selectedBatch && (React.createElement(Callout_js_1.default, null, "Run a consistency batch to compute agreement, spread, and instability for the selected grader.")),
-                metrics && metrics.sampleSize > 0 && (React.createElement(React.Fragment, null,
+                                : ""))),
+                !selectedBatch && (React.createElement(Callout_js_1.default, null, "Run a verify batch to generate repeated grading evidence.")),
+                selectedBatch && selectedBatch.requested > 0 && (React.createElement(React.Fragment, null,
+                    React.createElement("div", { className: "verify-progress-row" },
+                        React.createElement("span", null,
+                            "Scenario runs: ",
+                            selectedBatch.scenarioRunsCompleted,
+                            "/",
+                            selectedBatch.scenarioRuns),
+                        React.createElement("span", null,
+                            "Scenario failures: ",
+                            selectedBatch.scenarioRunsFailed),
+                        React.createElement("span", null,
+                            "Queued: ",
+                            queuedCount),
+                        React.createElement("span", null,
+                            "Running: ",
+                            selectedBatch.active),
+                        React.createElement("span", null,
+                            "Completed: ",
+                            selectedBatch.completed),
+                        React.createElement("span", null,
+                            "Failed: ",
+                            selectedBatch.failed)))),
+                metrics && (React.createElement(React.Fragment, null,
                     React.createElement("div", { className: "verify-metric-grid" },
                         React.createElement("div", { className: "verify-metric-card" },
-                            React.createElement("div", { className: "verify-metric-label" }, "Sample size"),
-                            React.createElement("div", { className: "verify-metric-value" }, metrics.sampleSize)),
+                            React.createElement("div", { className: "verify-metric-label" }, "Scenario runs"),
+                            React.createElement("div", { className: "verify-metric-value" },
+                                metrics.scenarioRunCountCompleted,
+                                "/",
+                                metrics.scenarioRunCountRequested)),
                         React.createElement("div", { className: "verify-metric-card" },
-                            React.createElement("div", { className: "verify-metric-label" }, "Agreement rate"),
-                            React.createElement("div", { className: "verify-metric-value" }, metrics.agreementRate === null
-                                ? "—"
-                                : `${Math.round(metrics.agreementRate * 100)}%`)),
+                            React.createElement("div", { className: "verify-metric-label" }, "Grade samples"),
+                            React.createElement("div", { className: "verify-metric-value" },
+                                metrics.gradeSampleCountCompleted,
+                                "/",
+                                metrics.gradeSampleCountRequested)),
                         React.createElement("div", { className: "verify-metric-card" },
-                            React.createElement("div", { className: "verify-metric-label" }, "Score spread (min/median/max)"),
-                            React.createElement("div", { className: "verify-metric-value verify-metric-value--compact" }, metrics.scoreSpreadMin === null
-                                ? "—"
-                                : `${metrics.scoreSpreadMin} / ${metrics.scoreSpreadMedian ?? "—"} / ${metrics.scoreSpreadMax ?? "—"}`)),
+                            React.createElement("div", { className: "verify-metric-label" }, "Pass rate"),
+                            React.createElement("div", { className: "verify-metric-value" }, formatPercent(metrics.passRate))),
                         React.createElement("div", { className: "verify-metric-card" },
-                            React.createElement("div", { className: "verify-metric-label" }, "Instability count"),
-                            React.createElement("div", { className: "verify-metric-value" }, metrics.instabilityCount))),
-                    React.createElement(Callout_js_1.default, { variant: metrics.verdict === "FAIL"
-                            ? "danger"
-                            : metrics.verdict === "WARN"
-                                ? "emphasis"
-                                : "muted", title: `Verdict: ${metrics.verdict}` }, metrics.verdictReason))),
-                React.createElement(Callout_js_1.default, { title: "Thresholds in code" },
-                    "Min sample size: ",
-                    verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.minSampleSize,
-                    " ",
-                    "\u00B7 PASS requires agreement \u2265 ",
-                    Math.round(verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.pass.agreementMin * 100),
-                    "%, spread \u2264",
-                    " ",
-                    verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.pass.maxSpread,
-                    ", instability \u2264",
-                    " ",
-                    verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.pass.maxInstabilityCount,
-                    " ",
-                    "\u00B7 WARN allows agreement \u2265 ",
-                    Math.round(verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.warn.agreementMin * 100),
-                    "%, spread \u2264",
-                    " ",
-                    verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.warn.maxSpread,
-                    ", instability \u2264",
-                    " ",
-                    verify_metrics_js_1.VERIFY_CONSISTENCY_THRESHOLDS.warn.maxInstabilityCount,
-                    "."),
+                            React.createElement("div", { className: "verify-metric-label" }, "Score mean"),
+                            React.createElement("div", { className: "verify-metric-value" }, metrics.scoreMean === null ? "-" : metrics.scoreMean)),
+                        React.createElement("div", { className: "verify-metric-card" },
+                            React.createElement("div", { className: "verify-metric-label" }, "Score min/median/max"),
+                            React.createElement("div", { className: "verify-metric-value verify-metric-value--compact" }, metrics.scoreMin === null
+                                ? "-"
+                                : `${metrics.scoreMin} / ${metrics.scoreMedian ?? "-"} / ${metrics.scoreMax ?? "-"}`)),
+                        React.createElement("div", { className: "verify-metric-card" },
+                            React.createElement("div", { className: "verify-metric-label" }, "Execution failures"),
+                            React.createElement("div", { className: "verify-metric-value" }, metrics.executionFailureCount)),
+                        React.createElement("div", { className: "verify-metric-card" },
+                            React.createElement("div", { className: "verify-metric-label" }, "Grading failures"),
+                            React.createElement("div", { className: "verify-metric-value" }, metrics.gradingFailureCount))))),
                 React.createElement("div", { className: "verify-section" },
-                    React.createElement("strong", null, "Most inconsistent examples"),
-                    topOutliers.length === 0
-                        ? (React.createElement(Callout_js_1.default, null, "Inconsistent examples will appear here as soon as at least one completed run is available in this batch."))
-                        : (React.createElement("div", { className: "verify-outlier-list" }, topOutliers.map((outlier) => {
-                            const runLinks = [outlier.maxRunId, outlier.minRunId]
+                    React.createElement("strong", null, "Outlier scenario runs"),
+                    topOutlierScenarioRuns.length === 0
+                        ? (React.createElement(Callout_js_1.default, null, "Outlier scenario runs appear as soon as completed grade samples are available."))
+                        : (React.createElement("div", { className: "verify-outlier-list" }, topOutlierScenarioRuns.map((outlier) => {
+                            const runLinks = [outlier.minRunId, outlier.maxRunId]
                                 .filter((value) => Boolean(value));
                             const uniqueRunLinks = [...new Set(runLinks)];
                             return (React.createElement("div", { key: outlier.key, className: "verify-outlier-card" },
                                 React.createElement("div", { className: "verify-outlier-header" },
-                                    React.createElement("strong", null, outlier.label),
-                                    React.createElement(Badge_js_1.default, { variant: outlier.instability
-                                            ? "error"
-                                            : "completed" }, outlier.instability ? "Unstable" : "Stable")),
+                                    React.createElement("strong", null, (0, utils_js_1.scenarioNameFromValue)(outlier.scenarioRunId) ??
+                                        outlier.scenarioRunId),
+                                    React.createElement(Badge_js_1.default, { variant: outlier.failed ? "error" : "completed" }, outlier.failed ? "Failed" : "Scored")),
                                 React.createElement("div", { className: "verify-outlier-meta" },
-                                    "agreement ",
-                                    outlier.agreementRate === null
-                                        ? "—"
-                                        : `${Math.round(outlier.agreementRate * 100)}%`,
+                                    "avg ",
+                                    outlier.averageScore ?? "-",
+                                    " \u00B7 min/max",
+                                    " ",
+                                    outlier.minScore ?? "-",
+                                    "/",
+                                    outlier.maxScore ?? "-",
+                                    " ",
+                                    "\u00B7 samples ",
+                                    outlier.completedSampleCount,
+                                    "/",
+                                    outlier.gradeSampleCount,
+                                    " \u00B7 execution failures",
                                     " ",
-                                    "\u00B7 delta ",
-                                    outlier.scoreDelta ?? "—",
-                                    " \u00B7 samples",
+                                    outlier.executionFailureCount,
+                                    " \u00B7 grading failures",
                                     " ",
-                                    outlier.sampleSize,
-                                    outlier.passFlip ? " · pass/fail flip" : "",
+                                    outlier.gradingFailureCount,
                                     outlier.messageRefId
                                         ? ` · ref ${outlier.messageRefId}`
                                         : ""),
@@ -500,6 +540,18 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                                     "Open grade run ",
                                     runId)))))));
                         })))),
+                React.createElement("div", { className: "verify-section" },
+                    React.createElement("strong", null, "Failure reasons"),
+                    !metrics || metrics.failureReasons.length === 0
+                        ? React.createElement(Callout_js_1.default, null, "No failure reasons captured yet.")
+                        : (React.createElement("ul", { className: "verify-request-list" }, metrics.failureReasons.map((reason) => (React.createElement("li", { key: reason.key, className: "verify-request-row" },
+                            React.createElement(Badge_js_1.default, { variant: reason.kind === "execution"
+                                    ? "error"
+                                    : "running" }, reason.kind),
+                            React.createElement("span", null, reason.reason),
+                            React.createElement("span", { className: "secondary-note" },
+                                "x",
+                                reason.count))))))),
                 visibleBatches.length > 0 && (React.createElement("div", { className: "verify-section" },
                     React.createElement("strong", null, "Batch history"),
                     React.createElement("ul", { className: "verify-request-list" }, visibleBatches.slice(0, 15).map((batch, index) => (React.createElement("li", { key: batch.id, className: "verify-request-row" },
@@ -509,10 +561,10 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                         React.createElement(Badge_js_1.default, { status: batch.status }, batch.status),
                         React.createElement("button", { type: "button", className: "link-button", onClick: () => setSelectedBatchId(batch.id) }, batch.id),
                         React.createElement("span", { className: "secondary-note" },
-                            batch.completed,
-                            "/",
-                            batch.requested,
-                            " complete"))))))),
+                            batch.scenarioRuns,
+                            " runs \u00D7",
+                            " ",
+                            batch.graderRepeatsPerScenario))))))),
                 selectedBatch?.requests.length
                     ? (React.createElement("div", { className: "verify-section" },
                         React.createElement("strong", null, "Batch requests"),
@@ -523,6 +575,8 @@ exports.SimulatorVerifyPage = (0, iso_js_1.iso)(`
                             React.createElement(Badge_js_1.default, { status: request.status === "queued"
                                     ? "idle"
                                     : request.status }, request.status),
+                            request.scenarioRunId && (React.createElement("span", { className: "secondary-note" }, (0, utils_js_1.scenarioNameFromValue)(request.scenarioRunId) ??
+                                request.scenarioRunId)),
                             request.runId
                                 ? (React.createElement("a", { href: toPrefixedPath((0, workspace_routes_js_1.buildWorkspacePath)("grade", workspaceId, {
                                         runId: request.runId,