@bolt-foundry/gambit 0.8.6-rc.1 → 0.8.6-rc.3

package/script/src/simulator_graphql.js

@@ -60,6 +60,7 @@ function logBuildChatDebug(event, payload) {
 const builder = new core_1.default({
     plugins: [plugin_relay_1.default],
     relay: {},
+    defaultFieldNullability: false,
 });
 const WorkspaceRelativePathScalar = builder.scalarType("WorkspaceRelativePath", {
     serialize: (value) => {
@@ -137,25 +138,35 @@ function isWorkspaceConversationSessionRecord(value) {
     return !!value && typeof value === "object" &&
         typeof value.kind === "string";
 }
-function materializeOpenResponseOutputItems(run) {
-    const materialized = run.messages.flatMap((message, index) => {
-        const content = typeof message.content === "string" ? message.content : "";
-        if (!content.trim())
-            return [];
-        return [{
-                __typename: "OutputMessage",
-                id: `${run.id}:item:${index}`,
-                role: message.role,
-                content,
-                messageRefId: message.messageRefId,
-            }];
-    });
-    logBuildChatDebug("materialize.outputItems", {
-        runId: run.id,
-        messageCount: run.messages.length,
-        outputCount: materialized.length,
-    });
-    return materialized;
+function asRecord(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : null;
+}
+function asString(value) {
+    return typeof value === "string" ? value : "";
+}
+function toEventMessageText(value) {
+    if (typeof value === "string")
+        return value;
+    if (Array.isArray(value)) {
+        return value.map((entry) => {
+            const record = asRecord(entry);
+            if (!record)
+                return "";
+            const text = asString(record.text);
+            if (text.length > 0)
+                return text;
+            return toEventMessageText(record.content);
+        }).join("");
+    }
+    const record = asRecord(value);
+    if (!record)
+        return "";
+    const recordText = asString(record.text);
+    if (recordText.length > 0)
+        return recordText;
+    return toEventMessageText(record.content);
 }
 const WorkspaceSessionMetaType = builder.objectRef("WorkspaceSessionMeta");
 WorkspaceSessionMetaType.implement({
@@ -433,6 +444,19 @@ OpenResponseEventType.implement({
         data: t.field({ type: "JSON", resolve: (parent) => parent.data }),
     }),
 });
+const WorkspaceOpenResponseEventsLiveEdgeType = builder.objectRef("WorkspaceOpenResponseEventsLiveEdge");
+WorkspaceOpenResponseEventsLiveEdgeType.implement({
+    fields: (t) => ({
+        workspaceId: t.id({ resolve: (parent) => (0, gambit_id_js_1.asGambitID)(parent.workspaceId) }),
+        runId: t.id({ resolve: (parent) => parent.runId }),
+        sourceSequence: t.int({ resolve: (parent) => parent.sourceSequence }),
+        occurredAt: t.string({ resolve: (parent) => parent.occurredAt }),
+        event: t.field({
+            type: OpenResponseEventType,
+            resolve: (parent) => parent.event,
+        }),
+    }),
+});
 const OpenResponseType = builder.objectRef("OpenResponse");
 OpenResponseType.implement({
     fields: (t) => ({
@@ -448,8 +472,18 @@ OpenResponseType.implement({
                 first: t.arg.int(),
                 after: t.arg.string(),
             },
-            resolve: (parent, args) => {
-                const items = materializeOpenResponseOutputItems(parent);
+            resolve: async (parent, args, ctx) => {
+                const workspaceId = typeof parent.workspaceId === "string" &&
+                    parent.workspaceId.trim().length > 0
+                    ? parent.workspaceId
+                    : null;
+                if (!workspaceId || !ctx.readWorkspaceOpenResponseOutputItems) {
+                    return (0, plugin_relay_1.resolveArrayConnection)({ args }, []);
+                }
+                const items = await ctx.readWorkspaceOpenResponseOutputItems({
+                    workspaceId,
+                    runId: parent.id,
+                });
                 logBuildChatDebug("resolver.openResponse.outputItems", {
                     runId: parent.id,
                     itemCount: items.length,
@@ -463,12 +497,29 @@ OpenResponseType.implement({
                 first: t.arg.int(),
                 after: t.arg.string(),
             },
-            resolve: (parent, args) => {
+            resolve: async (parent, args, ctx) => {
+                const workspaceId = typeof parent.workspaceId === "string" &&
+                    parent.workspaceId.trim().length > 0
+                    ? parent.workspaceId
+                    : null;
+                if (!workspaceId || !ctx.readWorkspaceOpenResponseEvents) {
+                    return (0, plugin_relay_1.resolveArrayConnection)({ args }, []);
+                }
+                const records = await ctx.readWorkspaceOpenResponseEvents({
+                    workspaceId,
+                    runId: parent.id,
+                    fromSequence: 0,
+                });
+                const events = records.map((event) => ({
+                    id: `${parent.id}:event:${event.sequence}`,
+                    type: event.event_type,
+                    data: event.payload,
+                }));
                 logBuildChatDebug("resolver.openResponse.events", {
                     runId: parent.id,
-                    eventCount: 0,
+                    eventCount: events.length,
                 });
-                return (0, plugin_relay_1.resolveArrayConnection)({ args }, []);
+                return (0, plugin_relay_1.resolveArrayConnection)({ args }, events);
             },
         }),
     }),
@@ -494,21 +545,6 @@ const BuildStateType = builder.objectRef("BuildState");
 BuildStateType.implement({
     fields: (t) => ({
         workspaceId: t.id({ resolve: (parent) => (0, gambit_id_js_1.asGambitID)(parent.workspaceId) }),
-        transcript: t.connection({
-            type: OpenResponseOutputItemType,
-            args: {
-                first: t.arg.int(),
-                after: t.arg.string(),
-            },
-            resolve: (parent, args) => {
-                const outputItems = materializeOpenResponseOutputItems(parent.run);
-                logBuildChatDebug("resolver.buildState.transcript", {
-                    runId: parent.run.id,
-                    itemCount: outputItems.length,
-                });
-                return (0, plugin_relay_1.resolveArrayConnection)({ args }, outputItems);
-            },
-        }),
         runStatus: t.field({
             type: OpenResponseStatusEnum,
             resolve: (parent) => toOpenResponseStatus(parent.run.status),
@@ -576,20 +612,6 @@ function toWorkspaceGradeRunStatus(value) {
         return "COMPLETED";
     return "ERROR";
 }
-const VERIFY_CONSISTENCY_THRESHOLDS = {
-    minSampleSize: 6,
-    instabilityScoreDelta: 1.5,
-    pass: {
-        agreementMin: 0.9,
-        maxSpread: 1,
-        maxInstabilityCount: 0,
-    },
-    warn: {
-        agreementMin: 0.75,
-        maxSpread: 2,
-        maxInstabilityCount: 2,
-    },
-};
 function round2(value) {
     return Math.round(value * 100) / 100;
 }
@@ -602,195 +624,179 @@ function median(values) {
         return sorted[mid];
     return (sorted[mid - 1] + sorted[mid]) / 2;
 }
-function flattenVerifyRunExamples(run) {
-    if (run.turns.length > 0) {
-        return run.turns.map((turn, fallbackIndex) => {
-            const index = typeof turn.turnIndex === "number"
-                ? turn.turnIndex
-                : fallbackIndex;
-            const key = turn.refId && turn.refId.trim().length > 0
-                ? `ref:${turn.refId}`
-                : `turn:${index}`;
-            const pass = typeof turn.score === "number" ? turn.score >= 0 : undefined;
-            return {
-                key,
-                label: `Assistant turn ${turn.turnNumber || fallbackIndex + 1}`,
-                points: [{
-                        runId: run.id,
-                        score: turn.score,
-                        pass,
-                        reason: turn.reason,
-                        turnIndex: index,
-                        messageRefId: turn.refId,
-                    }],
-            };
-        });
+function extractRepresentativeScore(run) {
+    const summaryScore = run.summary?.score;
+    if (typeof summaryScore === "number" && Number.isFinite(summaryScore)) {
+        return summaryScore;
     }
-    const score = run.summary?.score;
-    const reason = run.summary?.reason;
-    const pass = typeof score === "number" ? score >= 0 : undefined;
-    return [{
-            key: "conversation",
-            label: "Conversation score",
-            points: [{
-                    runId: run.id,
-                    score,
-                    pass,
-                    reason,
-                }],
-        }];
+    const turnScores = run.turns
+        .map((turn) => turn.score)
+        .filter((score) => typeof score === "number" && Number.isFinite(score));
+    if (turnScores.length === 0)
+        return null;
+    return median(turnScores);
 }
-function resolveVerifyVerdict(input) {
-    if (input.sampleSize < VERIFY_CONSISTENCY_THRESHOLDS.minSampleSize) {
-        return {
-            verdict: "WARN",
-            reason: `Need at least ${VERIFY_CONSISTENCY_THRESHOLDS.minSampleSize} samples before issuing a firm verdict.`,
-        };
-    }
-    if (input.agreementRate === null) {
-        return {
-            verdict: "WARN",
-            reason: "No comparable pass/fail evidence was found in the sampled runs.",
-        };
-    }
-    const spreadMax = input.spreadMax ?? 0;
-    if (input.agreementRate >= VERIFY_CONSISTENCY_THRESHOLDS.pass.agreementMin &&
-        spreadMax <= VERIFY_CONSISTENCY_THRESHOLDS.pass.maxSpread &&
-        input.instabilityCount <=
-            VERIFY_CONSISTENCY_THRESHOLDS.pass.maxInstabilityCount) {
-        return {
-            verdict: "PASS",
-            reason: "Agreement, spread, and instability all meet PASS thresholds.",
-        };
-    }
-    if (input.agreementRate >= VERIFY_CONSISTENCY_THRESHOLDS.warn.agreementMin &&
-        spreadMax <= VERIFY_CONSISTENCY_THRESHOLDS.warn.maxSpread &&
-        input.instabilityCount <=
-            VERIFY_CONSISTENCY_THRESHOLDS.warn.maxInstabilityCount) {
-        return {
-            verdict: "WARN",
-            reason: "Some variation was detected, but results remain within WARN thresholds.",
-        };
+function extractRepresentativeReason(run) {
+    if (typeof run.summary?.reason === "string" &&
+        run.summary.reason.trim().length > 0) {
+        return run.summary.reason.trim();
     }
-    return {
-        verdict: "FAIL",
-        reason: "Agreement/spread instability exceeds WARN thresholds.",
-    };
+    const turnReason = run.turns.find((turn) => typeof turn.reason === "string" && turn.reason.trim().length > 0)?.reason;
+    return typeof turnReason === "string" ? turnReason.trim() : null;
 }
-function buildVerifyMetricsFromRuns(runs) {
-    const completedRuns = runs.filter((run) => run.status === "completed");
-    const sampleSize = completedRuns.length;
-    const bucketsByKey = new Map();
-    completedRuns.forEach((run) => {
-        flattenVerifyRunExamples(run).forEach((entry) => {
-            const existing = bucketsByKey.get(entry.key);
-            if (!existing) {
-                bucketsByKey.set(entry.key, {
-                    key: entry.key,
-                    label: entry.label,
-                    points: [...entry.points],
-                });
-                return;
-            }
-            existing.points.push(...entry.points);
-        });
-    });
-    const outliers = [];
-    let agreementVotes = 0;
-    let agreementTotal = 0;
-    const scoreDeltas = [];
-    bucketsByKey.forEach((bucket) => {
-        const scores = bucket.points
-            .map((point) => point.score)
-            .filter((score) => typeof score === "number" && Number.isFinite(score));
-        const minScore = scores.length > 0 ? Math.min(...scores) : null;
-        const maxScore = scores.length > 0 ? Math.max(...scores) : null;
-        const scoreDelta = minScore !== null && maxScore !== null
-            ? round2(maxScore - minScore)
-            : null;
-        const passVotes = bucket.points
-            .map((point) => point.pass)
-            .filter((pass) => typeof pass === "boolean");
-        const passCount = passVotes.filter((value) => value).length;
-        const failCount = passVotes.length - passCount;
-        const agreementRate = passVotes.length > 0
-            ? round2(Math.max(passCount, failCount) / passVotes.length)
-            : null;
-        if (passVotes.length > 0) {
-            agreementVotes += Math.max(passCount, failCount);
-            agreementTotal += passVotes.length;
+function buildVerifyMetrics(args) {
+    const runById = new Map(args.runs.map((run) => [run.id, run]));
+    const reasonCounter = new Map();
+    const scoreValues = [];
+    let passCount = 0;
+    let scoreCount = 0;
+    let gradingFailureCount = 0;
+    const scenarioGroups = new Map();
+    const addReason = (kind, reason) => {
+        const normalized = typeof reason === "string" ? reason.trim() : "";
+        if (!normalized)
+            return;
+        const key = `${kind}:${normalized}`;
+        const existing = reasonCounter.get(key);
+        if (existing) {
+            existing.count += 1;
+            return;
         }
-        if (scoreDelta !== null) {
-            scoreDeltas.push(scoreDelta);
+        reasonCounter.set(key, { key, kind, reason: normalized, count: 1 });
+    };
+    for (const request of args.batch.requests) {
+        if (!request.scenarioRunId)
+            continue;
+        const existing = scenarioGroups.get(request.scenarioRunId);
+        if (existing) {
+            existing.requests.push(request);
+            continue;
         }
-        const passFlip = passCount > 0 && failCount > 0;
-        const instability = passFlip ||
-            (scoreDelta !== null &&
-                scoreDelta > VERIFY_CONSISTENCY_THRESHOLDS.instabilityScoreDelta);
-        const minPoint = minScore === null
-            ? undefined
-            : bucket.points.find((point) => point.score === minScore);
-        const maxPoint = maxScore === null
-            ? undefined
-            : bucket.points.find((point) => point.score === maxScore);
-        outliers.push({
-            key: bucket.key,
-            label: bucket.label,
-            sampleSize: bucket.points.length,
-            agreementRate,
-            scoreDelta,
-            passFlip,
-            instability,
-            minRunId: minPoint?.runId,
-            maxRunId: maxPoint?.runId,
-            turnIndex: maxPoint?.turnIndex ?? minPoint?.turnIndex,
-            messageRefId: maxPoint?.messageRefId ?? minPoint?.messageRefId,
+        scenarioGroups.set(request.scenarioRunId, {
+            scenarioRunId: request.scenarioRunId,
+            requests: [request],
+            scores: [],
+            executionFailureCount: 0,
+            gradingFailureCount: 0,
+            completedSampleCount: 0,
         });
-    });
-    outliers.sort((left, right) => {
-        if (left.instability !== right.instability) {
-            return left.instability ? -1 : 1;
+    }
+    for (const request of args.batch.requests) {
+        if (request.status === "error") {
+            addReason("execution", request.error);
+            if (request.scenarioRunId) {
+                const group = scenarioGroups.get(request.scenarioRunId);
+                if (group)
+                    group.executionFailureCount += 1;
+            }
+            continue;
+        }
+        if (request.status !== "completed" || !request.runId)
+            continue;
+        const run = runById.get(request.runId);
+        if (!run || run.status !== "completed")
+            continue;
+        const score = extractRepresentativeScore(run);
+        if (request.scenarioRunId) {
+            const group = scenarioGroups.get(request.scenarioRunId);
+            if (group) {
+                group.completedSampleCount += 1;
+            }
         }
-        if (left.passFlip !== right.passFlip)
-            return left.passFlip ? -1 : 1;
-        const leftDelta = left.scoreDelta ?? -1;
-        const rightDelta = right.scoreDelta ?? -1;
-        if (leftDelta !== rightDelta)
-            return rightDelta - leftDelta;
-        if (left.sampleSize !== right.sampleSize) {
-            return right.sampleSize - left.sampleSize;
+        if (score === null)
+            continue;
+        const rounded = round2(score);
+        scoreValues.push(rounded);
+        scoreCount += 1;
+        if (rounded >= 0) {
+            passCount += 1;
         }
-        return left.label.localeCompare(right.label);
+        else {
+            gradingFailureCount += 1;
+            addReason("grading", extractRepresentativeReason(run));
+            if (request.scenarioRunId) {
+                const group = scenarioGroups.get(request.scenarioRunId);
+                if (group)
+                    group.gradingFailureCount += 1;
+            }
+        }
+        if (request.scenarioRunId) {
+            const group = scenarioGroups.get(request.scenarioRunId);
+            if (group) {
+                const messageRefId = run.turns.find((turn) => typeof turn.refId === "string" && turn.refId.trim().length > 0)?.refId;
+                group.scores.push({
+                    runId: run.id,
+                    value: rounded,
+                    messageRefId,
+                });
+            }
+        }
+    }
+    const outlierScenarioRuns = [
+        ...scenarioGroups.values(),
+    ].map((group) => {
+        const sortedScores = [...group.scores].sort((left, right) => left.value - right.value);
+        const numericScores = sortedScores.map((entry) => entry.value);
+        const min = numericScores.length > 0 ? numericScores[0] : null;
+        const max = numericScores.length > 0
+            ? numericScores[numericScores.length - 1]
+            : null;
+        const avg = numericScores.length > 0
+            ? round2(numericScores.reduce((sum, value) => sum + value, 0) /
+                numericScores.length)
+            : null;
+        return {
+            key: group.scenarioRunId,
+            scenarioRunId: group.scenarioRunId,
+            gradeSampleCount: group.requests.length,
+            completedSampleCount: group.completedSampleCount,
+            executionFailureCount: group.executionFailureCount,
+            gradingFailureCount: group.gradingFailureCount,
+            averageScore: avg,
+            minScore: min === null ? null : round2(min),
+            maxScore: max === null ? null : round2(max),
+            failed: group.executionFailureCount > 0,
+            minRunId: sortedScores[0]?.runId,
+            maxRunId: sortedScores[sortedScores.length - 1]?.runId,
+            messageRefId: sortedScores[0]?.messageRefId ??
+                sortedScores[sortedScores.length - 1]?.messageRefId,
+        };
+    }).sort((left, right) => {
+        if (left.failed !== right.failed)
+            return left.failed ? -1 : 1;
+        const leftScore = left.averageScore ?? Number.POSITIVE_INFINITY;
+        const rightScore = right.averageScore ?? Number.POSITIVE_INFINITY;
+        if (leftScore !== rightScore)
+            return leftScore - rightScore;
+        return left.scenarioRunId.localeCompare(right.scenarioRunId);
     });
-    const agreementRate = agreementTotal > 0
-        ? round2(agreementVotes / agreementTotal)
-        : null;
-    const scoreSpreadMin = scoreDeltas.length > 0
-        ? Math.min(...scoreDeltas)
+    const scoreMin = scoreValues.length > 0 ? Math.min(...scoreValues) : null;
+    const scoreMax = scoreValues.length > 0 ? Math.max(...scoreValues) : null;
+    const scoreMedian = median(scoreValues);
+    const scoreMean = scoreValues.length > 0
+        ? round2(scoreValues.reduce((sum, value) => sum + value, 0) / scoreValues.length)
         : null;
-    const scoreSpreadMax = scoreDeltas.length > 0
-        ? Math.max(...scoreDeltas)
-        : null;
-    const scoreSpreadMedian = median(scoreDeltas);
-    const instabilityCount = outliers.filter((entry) => entry.instability).length;
-    const verdict = resolveVerifyVerdict({
-        sampleSize,
-        agreementRate,
-        spreadMax: scoreSpreadMax,
-        instabilityCount,
-    });
+    const passRate = scoreCount > 0 ? round2(passCount / scoreCount) : null;
     return {
-        sampleSize,
-        agreementRate,
-        scoreSpreadMin: scoreSpreadMin === null ? null : round2(scoreSpreadMin),
-        scoreSpreadMedian: scoreSpreadMedian === null
-            ? null
-            : round2(scoreSpreadMedian),
-        scoreSpreadMax: scoreSpreadMax === null ? null : round2(scoreSpreadMax),
-        instabilityCount,
-        verdict: verdict.verdict,
-        verdictReason: verdict.reason,
-        outliers,
+        scenarioRunCountRequested: args.batch.scenarioRuns,
+        scenarioRunCountCompleted: args.batch.scenarioRunsCompleted,
+        scenarioRunCountFailed: args.batch.scenarioRunsFailed,
+        gradeSampleCountRequested: args.batch.requested,
+        gradeSampleCountCompleted: args.batch.completed,
+        gradeSampleCountFailed: args.batch.failed,
+        executionFailureCount: args.batch.failed,
+        gradingFailureCount,
+        passRate,
+        scoreMin: scoreMin === null ? null : round2(scoreMin),
+        scoreMedian: scoreMedian === null ? null : round2(scoreMedian),
+        scoreMax: scoreMax === null ? null : round2(scoreMax),
+        scoreMean,
+        outlierScenarioRuns,
+        failureReasons: [...reasonCounter.values()].sort((left, right) => {
+            if (left.count !== right.count)
+                return right.count - left.count;
+            return left.reason.localeCompare(right.reason);
+        }),
     };
 }
 function toWorkspaceVerifyBatchStatus(value) {
@@ -817,8 +823,8 @@ const WorkspaceVerifyBatchStatusEnum = builder.enumType("WorkspaceVerifyBatchSta
 const WorkspaceVerifyBatchRequestStatusEnum = builder.enumType("WorkspaceVerifyBatchRequestStatus", {
     values: ["QUEUED", "RUNNING", "COMPLETED", "ERROR"],
 });
-const WorkspaceVerifyVerdictEnum = builder.enumType("WorkspaceVerifyVerdict", {
-    values: ["PASS", "WARN", "FAIL"],
+const WorkspaceVerifyFailureReasonKindEnum = builder.enumType("WorkspaceVerifyFailureReasonKind", {
+    values: ["EXECUTION", "GRADING"],
 });
 const WorkspaceGradeTurnType = builder.objectRef("WorkspaceGradeTurn");
 WorkspaceGradeTurnType.implement({
@@ -969,28 +975,44 @@ WorkspaceVerifyBatchRequestType.implement({
             nullable: true,
             resolve: (parent) => parent.runId ?? null,
         }),
+        scenarioRunId: t.id({
+            nullable: true,
+            resolve: (parent) => parent.scenarioRunId ?? null,
+        }),
         error: t.string({
             nullable: true,
             resolve: (parent) => parent.error ?? null,
         }),
     }),
 });
-const WorkspaceVerifyOutlierType = builder.objectRef("WorkspaceVerifyOutlier");
-WorkspaceVerifyOutlierType.implement({
+const WorkspaceVerifyScenarioOutlierType = builder.objectRef("WorkspaceVerifyScenarioOutlier");
+WorkspaceVerifyScenarioOutlierType.implement({
     fields: (t) => ({
         key: t.id({ resolve: (parent) => parent.key }),
-        label: t.string({ resolve: (parent) => parent.label }),
-        sampleSize: t.int({ resolve: (parent) => parent.sampleSize }),
-        agreementRate: t.float({
+        scenarioRunId: t.id({ resolve: (parent) => parent.scenarioRunId }),
+        gradeSampleCount: t.int({ resolve: (parent) => parent.gradeSampleCount }),
+        completedSampleCount: t.int({
+            resolve: (parent) => parent.completedSampleCount,
+        }),
+        executionFailureCount: t.int({
+            resolve: (parent) => parent.executionFailureCount,
+        }),
+        gradingFailureCount: t.int({
+            resolve: (parent) => parent.gradingFailureCount,
+        }),
+        averageScore: t.float({
+            nullable: true,
+            resolve: (parent) => parent.averageScore,
+        }),
+        minScore: t.float({
             nullable: true,
-            resolve: (parent) => parent.agreementRate,
+            resolve: (parent) => parent.minScore,
         }),
-        scoreDelta: t.float({
+        maxScore: t.float({
             nullable: true,
-            resolve: (parent) => parent.scoreDelta,
+            resolve: (parent) => parent.maxScore,
         }),
-        passFlip: t.boolean({ resolve: (parent) => parent.passFlip }),
-        instability: t.boolean({ resolve: (parent) => parent.instability }),
+        failed: t.boolean({ resolve: (parent) => parent.failed }),
         minRunId: t.id({
             nullable: true,
             resolve: (parent) => parent.minRunId ?? null,
@@ -999,53 +1021,86 @@ WorkspaceVerifyOutlierType.implement({
             nullable: true,
             resolve: (parent) => parent.maxRunId ?? null,
         }),
-        turnIndex: t.int({
-            nullable: true,
-            resolve: (parent) => parent.turnIndex ?? null,
-        }),
         messageRefId: t.id({
             nullable: true,
             resolve: (parent) => parent.messageRefId ?? null,
         }),
     }),
 });
+const WorkspaceVerifyFailureReasonGroupType = builder.objectRef("WorkspaceVerifyFailureReasonGroup");
+WorkspaceVerifyFailureReasonGroupType.implement({
+    fields: (t) => ({
+        key: t.id({ resolve: (parent) => parent.key }),
+        kind: t.field({
+            type: WorkspaceVerifyFailureReasonKindEnum,
+            resolve: (parent) => parent.kind === "execution" ? "EXECUTION" : "GRADING",
+        }),
+        reason: t.string({ resolve: (parent) => parent.reason }),
+        count: t.int({ resolve: (parent) => parent.count }),
+    }),
+});
 const WorkspaceVerifyMetricsType = builder.objectRef("WorkspaceVerifyMetrics");
 WorkspaceVerifyMetricsType.implement({
     fields: (t) => ({
-        sampleSize: t.int({ resolve: (parent) => parent.sampleSize }),
-        agreementRate: t.float({
-            nullable: true,
-            resolve: (parent) => parent.agreementRate,
+        scenarioRunCountRequested: t.int({
+            resolve: (parent) => parent.scenarioRunCountRequested,
+        }),
+        scenarioRunCountCompleted: t.int({
+            resolve: (parent) => parent.scenarioRunCountCompleted,
+        }),
+        scenarioRunCountFailed: t.int({
+            resolve: (parent) => parent.scenarioRunCountFailed,
+        }),
+        gradeSampleCountRequested: t.int({
+            resolve: (parent) => parent.gradeSampleCountRequested,
+        }),
+        gradeSampleCountCompleted: t.int({
+            resolve: (parent) => parent.gradeSampleCountCompleted,
         }),
-        scoreSpreadMin: t.float({
+        gradeSampleCountFailed: t.int({
+            resolve: (parent) => parent.gradeSampleCountFailed,
+        }),
+        executionFailureCount: t.int({
+            resolve: (parent) => parent.executionFailureCount,
+        }),
+        gradingFailureCount: t.int({
+            resolve: (parent) => parent.gradingFailureCount,
+        }),
+        passRate: t.float({
             nullable: true,
-            resolve: (parent) => parent.scoreSpreadMin,
+            resolve: (parent) => parent.passRate,
         }),
-        scoreSpreadMedian: t.float({
+        scoreMin: t.float({
             nullable: true,
-            resolve: (parent) => parent.scoreSpreadMedian,
+            resolve: (parent) => parent.scoreMin,
         }),
-        scoreSpreadMax: t.float({
+        scoreMedian: t.float({
             nullable: true,
-            resolve: (parent) => parent.scoreSpreadMax,
+            resolve: (parent) => parent.scoreMedian,
         }),
-        instabilityCount: t.int({
-            resolve: (parent) => parent.instabilityCount,
+        scoreMax: t.float({
+            nullable: true,
+            resolve: (parent) => parent.scoreMax,
         }),
-        verdict: t.field({
-            type: WorkspaceVerifyVerdictEnum,
-            resolve: (parent) => parent.verdict,
+        scoreMean: t.float({
+            nullable: true,
+            resolve: (parent) => parent.scoreMean,
         }),
-        verdictReason: t.string({
-            resolve: (parent) => parent.verdictReason,
+        outlierScenarioRuns: t.connection({
+            type: WorkspaceVerifyScenarioOutlierType,
+            args: {
+                first: t.arg.int(),
+                after: t.arg.string(),
+            },
+            resolve: (parent, args) => (0, plugin_relay_1.resolveArrayConnection)({ args }, parent.outlierScenarioRuns),
         }),
-        outliers: t.connection({
-            type: WorkspaceVerifyOutlierType,
+        failureReasons: t.connection({
+            type: WorkspaceVerifyFailureReasonGroupType,
             args: {
                 first: t.arg.int(),
                 after: t.arg.string(),
             },
-            resolve: (parent, args) => (0, plugin_relay_1.resolveArrayConnection)({ args }, parent.outliers),
+            resolve: (parent, args) => (0, plugin_relay_1.resolveArrayConnection)({ args }, parent.failureReasons),
         }),
     }),
 });
@@ -1054,10 +1109,22 @@ WorkspaceVerifyBatchType.implement({
     fields: (t) => ({
         id: t.id({ resolve: (parent) => parent.id }),
         workspaceId: t.id({ resolve: (parent) => parent.workspaceId }),
-        graderId: t.id({ resolve: (parent) => parent.graderId }),
-        scenarioRunId: t.id({
+        scenarioDeckId: t.id({
             nullable: true,
-            resolve: (parent) => parent.scenarioRunId ?? null,
+            resolve: (parent) => parent.scenarioDeckId ?? null,
+        }),
+        graderId: t.id({ resolve: (parent) => parent.graderId }),
+        scenarioRuns: t.int({
+            resolve: (parent) => parent.scenarioRuns,
+        }),
+        graderRepeatsPerScenario: t.int({
+            resolve: (parent) => parent.graderRepeatsPerScenario,
+        }),
+        scenarioRunsCompleted: t.int({
+            resolve: (parent) => parent.scenarioRunsCompleted,
+        }),
+        scenarioRunsFailed: t.int({
+            resolve: (parent) => parent.scenarioRunsFailed,
         }),
         status: t.field({
             type: WorkspaceVerifyBatchStatusEnum,
@@ -1089,19 +1156,17 @@ WorkspaceVerifyBatchType.implement({
             resolve: async (parent, _args, context) => {
                 if (!context.readWorkspaceGradeRuns)
                     return null;
-                const completedRunIds = parent.requests
-                    .filter((request) => request.status === "completed" &&
-                    typeof request.runId === "string" &&
+                const runIds = parent.requests
+                    .filter((request) => typeof request.runId === "string" &&
                     request.runId.trim().length > 0)
                     .map((request) => request.runId);
-                if (completedRunIds.length === 0)
-                    return null;
-                const runIdSet = new Set(completedRunIds);
+                const runIdSet = new Set(runIds);
                 const allRuns = await context.readWorkspaceGradeRuns(parent.workspaceId);
                 const selectedRuns = allRuns.filter((run) => runIdSet.has(run.id));
-                if (selectedRuns.length === 0)
-                    return null;
-                return buildVerifyMetricsFromRuns(selectedRuns);
+                return buildVerifyMetrics({
+                    batch: parent,
+                    runs: selectedRuns,
+                });
             },
         }),
     }),
@@ -1140,25 +1205,6 @@ WorkspaceConversationSessionInterface.implement({
             nullable: true,
             resolve: (parent) => parent.finishedAt ?? null,
         }),
-        transcript: t.connection({
-            type: OpenResponseOutputItemType,
-            args: {
-                first: t.arg.int(),
-                after: t.arg.string(),
-            },
-            resolve: (parent, args) => {
-                const run = parent.buildRun ?? parent.scenarioRun;
-                if (!run)
-                    return (0, plugin_relay_1.resolveArrayConnection)({ args }, []);
-                const normalizedRun = {
-                    ...run,
-                    conversationRunKind: parent.kind === "scenario"
-                        ? "scenario"
-                        : "build",
-                };
-                return (0, plugin_relay_1.resolveArrayConnection)({ args }, materializeOpenResponseOutputItems(normalizedRun));
-            },
-        }),
     }),
 });
 const WorkspaceBuildConversationSessionType = builder.objectRef("WorkspaceBuildConversationSession");
@@ -1946,7 +1992,8 @@ const WorkspaceConversationSessionStartInput = builder.inputType("WorkspaceConve
         assistantInit: t.field({ type: "JSON" }),
         graderId: t.id(),
         scenarioRunId: t.id(),
-        batchSize: t.int(),
+        scenarioRuns: t.int(),
+        graderRepeatsPerScenario: t.int(),
         concurrency: t.int(),
     }),
 });
@@ -1978,9 +2025,10 @@ const WorkspaceGradeRunCreateInput = builder.inputType("WorkspaceGradeRunCreateI
 const WorkspaceVerifyBatchRunCreateInput = builder.inputType("WorkspaceVerifyBatchRunCreateInput", {
     fields: (t) => ({
         workspaceId: t.id({ required: true }),
+        scenarioDeckId: t.id(),
         graderId: t.id({ required: true }),
-        scenarioRunId: t.id(),
-        batchSize: t.int({ required: true }),
+        scenarioRuns: t.int({ required: true }),
+        graderRepeatsPerScenario: t.int({ required: true }),
         concurrency: t.int({ required: true }),
     }),
 });
@@ -2188,7 +2236,8 @@ builder.mutationType({
                     assistantInit: args.input.assistantInit,
                     graderId: args.input.graderId ?? null,
                     scenarioRunId: args.input.scenarioRunId ?? null,
-                    batchSize: args.input.batchSize ?? null,
+                    scenarioRuns: args.input.scenarioRuns ?? null,
+                    graderRepeatsPerScenario: args.input.graderRepeatsPerScenario ?? null,
                     concurrency: args.input.concurrency ?? null,
                 });
                 return {
@@ -2308,9 +2357,10 @@ builder.mutationType({
                 }
                 const batch = await context.createWorkspaceVerifyBatchRun({
                     workspaceId: args.input.workspaceId,
+                    scenarioDeckId: args.input.scenarioDeckId ?? null,
                     graderId: args.input.graderId,
-                    scenarioRunId: args.input.scenarioRunId ?? null,
-                    batchSize: args.input.batchSize,
+                    scenarioRuns: args.input.scenarioRuns,
+                    graderRepeatsPerScenario: args.input.graderRepeatsPerScenario,
                     concurrency: args.input.concurrency,
                 });
                 return {
@@ -2408,6 +2458,72 @@ builder.mutationType({
 });
 builder.subscriptionType({
     fields: (t) => ({
+        workspaceOpenResponseEventsLive: t.field({
+            type: WorkspaceOpenResponseEventsLiveEdgeType,
+            args: {
+                workspaceId: t.arg.id({ required: true }),
+                runId: t.arg.id({ required: true }),
+                fromSequence: t.arg.int(),
+            },
+            subscribe: async function* (_parent, args, context) {
+                const workspaceId = normalizeWorkspaceId(args.workspaceId);
+                const runId = typeof args.runId === "string" ? args.runId.trim() : "";
+                if (!workspaceId || !runId)
+                    return;
+                const fromSequence = typeof args.fromSequence === "number" &&
+                    Number.isFinite(args.fromSequence)
+                    ? Math.max(0, Math.floor(args.fromSequence))
+                    : 0;
+                if (context.subscribeWorkspaceOpenResponseEvents) {
+                    const abortController = new AbortController();
+                    try {
+                        for await (const event of context.subscribeWorkspaceOpenResponseEvents({
+                            workspaceId,
+                            runId,
+                            fromSequence,
+                            signal: abortController.signal,
+                        })) {
+                            yield {
+                                workspaceId,
+                                runId,
+                                sourceSequence: event.sequence,
+                                occurredAt: event.created_at,
+                                event: {
+                                    id: `${runId}:event:${event.sequence}`,
+                                    type: event.event_type,
+                                    data: event.payload,
+                                },
+                            };
+                        }
+                    }
+                    finally {
+                        abortController.abort();
+                    }
+                    return;
+                }
+                if (!context.readWorkspaceOpenResponseEvents)
+                    return;
+                const replay = await context.readWorkspaceOpenResponseEvents({
+                    workspaceId,
+                    runId,
+                    fromSequence,
+                });
+                for (const event of replay) {
+                    yield {
+                        workspaceId,
+                        runId,
+                        sourceSequence: event.sequence,
+                        occurredAt: event.created_at,
+                        event: {
+                            id: `${runId}:event:${event.sequence}`,
+                            type: event.event_type,
+                            data: event.payload,
+                        },
+                    };
+                }
+            },
+            resolve: (parent) => parent,
+        }),
         workspaceUpdates: t.field({
             type: WorkspaceUpdateType,
             args: {