@bolt-foundry/gambit 0.8.6-rc.1 → 0.8.6-rc.3

package/esm/src/server.js

@@ -15,8 +15,7 @@ import { createSessionStore } from "./server_session_store.js";
 import { handleUiRoutes } from "./server_ui_routes.js";
 import { resolveWorkspaceIdFromRecord, resolveWorkspaceIdFromSearchParams, WORKSPACE_ROUTE_BASE, WORKSPACE_STATE_SCHEMA_VERSION, workspaceSchemaError, } from "./workspace_routes.js";
 import { appendDurableStreamEvent, GRAPHQL_STREAMS_PREFIX, handleDurableStreamRequest, } from "./durable_streams.js";
-import { handleCheckCommand } from "./commands/check.js";
-import { readCodexLoginStatus } from "./codex_preflight.js";
+import { createReadCodexWorkspaceStatus, handleBuildProviderStatusRequest, normalizeBuildChatProvider, persistBuildChatProviderMeta, } from "./server_build_chat_provider.js";
 import { handleGraphqlStreamMultiplexRequest } from "./graphql_stream_multiplex.js";
 import { handleGraphqlSubscriptionStreamRequest } from "./graphql_subscription_stream.js";
 import { asGambitID } from "./gambit_id.js";
@@ -108,8 +107,12 @@ const _GRADE_STREAM_ID = "gambit-grade";
 const TEST_STREAM_ID = "gambit-test";
 const WORKSPACE_API_BASE = "/api/workspace";
 const WORKSPACES_API_BASE = "/api/workspaces";
-const VERIFY_BATCH_SIZE_MAX = 24;
+const VERIFY_SCENARIO_RUNS_MAX = 24;
+const VERIFY_GRADER_REPEATS_MAX = 24;
 const VERIFY_BATCH_CONCURRENCY_MAX = 6;
+const DEFAULT_VERIFY_SCENARIO_RUNS = 10;
+const DEFAULT_VERIFY_GRADER_REPEATS = 10;
+const DEFAULT_VERIFY_CONCURRENCY = 4;
 const WORKSPACE_REFRESH_DEBUG = (() => {
     const value = (dntShim.Deno.env.get("GAMBIT_WORKSPACE_REFRESH_DEBUG") ?? "")
         .toLowerCase()
@@ -151,6 +154,13 @@ const _safeJsonStringify = (value) => {
         return candidate;
     });
 };
+const sanitizeJsonRecord = (value) => {
+    const parsed = JSON.parse(_safeJsonStringify(value));
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        return {};
+    }
+    return parsed;
+};
 const GAMBIT_BOT_SOURCE_DECK_URL = new URL("./decks/gambit-bot/PROMPT.md", globalThis[Symbol.for("import-meta-ponyfill-esmodule")](import.meta).url);
 const GAMBIT_BOT_SOURCE_DECK_PATH = GAMBIT_BOT_SOURCE_DECK_URL.protocol === "file:"
     ? path.fromFileUrl(GAMBIT_BOT_SOURCE_DECK_URL)
@@ -1362,30 +1372,13 @@ export function startWebSocketSimulator(opts) {
         buildBotRootCache.set(cacheKey, root);
         return root;
     };
-    const readCodexWorkspaceStatus = async (workspaceId, online) => {
-        const trustedPath = await resolveBuildBotRoot(workspaceId);
-        const record = resolveWorkspaceRecord(workspaceId);
-        const deckPath = record?.rootDeckPath ?? resolvedDeckPath;
-        const login = await readCodexLoginStatus();
-        let check;
-        try {
-            check = await handleCheckCommand({
-                deckPath,
-                checkOnline: Boolean(online),
-                openRouterApiKey: dntShim.Deno.env.get("OPENROUTER_API_KEY")?.trim() ||
-                    undefined,
-                googleApiKey: (dntShim.Deno.env.get("GOOGLE_API_KEY") ??
-                    dntShim.Deno.env.get("GEMINI_API_KEY"))?.trim() || undefined,
-                ollamaBaseURL: dntShim.Deno.env.get("OLLAMA_BASE_URL") ?? undefined,
-                json: true,
-            });
-        }
-        catch {
-            // Keep status endpoint resilient even if check cannot run.
-            check = undefined;
-        }
-        return { trustedPath, writeEnabled: true, ...login, check };
-    };
+    const readCodexWorkspaceStatus = createReadCodexWorkspaceStatus({
+        resolveBuildBotRoot,
+        resolveWorkspaceDeckPath: (workspaceId) => {
+            const record = resolveWorkspaceRecord(workspaceId);
+            return record?.rootDeckPath ?? resolvedDeckPath;
+        },
+    });
     const logWorkspaceBotRoot = async (endpoint, workspaceId) => {
         try {
             const root = await resolveBuildBotRoot(workspaceId);
@@ -1489,7 +1482,7 @@ export function startWebSocketSimulator(opts) {
             : undefined;
         return { state: { ...state, meta }, dir };
     };
-    const { selectCanonicalScenarioRunSummary, appendWorkspaceEnvelope, appendSessionEvent, appendGradingLog, persistSessionState, readSessionStateStrict, readSessionState, readBuildState, } = createSessionStore({
+    const { selectCanonicalScenarioRunSummary, appendWorkspaceEnvelope, appendOpenResponsesRunEvent, appendSessionEvent, appendGradingLog, persistSessionState, readSessionStateStrict, readSessionState, readBuildState, listOpenResponsesRunEvents, listOpenResponsesOutputItems, subscribeOpenResponsesRunEvents, } = createSessionStore({
         sessionsRoot,
         randomId,
         logger,
@@ -1520,6 +1513,83 @@ export function startWebSocketSimulator(opts) {
                 return "trace";
         }
     };
+    const hashStringFNV1a = (value) => {
+        let hash = 0x811c9dc5;
+        for (let i = 0; i < value.length; i += 1) {
+            hash ^= value.charCodeAt(i);
+            hash = Math.imul(hash, 0x01000193) >>> 0;
+        }
+        return hash.toString(16).padStart(8, "0");
+    };
+    const tracePayloadFingerprint = (payload) => hashStringFNV1a(JSON.stringify(asJsonValue(payload)));
+    const persistOpenResponsesTraceEvent = (state, trace, fallbackRunId) => {
+        if (!state)
+            return;
+        // TraceEvent is a broad union; we persist by reading dynamic keys at runtime.
+        const rawPayload = trace;
+        const eventType = typeof rawPayload.type === "string"
+            ? rawPayload.type
+            : "";
+        if (!eventType.startsWith("response."))
+            return;
+        const runId = typeof rawPayload.runId === "string" &&
+            rawPayload.runId.length > 0
+            ? rawPayload.runId
+            : fallbackRunId;
+        if (!runId)
+            return;
+        const eventTs = typeof rawPayload.ts === "number" &&
+            Number.isFinite(rawPayload.ts)
+            ? rawPayload.ts
+            : Date.now();
+        const eventScope = typeof rawPayload.item_id === "string"
+            ? rawPayload.item_id
+            : typeof rawPayload.actionCallId === "string"
+                ? rawPayload.actionCallId
+                : typeof rawPayload.output_index === "number"
+                    ? String(rawPayload.output_index)
+                    : typeof rawPayload.sequence_number === "number"
+                        ? String(rawPayload.sequence_number)
+                        : "event";
+        const payload = sanitizeJsonRecord(rawPayload);
+        const payloadFingerprint = tracePayloadFingerprint(payload);
+        void appendOpenResponsesRunEvent(state, {
+            workspace_id: "",
+            run_id: runId,
+            event_type: eventType,
+            payload: payload,
+            idempotency_key: `${runId}:${eventType}:${eventTs}:${eventScope}:${payloadFingerprint}`,
+            created_at: new Date(eventTs).toISOString(),
+        });
+    };
+    const persistCanonicalUserInputEvent = (args) => {
+        if (!args.state)
+            return;
+        const content = args.message.trim();
+        if (!content)
+            return;
+        const eventTs = Date.now();
+        void appendOpenResponsesRunEvent(args.state, {
+            workspace_id: "",
+            run_id: args.runId,
+            event_type: "input.item",
+            payload: {
+                type: "input.item",
+                role: "user",
+                content: [{ type: "input_text", text: content }],
+                source: args.source,
+            },
+            idempotency_key: `${args.runId}:input.item:${args.source}:${eventTs}:${content.slice(0, 64)}`,
+            created_at: new Date(eventTs).toISOString(),
+        });
+    };
+    const persistOpenResponsesTracesFromState = (state, fallbackRunId) => {
+        if (!state || !Array.isArray(state.traces))
+            return;
+        for (const trace of state.traces) {
+            persistOpenResponsesTraceEvent(state, trace, fallbackRunId);
+        }
+    };
     const buildWorkspaceMeta = (record, base) => {
         const createdAt = typeof base?.sessionCreatedAt ===
             "string"
@@ -2280,6 +2350,10 @@ export function startWebSocketSimulator(opts) {
                         requestRecord.runId.trim().length > 0
                         ? requestRecord.runId
                         : undefined;
+                    const scenarioRunId = typeof requestRecord.scenarioRunId === "string" &&
+                        requestRecord.scenarioRunId.trim().length > 0
+                        ? requestRecord.scenarioRunId
+                        : undefined;
                     const error = typeof requestRecord.error === "string" &&
                         requestRecord.error.trim().length > 0
                         ? requestRecord.error
@@ -2287,6 +2361,7 @@ export function startWebSocketSimulator(opts) {
                     return [{
                             id: requestId,
                             status: normalizeVerifyBatchRequestStatus(requestRecord.status),
+                            scenarioRunId,
                             runId,
                             error,
                         }];
@@ -2307,11 +2382,19 @@ export function startWebSocketSimulator(opts) {
                     workspaceId: typeof batch.workspaceId === "string" && batch.workspaceId.trim()
                         ? batch.workspaceId
                         : "",
-                    graderId: batch.graderId,
-                    scenarioRunId: typeof batch.scenarioRunId === "string" &&
-                        batch.scenarioRunId.trim().length > 0
-                        ? batch.scenarioRunId
+                    scenarioDeckId: typeof batch.scenarioDeckId === "string" &&
+                        batch.scenarioDeckId.trim().length > 0
+                        ? batch.scenarioDeckId
                         : undefined,
+                    graderId: batch.graderId,
+                    scenarioRuns: typeof batch.scenarioRuns === "number" &&
+                        Number.isFinite(batch.scenarioRuns)
+                        ? Math.max(0, Math.round(batch.scenarioRuns))
+                        : 0,
+                    graderRepeatsPerScenario: typeof batch.graderRepeatsPerScenario === "number" &&
+                        Number.isFinite(batch.graderRepeatsPerScenario)
+                        ? Math.max(0, Math.round(batch.graderRepeatsPerScenario))
+                        : 0,
                     status: normalizeVerifyBatchStatus(batch.status),
                     startedAt: typeof batch.startedAt === "string" && batch.startedAt.trim().length >
                         0
@@ -2325,6 +2408,14 @@ export function startWebSocketSimulator(opts) {
                     active,
                     completed,
                     failed,
+                    scenarioRunsCompleted: typeof batch.scenarioRunsCompleted === "number" &&
+                        Number.isFinite(batch.scenarioRunsCompleted)
+                        ? Math.max(0, Math.round(batch.scenarioRunsCompleted))
+                        : 0,
+                    scenarioRunsFailed: typeof batch.scenarioRunsFailed === "number" &&
+                        Number.isFinite(batch.scenarioRunsFailed)
+                        ? Math.max(0, Math.round(batch.scenarioRunsFailed))
+                        : 0,
                     requests,
                 }];
         });
@@ -2978,6 +3069,7 @@ export function startWebSocketSimulator(opts) {
             if (!pendingTraceEvents.length)
                 return;
             for (const pending of pendingTraceEvents) {
+                persistOpenResponsesTraceEvent(state, pending, runId);
                 appendSessionEvent(state, {
                     ...pending,
                     kind: "trace",
@@ -2991,6 +3083,7 @@ export function startWebSocketSimulator(opts) {
             capturedTraces.push(stamped);
             consoleTracer?.(stamped);
             if (savedState?.meta?.sessionId) {
+                persistOpenResponsesTraceEvent(savedState, stamped, runId);
                 appendSessionEvent(savedState, {
                     ...stamped,
                     kind: "trace",
@@ -3469,6 +3562,35 @@ export function startWebSocketSimulator(opts) {
         }
         return state;
     };
+    const buildAuthorityMeta = (sessionState, workspaceId) => {
+        const sessionMeta = sessionState.meta && typeof sessionState.meta === "object"
+            ? sessionState.meta
+            : {};
+        const next = { workspaceId };
+        for (const key of [
+            "sessionId",
+            "sessionDir",
+            "sessionStatePath",
+            "sessionEventsPath",
+            "sessionBuildStatePath",
+            "sessionSqlitePath",
+            "workspaceSchemaVersion",
+        ]) {
+            const value = sessionMeta[key];
+            if (typeof value === "string" && value.trim().length > 0) {
+                next[key] = value;
+            }
+        }
+        return next;
+    };
+    const normalizeBuildAuthorityState = (args) => ({
+        ...args.state,
+        runId: args.runId,
+        meta: {
+            ...(args.state.meta ?? {}),
+            ...buildAuthorityMeta(args.sessionState, args.workspaceId),
+        },
+    });
     const broadcastBuild = (workspaceId, payload) => {
         const state = readSessionState(workspaceId);
         if (state) {
@@ -3482,12 +3604,44 @@ export function startWebSocketSimulator(opts) {
         if (active?.run.status === "running") {
             throw new Error("Build run already in progress for this workspace.");
         }
-        const state = ensureWorkspaceStateForBuild(workspaceId);
+        const sessionState = ensureWorkspaceStateForBuild(workspaceId);
         const projection = readBuildState(workspaceId);
         const seedRun = projection?.run;
         const runId = seedRun?.id && seedRun.id.trim().length > 0
             ? seedRun.id
             : randomId("build");
+        const seededMessages = (() => {
+            if (projection?.state?.messages && Array.isArray(projection.state.messages)) {
+                return projection.state.messages;
+            }
+            return Array.isArray(seedRun?.messages)
+                ? seedRun.messages.map((message) => {
+                    const role = message.role === "assistant" || message.role === "system" ||
+                        message.role === "tool"
+                        ? message.role
+                        : "user";
+                    return {
+                        role,
+                        content: message.content,
+                    };
+                })
+                : [];
+        })();
+        const seedState = normalizeBuildAuthorityState({
+            state: projection?.state && typeof projection.state === "object"
+                ? projection.state
+                : {
+                    runId,
+                    messages: seededMessages,
+                    messageRefs: [],
+                    traces: Array.isArray(seedRun?.traces) ? [...seedRun.traces] : [],
+                    items: [],
+                    meta: {},
+                },
+            workspaceId,
+            runId,
+            sessionState,
+        });
         const trimmedMessage = args.message.trim();
         const run = {
             id: runId,
@@ -3506,29 +3660,35 @@ export function startWebSocketSimulator(opts) {
                 ...run.messages,
                 { role: "user", content: trimmedMessage, messageSource: "manual" },
             ];
+            persistCanonicalUserInputEvent({
+                state: sessionState,
+                runId,
+                message: trimmedMessage,
+                source: "build",
+            });
         }
         const controller = new AbortController();
         const entry = {
             run,
-            state,
+            state: seedState,
             promise: null,
             abort: controller,
         };
         buildBotRuns.set(workspaceId, entry);
         const onStateUpdate = (next) => {
-            const enriched = persistSessionState({
-                ...next,
-                meta: {
-                    ...(next.meta ?? {}),
-                    workspaceId,
-                },
+            const buildState = normalizeBuildAuthorityState({
+                state: next,
+                workspaceId,
+                runId,
+                sessionState,
             });
-            entry.state = enriched;
-            const snapshot = buildTestBotSnapshot(enriched);
+            persistOpenResponsesTracesFromState(buildState, runId);
+            entry.state = buildState;
+            const snapshot = buildTestBotSnapshot(buildState);
             run.messages = snapshot.messages;
             run.toolInserts = snapshot.toolInserts;
-            const nextTraces = Array.isArray(enriched.traces)
-                ? [...enriched.traces]
+            const nextTraces = Array.isArray(buildState.traces)
+                ? [...buildState.traces]
                 : [];
             if (nextTraces.length > 0) {
                 run.traces = nextTraces;
@@ -3540,12 +3700,14 @@ export function startWebSocketSimulator(opts) {
                 type: "buildBotStatus",
                 workspaceId,
                 run,
+                state: buildState,
             });
         };
         const tracer = (trace) => {
             const event = trace.ts ? trace : { ...trace, ts: Date.now() };
             const currentTraces = Array.isArray(run.traces) ? run.traces : [];
             run.traces = [...currentTraces, event];
+            persistOpenResponsesTraceEvent(entry.state, event, runId);
             broadcastBuild(workspaceId, {
                 type: "buildBotTrace",
                 workspaceId,
@@ -3562,6 +3724,7 @@ export function startWebSocketSimulator(opts) {
                     type: "buildBotStatus",
                     workspaceId,
                     run,
+                    state: entry.state ?? undefined,
                 });
                 await runDeck({
                     path: buildAssistantDeckPath,
@@ -3569,9 +3732,13 @@ export function startWebSocketSimulator(opts) {
                     inputProvided: hasInitialContext,
                     modelProvider: opts.modelProvider,
                     defaultModel: opts.model,
-                    modelOverride: opts.modelForce,
+                    modelOverride: opts.modelForce ??
+                        (!opts.model && args.buildChatProvider
+                            ? `${args.buildChatProvider}/default`
+                            : undefined),
                     trace: tracer,
                     stream: true,
+                    runId,
                     state: entry.state ?? undefined,
                     allowRootStringInput: true,
                     initialUserMessage: trimmedMessage.length > 0 ? trimmedMessage : "",
@@ -3650,6 +3817,7 @@ export function startWebSocketSimulator(opts) {
                     type: "buildBotStatus",
                     workspaceId,
                     run,
+                    state: entry.state ?? undefined,
                 });
             }
         })();
@@ -3678,14 +3846,6 @@ export function startWebSocketSimulator(opts) {
         catch {
             // Ignore aborted in-flight run.
         }
-        const state = ensureWorkspaceStateForBuild(workspaceId);
-        const reset = persistSessionState({
-            ...state,
-            messages: [],
-            messageRefs: [],
-            traces: [],
-            items: [],
-        });
         const run = {
             id: randomId("build"),
             status: "idle",
@@ -3693,10 +3853,24 @@ export function startWebSocketSimulator(opts) {
             traces: [],
             toolInserts: [],
         };
-        appendWorkspaceEnvelope(reset, "build", {
+        const sessionState = ensureWorkspaceStateForBuild(workspaceId);
+        const reset = normalizeBuildAuthorityState({
+            state: {
+                runId: run.id,
+                messages: [],
+                messageRefs: [],
+                traces: [],
+                items: [],
+            },
+            workspaceId,
+            runId: run.id,
+            sessionState,
+        });
+        appendWorkspaceEnvelope(sessionState, "build", {
             type: "buildBotStatus",
             workspaceId,
             run,
+            state: reset,
         });
         appendDurableStreamEvent(WORKSPACE_STREAM_ID, {
             type: "buildBotStatus",
@@ -3973,6 +4147,9 @@ export function startWebSocketSimulator(opts) {
         const tracer = (event) => {
             const stamped = event.ts ? event : { ...event, ts: Date.now() };
             capturedTraces.push(stamped);
+            if (savedState) {
+                persistOpenResponsesTraceEvent(savedState, stamped, args.runId);
+            }
             emitTestBot({
                 type: "testBotTrace",
                 workspaceId: args.workspaceId,
@@ -3988,6 +4165,12 @@ export function startWebSocketSimulator(opts) {
                 ...run.messages,
                 { role: "user", content: trimmedMessage },
             ];
+            persistCanonicalUserInputEvent({
+                state: savedState,
+                runId: args.runId,
+                message: trimmedMessage,
+                source: "scenario",
+            });
         }
         let hasStartedAssistantStreamMessage = false;
         entry.promise = (async () => {
@@ -4040,6 +4223,7 @@ export function startWebSocketSimulator(opts) {
                             meta: nextMeta,
                             traces: capturedTraces,
                         });
+                        persistOpenResponsesTracesFromState(enriched, args.runId);
                         savedState = enriched;
                         entry.state = enriched;
                         appendFromState(enriched);
@@ -4336,40 +4520,79 @@ export function startWebSocketSimulator(opts) {
         if (!state) {
             throw new Error(`Workspace ${args.workspaceId} not found`);
         }
-        const explicitScenarioRunId = typeof args.scenarioRunId === "string" &&
-            args.scenarioRunId.trim().length > 0
-            ? args.scenarioRunId.trim()
-            : null;
-        const runFromState = state.meta && typeof state.meta.scenarioRunId === "string"
-            ? state.meta.scenarioRunId
-            : null;
-        const selectedScenarioRunId = explicitScenarioRunId ?? runFromState;
-        const scenarioRun = selectedScenarioRunId
-            ? readWorkspaceScenarioRunsForGraphql(args.workspaceId).find((entry) => entry.id === selectedScenarioRunId)
-            : null;
-        if (selectedScenarioRunId && !scenarioRun) {
-            throw new Error(`Scenario run ${selectedScenarioRunId} not found`);
-        }
-        const normalizedBatchSize = Math.max(1, Math.min(VERIFY_BATCH_SIZE_MAX, Number.isFinite(args.batchSize) ? Math.round(args.batchSize) : 1));
-        const normalizedConcurrency = Math.max(1, Math.min(VERIFY_BATCH_CONCURRENCY_MAX, normalizedBatchSize, Number.isFinite(args.concurrency) ? Math.round(args.concurrency) : 1));
+        const selectedScenarioDeckId = typeof args.scenarioDeckId === "string" &&
+            args.scenarioDeckId.trim().length > 0
+            ? args.scenarioDeckId.trim()
+            : undefined;
+        const normalizedScenarioRuns = Math.max(1, Math.min(VERIFY_SCENARIO_RUNS_MAX, Number.isFinite(args.scenarioRuns)
+            ? Math.round(args.scenarioRuns)
+            : DEFAULT_VERIFY_SCENARIO_RUNS));
+        const normalizedRepeats = Math.max(1, Math.min(VERIFY_GRADER_REPEATS_MAX, Number.isFinite(args.graderRepeatsPerScenario)
+            ? Math.round(args.graderRepeatsPerScenario)
+            : DEFAULT_VERIFY_GRADER_REPEATS));
+        const requestedSamples = normalizedScenarioRuns * normalizedRepeats;
+        const normalizedConcurrency = Math.max(1, Math.min(VERIFY_BATCH_CONCURRENCY_MAX, requestedSamples, Number.isFinite(args.concurrency)
+            ? Math.round(args.concurrency)
+            : DEFAULT_VERIFY_CONCURRENCY));
         const now = new Date().toISOString();
         const batchId = randomId("vbatch");
         let batch = {
             id: batchId,
             workspaceId: args.workspaceId,
+            scenarioDeckId: selectedScenarioDeckId,
             graderId: grader.id,
-            scenarioRunId: selectedScenarioRunId ?? undefined,
+            scenarioRuns: normalizedScenarioRuns,
+            graderRepeatsPerScenario: normalizedRepeats,
             status: "running",
             startedAt: now,
             finishedAt: undefined,
-            requested: normalizedBatchSize,
+            requested: requestedSamples,
             active: 0,
             completed: 0,
             failed: 0,
-            requests: Array.from({ length: normalizedBatchSize }, (_value, index) => ({
-                id: `${batchId}:${index + 1}`,
+            scenarioRunsCompleted: 0,
+            scenarioRunsFailed: 0,
+            requests: Array.from({ length: normalizedScenarioRuns }, (_value, scenarioIndex) => Array.from({ length: normalizedRepeats }, (_repeat, repeatIndex) => ({
+                id: `${batchId}:s${scenarioIndex + 1}:r${repeatIndex + 1}`,
                 status: "queued",
-            })),
+            }))).flat(),
+        };
+        const recomputeBatchState = (baseBatch, nextRequests, options) => {
+            const active = nextRequests.filter((request) => request.status === "running").length;
+            const completed = nextRequests.filter((request) => request.status === "completed").length;
+            const failed = nextRequests.filter((request) => request.status === "error")
+                .length;
+            const terminal = completed + failed === requestedSamples && active === 0;
+            return {
+                ...baseBatch,
+                requests: nextRequests,
+                active,
+                completed,
+                failed,
+                scenarioRunsCompleted: options?.scenarioRunsCompleted ??
+                    baseBatch.scenarioRunsCompleted,
+                scenarioRunsFailed: options?.scenarioRunsFailed ??
+                    baseBatch.scenarioRunsFailed,
+                status: terminal ? (failed > 0 ? "error" : "completed") : "running",
+                finishedAt: terminal ? new Date().toISOString() : undefined,
+            };
+        };
+        const awaitScenarioRunTerminal = async (run) => {
+            if (run.status !== "running" && run.status !== "idle") {
+                return run;
+            }
+            const active = testBotRuns.get(run.id);
+            if (active?.promise) {
+                try {
+                    await active.promise;
+                }
+                catch {
+                    // Run status/error is projected on the run record.
+                }
+            }
+            const latest = readWorkspaceScenarioRunsForGraphql(args.workspaceId)
+                .find((entry) => entry.id === run.id);
+            return latest ?? run;
         };
         let persistedState = persistSessionState(upsertWorkspaceVerifyBatchInState(state, batch));
         const persistAndBroadcastBatch = (reason) => {
@@ -4391,18 +4614,27 @@ export function startWebSocketSimulator(opts) {
         };
         persistAndBroadcastBatch("created");
         let updateQueue = Promise.resolve();
-        const updateBatchRequest = async (requestIndex, patch) => {
+        const queueBatchUpdate = async (reason, updater) => {
             updateQueue = updateQueue.then(() => {
-                if (requestIndex < 0 || requestIndex >= batch.requests.length)
-                    return;
-                const nextRequests = batch.requests.map((request, index) => index === requestIndex ? { ...request, ...patch } : request);
+                batch = updater(batch);
+                persistAndBroadcastBatch(reason);
+            });
+            await updateQueue;
+        };
+        const updateBatchRequest = async (requestIndex, patch) => {
+            await queueBatchUpdate("request-update", (current) => {
+                if (requestIndex < 0 || requestIndex >= current.requests.length) {
+                    return current;
+                }
+                const nextRequests = current.requests.map((request, index) => index === requestIndex ? { ...request, ...patch } : request);
                 const active = nextRequests.filter((request) => request.status === "running").length;
                 const completed = nextRequests.filter((request) => request.status === "completed")
                     .length;
                 const failed = nextRequests.filter((request) => request.status === "error").length;
-                const terminal = completed + failed === batch.requested && active === 0;
-                batch = {
-                    ...batch,
+                const terminal = completed + failed === current.requested &&
+                    active === 0;
+                return {
+                    ...current,
                     requests: nextRequests,
                     active,
                     completed,
@@ -4410,27 +4642,63 @@ export function startWebSocketSimulator(opts) {
                     status: terminal ? (failed > 0 ? "error" : "completed") : "running",
                     finishedAt: terminal ? new Date().toISOString() : undefined,
                 };
-                persistAndBroadcastBatch("request-update");
             });
-            await updateQueue;
         };
-        let cursor = 0;
-        const workers = Array.from({ length: normalizedConcurrency }, () => (async () => {
+        const queuedGradeRequestIndexes = [];
+        let gradeQueueClosed = false;
+        const gradeQueueWaiters = [];
+        const wakeGradeWorkers = (count) => {
+            let remaining = Math.max(0, count);
+            while (remaining > 0 && gradeQueueWaiters.length > 0) {
+                const notify = gradeQueueWaiters.shift();
+                if (notify)
+                    notify();
+                remaining -= 1;
+            }
+        };
+        const enqueueGradeRequestIndexes = (requestIndexes) => {
+            if (requestIndexes.length === 0)
+                return;
+            queuedGradeRequestIndexes.push(...requestIndexes);
+            wakeGradeWorkers(requestIndexes.length);
+        };
+        const closeGradeQueue = () => {
+            gradeQueueClosed = true;
+            while (gradeQueueWaiters.length > 0) {
+                const notify = gradeQueueWaiters.shift();
+                if (notify)
+                    notify();
+            }
+        };
+        const nextQueuedGradeRequestIndex = async () => {
+            while (queuedGradeRequestIndexes.length === 0) {
+                if (gradeQueueClosed)
+                    return undefined;
+                await new Promise((resolve) => {
+                    gradeQueueWaiters.push(resolve);
+                });
+            }
+            return queuedGradeRequestIndexes.shift();
+        };
+        const gradeWorkers = Array.from({ length: normalizedConcurrency }, () => (async () => {
             while (true) {
-                const requestIndex = cursor;
-                cursor += 1;
-                if (requestIndex >= normalizedBatchSize)
+                const requestIndex = await nextQueuedGradeRequestIndex();
+                if (requestIndex === undefined)
                     return;
+                const scenarioRunId = batch.requests[requestIndex]?.scenarioRunId;
+                if (!scenarioRunId)
+                    continue;
                 await updateBatchRequest(requestIndex, { status: "running" });
                 try {
                     const run = await createWorkspaceGradeRunForGraphql({
                         workspaceId: args.workspaceId,
                         graderId: grader.id,
-                        scenarioRunId: selectedScenarioRunId,
+                        scenarioRunId,
                     });
                     if (run.status === "completed") {
                         await updateBatchRequest(requestIndex, {
                             status: "completed",
+                            scenarioRunId,
                             runId: run.id,
                             error: undefined,
                         });
@@ -4438,6 +4706,7 @@ export function startWebSocketSimulator(opts) {
                     else {
                         await updateBatchRequest(requestIndex, {
                             status: "error",
+                            scenarioRunId,
                             runId: run.id,
                             error: run.error ??
                                 `Grade run ended with status ${run.status}`,
@@ -4447,12 +4716,82 @@ export function startWebSocketSimulator(opts) {
                 catch (error) {
                     await updateBatchRequest(requestIndex, {
                         status: "error",
+                        scenarioRunId,
                         error: error instanceof Error ? error.message : String(error),
                     });
                 }
             }
         })());
-        await Promise.all(workers);
+        let scenarioCursor = 0;
+        const nextScenarioIndex = () => {
+            if (scenarioCursor >= normalizedScenarioRuns)
+                return undefined;
+            const current = scenarioCursor;
+            scenarioCursor += 1;
+            return current;
+        };
+        const scenarioWorkerCount = Math.max(1, Math.min(normalizedScenarioRuns, normalizedConcurrency));
+        const scenarioWorkers = Array.from({ length: scenarioWorkerCount }, () => (async () => {
+            while (true) {
+                const scenarioIndex = nextScenarioIndex();
+                if (scenarioIndex === undefined)
+                    return;
+                const requestStart = scenarioIndex * normalizedRepeats;
+                const requestEndExclusive = requestStart + normalizedRepeats;
+                try {
+                    const startedRun = await startWorkspaceScenarioRunForGraphql({
+                        workspaceId: args.workspaceId,
+                        scenarioDeckId: selectedScenarioDeckId,
+                    });
+                    const run = await awaitScenarioRunTerminal(startedRun);
+                    if (run.status !== "completed") {
+                        const scenarioError = run.error ??
+                            `Scenario run ended with status ${run.status}`;
+                        await queueBatchUpdate("scenario-run-failed", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                            requestIndex < requestEndExclusive
+                            ? {
+                                ...request,
+                                scenarioRunId: run.id,
+                                status: "error",
+                                error: scenarioError,
+                            }
+                            : request), {
+                            scenarioRunsFailed: current.scenarioRunsFailed + 1,
+                        }));
+                        continue;
+                    }
+                    await queueBatchUpdate("scenario-run-completed", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                        requestIndex < requestEndExclusive
+                        ? {
+                            ...request,
+                            scenarioRunId: run.id,
+                        }
+                        : request), {
+                        scenarioRunsCompleted: current.scenarioRunsCompleted + 1,
+                    }));
+                    const requestIndexes = Array.from({ length: normalizedRepeats }, (_value, repeatIndex) => requestStart + repeatIndex);
+                    enqueueGradeRequestIndexes(requestIndexes);
+                }
+                catch (error) {
+                    const scenarioError = error instanceof Error
+                        ? error.message
+                        : String(error);
+                    await queueBatchUpdate("scenario-run-error", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                        requestIndex < requestEndExclusive
+                        ? {
+                            ...request,
+                            status: "error",
+                            error: scenarioError,
+                        }
+                        : request), {
+                        scenarioRunsFailed: current.scenarioRunsFailed + 1,
+                    }));
+                }
+            }
+        })());
+        await Promise.all(scenarioWorkers);
+        closeGradeQueue();
+        await Promise.all(gradeWorkers);
         await updateQueue;
         return batch;
     };
@@ -4769,10 +5108,12 @@ export function startWebSocketSimulator(opts) {
             throw new Error("graderId is required");
         const batch = await createWorkspaceVerifyBatchRunForGraphql({
             workspaceId: args.workspaceId,
+            scenarioDeckId: args.scenarioDeckId ?? null,
             graderId: args.graderId,
-            scenarioRunId: args.scenarioRunId ?? null,
-            batchSize: args.batchSize ?? 1,
-            concurrency: args.concurrency ?? 1,
+            scenarioRuns: args.scenarioRuns ?? DEFAULT_VERIFY_SCENARIO_RUNS,
+            graderRepeatsPerScenario: args.graderRepeatsPerScenario ??
+                DEFAULT_VERIFY_GRADER_REPEATS,
+            concurrency: args.concurrency ?? DEFAULT_VERIFY_CONCURRENCY,
         });
         return toVerifyConversationSessionForGraphql(args.workspaceId, batch);
     };
@@ -5461,84 +5802,14 @@ export function startWebSocketSimulator(opts) {
                 return new Response("Method not allowed", { status: 405 });
             }
             const isLegacyCodexTrustEndpoint = url.pathname === "/api/codex/trust-workspace";
-            let provider = "codex-cli";
-            let workspaceId;
-            let online = true;
-            if (req.method === "POST") {
-                try {
-                    const body = await req.json();
-                    if (typeof body.workspaceId === "string") {
-                        workspaceId = body.workspaceId;
-                    }
-                    if (!isLegacyCodexTrustEndpoint &&
-                        body.provider === "claude-code-cli") {
-                        provider = "claude-code-cli";
-                    }
-                    if (body.online === true ||
-                        body.online === "true" ||
-                        body.online === 1 ||
-                        body.online === "1") {
-                        online = true;
-                    }
-                    else if (body.online === false ||
-                        body.online === "false" ||
-                        body.online === 0 ||
-                        body.online === "0" ||
-                        body.online === "no") {
-                        online = false;
-                    }
-                }
-                catch {
-                    // Ignore malformed body and fall back to query/default workspace.
-                }
-            }
-            if (!workspaceId) {
-                workspaceId = getWorkspaceIdFromQuery(url);
-            }
-            if (!isLegacyCodexTrustEndpoint &&
-                url.searchParams.get("provider") === "claude-code-cli") {
-                provider = "claude-code-cli";
-            }
-            const onlineQuery = url.searchParams.get("online");
-            if (onlineQuery === "1" || onlineQuery === "true" ||
-                onlineQuery === "yes") {
-                online = true;
-            }
-            else if (onlineQuery === "0" || onlineQuery === "false" ||
-                onlineQuery === "no") {
-                online = false;
-            }
-            try {
-                await logWorkspaceBotRoot(url.pathname, workspaceId);
-                if (!isLegacyCodexTrustEndpoint && provider === "claude-code-cli") {
-                    return new Response(JSON.stringify({
-                        ok: true,
-                        provider,
-                        loggedIn: false,
-                        loginStatus: "Claude Code status check is not yet supported in this build.",
-                        writeEnabled: false,
-                    }), { headers: { "content-type": "application/json" } });
-                }
-                const result = await readCodexWorkspaceStatus(workspaceId, online);
-                if (!isLegacyCodexTrustEndpoint) {
-                    return new Response(JSON.stringify({
-                        ok: true,
-                        provider,
-                        loggedIn: result.codexLoggedIn,
-                        loginStatus: result.codexLoginStatus,
-                        writeEnabled: result.writeEnabled,
-                        trustedPath: result.trustedPath,
-                    }), { headers: { "content-type": "application/json" } });
-                }
-                return new Response(JSON.stringify({
-                    ok: true,
-                    ...result,
-                }), { headers: { "content-type": "application/json" } });
-            }
-            catch (err) {
-                const message = err instanceof Error ? err.message : String(err);
-                return new Response(JSON.stringify({ ok: false, error: message }), { status: 400, headers: { "content-type": "application/json" } });
-            }
+            return await handleBuildProviderStatusRequest({
+                req,
+                url,
+                isLegacyCodexTrustEndpoint,
+                getWorkspaceIdFromQuery,
+                logWorkspaceBotRoot,
+                readCodexWorkspaceStatus,
+            });
         }
         if (url.pathname === "/api/simulator/run") {
             if (req.method !== "POST") {
@@ -5629,6 +5900,15 @@ export function startWebSocketSimulator(opts) {
                 _ensureWorkspaceSession(workspaceId);
                 await logWorkspaceBotRoot(url.pathname, workspaceId);
                 await activateWorkspaceDeck(workspaceId);
+                const buildChatProvider = normalizeBuildChatProvider(body.buildChatProvider) ?? undefined;
+                if (buildChatProvider) {
+                    const state = readSessionStateStrict(workspaceId, {
+                        withTraces: true,
+                    });
+                    if (state) {
+                        persistSessionState(persistBuildChatProviderMeta(state, workspaceId, buildChatProvider));
+                    }
+                }
                 const message = typeof body.message === "string"
                     ? body.message
                     : typeof body.input === "string"
@@ -5637,6 +5917,7 @@ export function startWebSocketSimulator(opts) {
                 const run = startWorkspaceBuildRun({
                     workspaceId,
                     message,
+                    buildChatProvider,
                 });
                 return new Response(JSON.stringify({ run }), {
                     headers: { "content-type": "application/json" },
@@ -5681,6 +5962,37 @@ export function startWebSocketSimulator(opts) {
                 });
             }
         }
+        if (url.pathname === "/api/build/provider") {
+            if (req.method !== "POST") {
+                return new Response("Method not allowed", { status: 405 });
+            }
+            try {
+                const body = await req.json().catch(() => ({}));
+                const workspaceId = _getWorkspaceIdFromBody(body);
+                if (!workspaceId) {
+                    throw new Error("Missing workspaceId");
+                }
+                const buildChatProvider = normalizeBuildChatProvider(body.buildChatProvider);
+                if (!buildChatProvider) {
+                    throw new Error("Invalid buildChatProvider; expected codex-cli or claude-code-cli");
+                }
+                await logWorkspaceBotRoot(url.pathname, workspaceId);
+                const state = readSessionStateStrict(workspaceId, {
+                    withTraces: true,
+                });
+                if (!state)
+                    throw new Error("Workspace not found");
+                persistSessionState(persistBuildChatProviderMeta(state, workspaceId, buildChatProvider));
+                return new Response(JSON.stringify({ ok: true, workspaceId, buildChatProvider }), { headers: { "content-type": "application/json" } });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                return new Response(JSON.stringify({ error: message }), {
+                    status: 400,
+                    headers: { "content-type": "application/json" },
+                });
+            }
+        }
         if (url.pathname === "/api/build/reset") {
             if (req.method !== "POST") {
                 return new Response("Method not allowed", { status: 405 });
@@ -5820,6 +6132,21 @@ export function startWebSocketSimulator(opts) {
                 return Promise.resolve({ ok: deleted });
             },
             readWorkspaceBuildRun: (workspaceId) => Promise.resolve(readWorkspaceBuildRunForGraphql(workspaceId)),
+            readWorkspaceOpenResponseEvents: (args) => Promise.resolve(listOpenResponsesRunEvents({
+                workspaceId: args.workspaceId,
+                runId: args.runId,
+                fromSequence: args.fromSequence,
+            })),
+            readWorkspaceOpenResponseOutputItems: (args) => Promise.resolve(listOpenResponsesOutputItems({
+                workspaceId: args.workspaceId,
+                runId: args.runId,
+            })),
+            subscribeWorkspaceOpenResponseEvents: (args) => subscribeOpenResponsesRunEvents({
+                workspaceId: args.workspaceId,
+                runId: args.runId,
+                fromSequence: args.fromSequence,
+                signal: args.signal,
+            }),
             createWorkspaceBuildRun: async (workspaceId, message) => await startWorkspaceBuildRun({ workspaceId, message }),
             stopWorkspaceBuildRun: async (workspaceId, runId) => await stopWorkspaceBuildRun({ workspaceId, runId }),
             resetWorkspaceBuild: async (workspaceId) => await resetWorkspaceBuild(workspaceId),