@bolt-foundry/gambit 0.8.6-rc.3 → 0.8.6-rc.4

package/script/src/server/workspace/verify.js

@@ -0,0 +1,453 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createWorkspaceVerifyService = void 0;
+const createWorkspaceVerifyService = (deps) => {
+    const normalizeVerifyBatchStatus = (value) => {
+        if (value === "running")
+            return "running";
+        if (value === "completed")
+            return "completed";
+        if (value === "error")
+            return "error";
+        return "idle";
+    };
+    const normalizeVerifyBatchRequestStatus = (value) => {
+        if (value === "running")
+            return "running";
+        if (value === "completed")
+            return "completed";
+        if (value === "error")
+            return "error";
+        return "queued";
+    };
+    const readWorkspaceVerifyBatchesFromState = (state) => {
+        const meta = state.meta && typeof state.meta === "object"
+            ? state.meta
+            : {};
+        if (!Array.isArray(meta.verifyBatches))
+            return [];
+        return meta.verifyBatches.flatMap((entry) => {
+            if (!entry || typeof entry !== "object")
+                return [];
+            const batch = entry;
+            if (typeof batch.graderId !== "string" || batch.graderId.trim() === "") {
+                return [];
+            }
+            const requests = Array.isArray(batch.requests)
+                ? batch.requests.flatMap((request, requestIndex) => {
+                    if (!request || typeof request !== "object")
+                        return [];
+                    const requestRecord = request;
+                    const requestId = typeof requestRecord.id === "string" &&
+                        requestRecord.id.trim().length > 0
+                        ? requestRecord.id
+                        : `${String(batch.id ?? deps.randomId("vbatch"))}:${requestIndex + 1}`;
+                    const runId = typeof requestRecord.runId === "string" &&
+                        requestRecord.runId.trim().length > 0
+                        ? requestRecord.runId
+                        : undefined;
+                    const scenarioRunId = typeof requestRecord.scenarioRunId === "string" &&
+                        requestRecord.scenarioRunId.trim().length > 0
+                        ? requestRecord.scenarioRunId
+                        : undefined;
+                    const error = typeof requestRecord.error === "string" &&
+                        requestRecord.error.trim().length > 0
+                        ? requestRecord.error
+                        : undefined;
+                    return [{
+                            id: requestId,
+                            status: normalizeVerifyBatchRequestStatus(requestRecord.status),
+                            scenarioRunId,
+                            runId,
+                            error,
+                        }];
+                })
+                : [];
+            const active = requests.filter((request) => request.status === "running")
+                .length;
+            const completed = requests.filter((request) => request.status === "completed").length;
+            const failed = requests.filter((request) => request.status === "error")
+                .length;
+            const requested = Math.max(requests.length, typeof batch.requested === "number" && Number.isFinite(batch.requested)
+                ? Math.max(0, Math.round(batch.requested))
+                : 0);
+            return [{
+                    id: typeof batch.id === "string" && batch.id.trim().length > 0
+                        ? batch.id
+                        : deps.randomId("vbatch"),
+                    workspaceId: typeof batch.workspaceId === "string" && batch.workspaceId.trim()
+                        ? batch.workspaceId
+                        : "",
+                    scenarioDeckId: typeof batch.scenarioDeckId === "string" &&
+                        batch.scenarioDeckId.trim().length > 0
+                        ? batch.scenarioDeckId
+                        : undefined,
+                    graderId: batch.graderId,
+                    scenarioRuns: typeof batch.scenarioRuns === "number" &&
+                        Number.isFinite(batch.scenarioRuns)
+                        ? Math.max(0, Math.round(batch.scenarioRuns))
+                        : 0,
+                    graderRepeatsPerScenario: typeof batch.graderRepeatsPerScenario === "number" &&
+                        Number.isFinite(batch.graderRepeatsPerScenario)
+                        ? Math.max(0, Math.round(batch.graderRepeatsPerScenario))
+                        : 0,
+                    status: normalizeVerifyBatchStatus(batch.status),
+                    startedAt: typeof batch.startedAt === "string" && batch.startedAt.trim().length >
+                        0
+                        ? batch.startedAt
+                        : undefined,
+                    finishedAt: typeof batch.finishedAt === "string" &&
+                        batch.finishedAt.trim().length > 0
+                        ? batch.finishedAt
+                        : undefined,
+                    requested,
+                    active,
+                    completed,
+                    failed,
+                    scenarioRunsCompleted: typeof batch.scenarioRunsCompleted === "number" &&
+                        Number.isFinite(batch.scenarioRunsCompleted)
+                        ? Math.max(0, Math.round(batch.scenarioRunsCompleted))
+                        : 0,
+                    scenarioRunsFailed: typeof batch.scenarioRunsFailed === "number" &&
+                        Number.isFinite(batch.scenarioRunsFailed)
+                        ? Math.max(0, Math.round(batch.scenarioRunsFailed))
+                        : 0,
+                    requests,
+                }];
+        });
+    };
+    const writeWorkspaceVerifyBatchesToState = (state, batches) => {
+        const currentMeta = state.meta && typeof state.meta === "object"
+            ? state.meta
+            : {};
+        return {
+            ...state,
+            meta: {
+                ...currentMeta,
+                verifyBatches: batches,
+            },
+        };
+    };
+    const upsertWorkspaceVerifyBatchInState = (state, nextBatch) => {
+        const existing = readWorkspaceVerifyBatchesFromState(state);
+        const nextBatches = [...existing];
+        const existingIndex = nextBatches.findIndex((entry) => entry.id === nextBatch.id);
+        if (existingIndex >= 0) {
+            nextBatches[existingIndex] = nextBatch;
+        }
+        else {
+            nextBatches.unshift(nextBatch);
+        }
+        return writeWorkspaceVerifyBatchesToState(state, nextBatches.slice(0, 50));
+    };
+    const emitVerifyWorkspaceEvent = (workspaceId, payload) => {
+        deps.appendDurableStreamEvent(deps.workspaceStreamId, {
+            ...payload,
+            workspaceId,
+        });
+    };
+    const readWorkspaceVerifyBatchesForGraphql = (workspaceId) => {
+        const state = deps.readSessionState(workspaceId);
+        if (!state)
+            return [];
+        return readWorkspaceVerifyBatchesFromState(state).map((batch) => ({
+            ...batch,
+            workspaceId,
+        }));
+    };
+    const createWorkspaceVerifyBatchRunForGraphql = async (args) => {
+        await deps.activateWorkspaceDeck(args.workspaceId, {
+            source: "graphql:createWorkspaceVerifyBatchRun",
+        });
+        const deckState = deps.readWorkspaceDeckStateStrict(args.workspaceId);
+        const grader = deps.resolveGraderDeckFromState(deckState, args.graderId);
+        if (!grader) {
+            throw new Error(`Unknown grader deck: ${args.graderId}`);
+        }
+        const state = deps.readSessionStateStrict(args.workspaceId, {
+            withTraces: true,
+        });
+        if (!state) {
+            throw new Error(`Workspace ${args.workspaceId} not found`);
+        }
+        const selectedScenarioDeckId = typeof args.scenarioDeckId === "string" &&
+            args.scenarioDeckId.trim().length > 0
+            ? args.scenarioDeckId.trim()
+            : undefined;
+        const normalizedScenarioRuns = Math.max(1, Math.min(deps.verifyScenarioRunsMax, Number.isFinite(args.scenarioRuns)
+            ? Math.round(args.scenarioRuns)
+            : deps.defaultVerifyScenarioRuns));
+        const normalizedRepeats = Math.max(1, Math.min(deps.verifyGraderRepeatsMax, Number.isFinite(args.graderRepeatsPerScenario)
+            ? Math.round(args.graderRepeatsPerScenario)
+            : deps.defaultVerifyGraderRepeats));
+        const requestedSamples = normalizedScenarioRuns * normalizedRepeats;
+        const normalizedConcurrency = Math.max(1, Math.min(deps.verifyBatchConcurrencyMax, requestedSamples, Number.isFinite(args.concurrency)
+            ? Math.round(args.concurrency)
+            : deps.defaultVerifyConcurrency));
+        const now = new Date().toISOString();
+        const batchId = deps.randomId("vbatch");
+        let batch = {
+            id: batchId,
+            workspaceId: args.workspaceId,
+            scenarioDeckId: selectedScenarioDeckId,
+            graderId: grader.id,
+            scenarioRuns: normalizedScenarioRuns,
+            graderRepeatsPerScenario: normalizedRepeats,
+            status: "running",
+            startedAt: now,
+            finishedAt: undefined,
+            requested: requestedSamples,
+            active: 0,
+            completed: 0,
+            failed: 0,
+            scenarioRunsCompleted: 0,
+            scenarioRunsFailed: 0,
+            requests: Array.from({ length: normalizedScenarioRuns }, (_value, scenarioIndex) => Array.from({ length: normalizedRepeats }, (_repeat, repeatIndex) => ({
+                id: `${batchId}:s${scenarioIndex + 1}:r${repeatIndex + 1}`,
+                status: "queued",
+            }))).flat(),
+        };
+        const recomputeBatchState = (baseBatch, nextRequests, options) => {
+            const active = nextRequests.filter((request) => request.status === "running").length;
+            const completed = nextRequests.filter((request) => request.status === "completed").length;
+            const failed = nextRequests.filter((request) => request.status === "error")
+                .length;
+            const terminal = completed + failed === requestedSamples && active === 0;
+            return {
+                ...baseBatch,
+                requests: nextRequests,
+                active,
+                completed,
+                failed,
+                scenarioRunsCompleted: options?.scenarioRunsCompleted ??
+                    baseBatch.scenarioRunsCompleted,
+                scenarioRunsFailed: options?.scenarioRunsFailed ??
+                    baseBatch.scenarioRunsFailed,
+                status: terminal ? (failed > 0 ? "error" : "completed") : "running",
+                finishedAt: terminal ? new Date().toISOString() : undefined,
+            };
+        };
+        const awaitScenarioRunTerminal = async (run) => {
+            if (run.status !== "running" && run.status !== "idle") {
+                return run;
+            }
+            const active = deps.getLiveTestRunEntry(run.id);
+            if (active?.promise) {
+                try {
+                    await active.promise;
+                }
+                catch {
+                    // Run status/error is projected on the run record.
+                }
+            }
+            const latest = deps.readWorkspaceScenarioRunsForGraphql(args.workspaceId)
+                .find((entry) => entry.id === run.id);
+            return latest ?? run;
+        };
+        let persistedState = deps.persistSessionState(upsertWorkspaceVerifyBatchInState(state, batch));
+        const persistAndBroadcastBatch = (reason) => {
+            const latest = deps.readSessionStateStrict(args.workspaceId, {
+                withTraces: true,
+            }) ?? persistedState;
+            persistedState = deps.persistSessionState(upsertWorkspaceVerifyBatchInState(latest, batch));
+            deps.appendGradingLog(persistedState, {
+                type: "gambit.verify.batch",
+                workspaceId: args.workspaceId,
+                reason,
+                batch,
+            });
+            emitVerifyWorkspaceEvent(args.workspaceId, {
+                type: "gambit.verify.batch",
+                reason,
+                batch,
+            });
+        };
+        persistAndBroadcastBatch("created");
+        let updateQueue = Promise.resolve();
+        const queueBatchUpdate = async (reason, updater) => {
+            updateQueue = updateQueue.then(() => {
+                batch = updater(batch);
+                persistAndBroadcastBatch(reason);
+            });
+            await updateQueue;
+        };
+        const updateBatchRequest = async (requestIndex, patch) => {
+            await queueBatchUpdate("request-update", (current) => {
+                if (requestIndex < 0 || requestIndex >= current.requests.length) {
+                    return current;
+                }
+                const nextRequests = current.requests.map((request, index) => index === requestIndex ? { ...request, ...patch } : request);
+                const active = nextRequests.filter((request) => request.status === "running").length;
+                const completed = nextRequests.filter((request) => request.status === "completed")
+                    .length;
+                const failed = nextRequests.filter((request) => request.status === "error").length;
+                const terminal = completed + failed === current.requested &&
+                    active === 0;
+                return {
+                    ...current,
+                    requests: nextRequests,
+                    active,
+                    completed,
+                    failed,
+                    status: terminal ? (failed > 0 ? "error" : "completed") : "running",
+                    finishedAt: terminal ? new Date().toISOString() : undefined,
+                };
+            });
+        };
+        const queuedGradeRequestIndexes = [];
+        let gradeQueueClosed = false;
+        const gradeQueueWaiters = [];
+        const wakeGradeWorkers = (count) => {
+            let remaining = Math.max(0, count);
+            while (remaining > 0 && gradeQueueWaiters.length > 0) {
+                const notify = gradeQueueWaiters.shift();
+                if (notify)
+                    notify();
+                remaining -= 1;
+            }
+        };
+        const enqueueGradeRequestIndexes = (requestIndexes) => {
+            if (requestIndexes.length === 0)
+                return;
+            queuedGradeRequestIndexes.push(...requestIndexes);
+            wakeGradeWorkers(requestIndexes.length);
+        };
+        const closeGradeQueue = () => {
+            gradeQueueClosed = true;
+            while (gradeQueueWaiters.length > 0) {
+                const notify = gradeQueueWaiters.shift();
+                if (notify)
+                    notify();
+            }
+        };
+        const nextQueuedGradeRequestIndex = async () => {
+            while (queuedGradeRequestIndexes.length === 0) {
+                if (gradeQueueClosed)
+                    return undefined;
+                await new Promise((resolve) => {
+                    gradeQueueWaiters.push(resolve);
+                });
+            }
+            return queuedGradeRequestIndexes.shift();
+        };
+        const gradeWorkers = Array.from({ length: normalizedConcurrency }, () => (async () => {
+            while (true) {
+                const requestIndex = await nextQueuedGradeRequestIndex();
+                if (requestIndex === undefined)
+                    return;
+                const scenarioRunId = batch.requests[requestIndex]?.scenarioRunId;
+                if (!scenarioRunId)
+                    continue;
+                await updateBatchRequest(requestIndex, { status: "running" });
+                try {
+                    const run = await deps.createWorkspaceGradeRunForGraphql({
+                        workspaceId: args.workspaceId,
+                        graderId: grader.id,
+                        scenarioRunId,
+                    });
+                    if (run.status === "completed") {
+                        await updateBatchRequest(requestIndex, {
+                            status: "completed",
+                            scenarioRunId,
+                            runId: run.id,
+                            error: undefined,
+                        });
+                    }
+                    else {
+                        await updateBatchRequest(requestIndex, {
+                            status: "error",
+                            scenarioRunId,
+                            runId: run.id,
+                            error: run.error ??
+                                `Grade run ended with status ${run.status}`,
+                        });
+                    }
+                }
+                catch (error) {
+                    await updateBatchRequest(requestIndex, {
+                        status: "error",
+                        scenarioRunId,
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+        })());
+        let scenarioCursor = 0;
+        const nextScenarioIndex = () => {
+            if (scenarioCursor >= normalizedScenarioRuns)
+                return undefined;
+            const current = scenarioCursor;
+            scenarioCursor += 1;
+            return current;
+        };
+        const scenarioWorkerCount = Math.max(1, Math.min(normalizedScenarioRuns, normalizedConcurrency));
+        const scenarioWorkers = Array.from({ length: scenarioWorkerCount }, () => (async () => {
+            while (true) {
+                const scenarioIndex = nextScenarioIndex();
+                if (scenarioIndex === undefined)
+                    return;
+                const requestStart = scenarioIndex * normalizedRepeats;
+                const requestEndExclusive = requestStart + normalizedRepeats;
+                try {
+                    const startedRun = await deps.startWorkspaceScenarioRunForGraphql({
+                        workspaceId: args.workspaceId,
+                        scenarioDeckId: selectedScenarioDeckId,
+                    });
+                    const run = await awaitScenarioRunTerminal(startedRun);
+                    if (run.status !== "completed") {
+                        const scenarioError = run.error ??
+                            `Scenario run ended with status ${run.status}`;
+                        await queueBatchUpdate("scenario-run-failed", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                            requestIndex < requestEndExclusive
+                            ? {
+                                ...request,
+                                scenarioRunId: run.id,
+                                status: "error",
+                                error: scenarioError,
+                            }
+                            : request), {
+                            scenarioRunsFailed: current.scenarioRunsFailed + 1,
+                        }));
+                        continue;
+                    }
+                    await queueBatchUpdate("scenario-run-completed", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                        requestIndex < requestEndExclusive
+                        ? {
+                            ...request,
+                            scenarioRunId: run.id,
+                        }
+                        : request), {
+                        scenarioRunsCompleted: current.scenarioRunsCompleted + 1,
+                    }));
+                    const requestIndexes = Array.from({ length: normalizedRepeats }, (_value, repeatIndex) => requestStart + repeatIndex);
+                    enqueueGradeRequestIndexes(requestIndexes);
+                }
+                catch (error) {
+                    const scenarioError = error instanceof Error
+                        ? error.message
+                        : String(error);
+                    await queueBatchUpdate("scenario-run-error", (current) => recomputeBatchState(current, current.requests.map((request, requestIndex) => requestIndex >= requestStart &&
+                        requestIndex < requestEndExclusive
+                        ? {
+                            ...request,
+                            status: "error",
+                            error: scenarioError,
+                        }
+                        : request), {
+                        scenarioRunsFailed: current.scenarioRunsFailed + 1,
+                    }));
+                }
+            }
+        })());
+        await Promise.all(scenarioWorkers);
+        closeGradeQueue();
+        await Promise.all(gradeWorkers);
+        await updateQueue;
+        return batch;
+    };
+    return {
+        readWorkspaceVerifyBatchesForGraphql,
+        createWorkspaceVerifyBatchRunForGraphql,
+    };
+};
+exports.createWorkspaceVerifyService = createWorkspaceVerifyService;

package/script/src/server.d.ts

@@ -1,8 +1,5 @@
 import * as dntShim from "../_dnt.shims.js";
 import type { ModelProvider } from "@bolt-foundry/gambit-core";
-/**
- * Start the WebSocket simulator server used by the Gambit debug UI.
- */
 export declare function startWebSocketSimulator(opts: {
     deckPath: string;
     modelProvider: ModelProvider;

package/script/src/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/src/server.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AA6E5C,OAAO,KAAK,EAMV,aAAa,EAOd,MAAM,2BAA2B,CAAC;AAipCnC;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,aAAa,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE;QACV,EAAE,EAAE,MAAM,CAAC;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,UAAU,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAosNxC"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/src/server.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAM5C,OAAO,KAAK,EAEV,aAAa,EAEd,MAAM,2BAA2B,CAAC;AAuGnC,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,aAAa,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE;QACV,EAAE,EAAE,MAAM,CAAC;QACX,YAAY,EAAE,MAAM,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB,GAAG,UAAU,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAwsBxC"}