@straiffi/archon 1.7.7 → 1.7.9

package/dist/server/lib/checkRunner.js

@@ -0,0 +1,674 @@
+import { randomUUID } from 'crypto';
+import { spawn } from 'child_process';
+import { existsSync, lstatSync, realpathSync } from 'fs';
+import { relative, resolve } from 'path';
+import { StringDecoder } from 'string_decoder';
+import { appendCheckAttemptOutput, findActiveCheckBatchForTarget, getCheckBatchDetails, listCheckAttemptDetails, rerunCheckBatch, setCheckBatchStopRequested, } from './checkRuns.js';
+import { getCheckDefinitionFingerprint } from './checkDefinitions.js';
+import { captureCheckWorkspaceIdentity } from './checkWorkspaceIdentity.js';
+import { getProjectById, getProjectCheckDefinitions } from './projects.js';
+import { aggregateCheckBatch, completeCheckAttempt, startCheckAttempt, withExecutionRunTransaction, } from './executionRuns.js';
+import { resolveCommandWorkspacePath } from './testCommandRunner.js';
+const MAX_CONCURRENT_CHECK_ATTEMPTS = 2;
+const OUTPUT_FLUSH_BYTES = 32 * 1024;
+const OUTPUT_FLUSH_DELAY_MS = 25;
+const OUTPUT_PERSIST_RETRY_DELAYS_MS = [50, 150, 500];
+const STOP_GRACE_MS = 2_000;
+const queue = [];
+const processes = new Map();
+const admittedEntries = new Map();
+const invalidatedBatches = new Set();
+const stoppedBatches = new Set();
+const forcedBatchOutcomes = new Map();
+const batchWaiters = new Map();
+let admittedAttempts = 0;
+let acceptingAdmissions = true;
+let inFlightAdmissions = 0;
+const admissionWaiters = new Set();
+const assertAcceptingAdmissions = () => {
+    if (!acceptingAdmissions) {
+        throw new Error('Check runner is shutting down');
+    }
+};
+const withCheckAdmission = async (operation) => {
+    assertAcceptingAdmissions();
+    inFlightAdmissions += 1;
+    try {
+        return await operation();
+    }
+    finally {
+        inFlightAdmissions -= 1;
+        if (inFlightAdmissions === 0) {
+            for (const resolveWaiter of admissionWaiters) {
+                resolveWaiter();
+            }
+            admissionWaiters.clear();
+        }
+    }
+};
+const waitForCheckAdmissions = () => inFlightAdmissions === 0
+    ? Promise.resolve()
+    : new Promise(resolveWaiter => admissionWaiters.add(resolveWaiter));
+export const closeCheckRunnerAdmissions = () => {
+    acceptingAdmissions = false;
+};
+export const openCheckRunnerAdmissions = () => {
+    acceptingAdmissions = true;
+};
+const resolveShell = () => process.env.SHELL?.trim() || '/bin/zsh';
+const createShellEnvironment = () => {
+    const env = { ...process.env };
+    delete env.npm_config_prefix;
+    delete env.PORT;
+    return env;
+};
+const identitiesMatch = (left, right) => (left.state === 'available'
+    && right.state === 'available'
+    && left.fingerprint === right.fingerprint);
+const isPathInside = (root, candidate) => {
+    const pathFromRoot = relative(root, candidate).replaceAll('\\', '/');
+    return pathFromRoot === '' || (!pathFromRoot.startsWith('..') && !pathFromRoot.includes('/../'));
+};
+const resolveDefinitionCwd = (workspaceCwd, definition) => {
+    const workspacePath = resolveCommandWorkspacePath(definition);
+    if (!workspacePath) {
+        return workspaceCwd;
+    }
+    const resolvedPath = resolve(workspaceCwd, workspacePath);
+    if (!isPathInside(workspaceCwd, resolvedPath)) {
+        throw new Error(`Check definition ${definition.id} working directory must stay inside the target workspace`);
+    }
+    let realWorkspacePath;
+    let realResolvedPath;
+    try {
+        realWorkspacePath = realpathSync(workspaceCwd);
+        realResolvedPath = realpathSync(resolvedPath);
+    }
+    catch {
+        throw new Error(`Check definition ${definition.id} working directory is unavailable`);
+    }
+    if (!isPathInside(realWorkspacePath, realResolvedPath)) {
+        throw new Error(`Check definition ${definition.id} working directory must stay inside the target workspace`);
+    }
+    if (!lstatSync(realResolvedPath).isDirectory()) {
+        throw new Error(`Check definition ${definition.id} working directory is unavailable`);
+    }
+    return definition.cwd?.trim() ? realResolvedPath : workspaceCwd;
+};
+const validateDefinitions = (selectedDefinitions, availableDefinitions) => {
+    if (selectedDefinitions.length === 0) {
+        throw new Error('At least one check definition must be selected');
+    }
+    const availableById = new Map(availableDefinitions.map(definition => [definition.id, definition]));
+    const selectedIds = new Set();
+    for (const definition of selectedDefinitions) {
+        if (selectedIds.has(definition.id)) {
+            throw new Error(`Check definition ${definition.id} was selected more than once`);
+        }
+        selectedIds.add(definition.id);
+        const available = availableById.get(definition.id);
+        if (!available || getCheckDefinitionFingerprint(available) !== getCheckDefinitionFingerprint(definition)) {
+            throw new Error(`Check definition ${definition.id} is stale or unavailable`);
+        }
+    }
+};
+const emitOutput = (entry, chunk) => {
+    const payload = appendCheckAttemptOutput({
+        projectId: entry.projectId,
+        runId: entry.attemptRunId,
+        chunk,
+    });
+    try {
+        entry.emitter?.emit('check-attempt:output', payload);
+    }
+    catch {
+        // Output remains durable when best-effort realtime delivery fails.
+    }
+};
+const isSqliteLockError = (error) => {
+    const code = error?.code;
+    return code === 'SQLITE_BUSY'
+        || code === 'SQLITE_LOCKED'
+        || (typeof code === 'string' && (code.startsWith('SQLITE_BUSY_') || code.startsWith('SQLITE_LOCKED_')));
+};
+const wait = (delayMs) => new Promise(resolveWaiter => setTimeout(resolveWaiter, delayMs));
+const persistOutput = async (record, chunk) => {
+    for (let attempt = 0;; attempt += 1) {
+        try {
+            emitOutput(record, chunk);
+            return;
+        }
+        catch (error) {
+            const retryDelay = OUTPUT_PERSIST_RETRY_DELAYS_MS[attempt];
+            if (!isSqliteLockError(error) || retryDelay === undefined) {
+                throw error;
+            }
+            await wait(retryDelay);
+        }
+    }
+};
+const takeUtf8Prefix = (value, byteLimit) => {
+    if (Buffer.byteLength(value, 'utf8') <= byteLimit) {
+        return value.length;
+    }
+    let low = 0;
+    let high = value.length;
+    while (low < high) {
+        const midpoint = Math.ceil((low + high) / 2);
+        if (Buffer.byteLength(value.slice(0, midpoint), 'utf8') <= byteLimit) {
+            low = midpoint;
+        }
+        else {
+            high = midpoint - 1;
+        }
+    }
+    let end = low;
+    if (end > 0 && end < value.length) {
+        const previous = value.charCodeAt(end - 1);
+        const next = value.charCodeAt(end);
+        if (previous >= 0xd800 && previous <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) {
+            end -= 1;
+        }
+    }
+    return end;
+};
+const flushOutput = (record) => {
+    if (record.flushTimer) {
+        clearTimeout(record.flushTimer);
+        record.flushTimer = null;
+    }
+    if (record.outputFlushPromise) {
+        return record.outputFlushPromise;
+    }
+    const flushPromise = (async () => {
+        while (record.pendingOutput) {
+            const end = takeUtf8Prefix(record.pendingOutput, OUTPUT_FLUSH_BYTES);
+            const chunk = record.pendingOutput.slice(0, end);
+            await persistOutput(record, chunk);
+            record.pendingOutput = record.pendingOutput.slice(end);
+            record.pendingOutputBytes -= Buffer.byteLength(chunk, 'utf8');
+        }
+        record.pendingOutputBytes = 0;
+    })();
+    record.outputFlushPromise = flushPromise;
+    void flushPromise.then(() => {
+        if (record.outputFlushPromise === flushPromise) {
+            record.outputFlushPromise = null;
+        }
+    }, () => {
+        if (record.outputFlushPromise === flushPromise) {
+            record.outputFlushPromise = null;
+        }
+    });
+    return flushPromise;
+};
+const requestOutputFlush = (record) => {
+    void flushOutput(record).catch(error => {
+        console.error('Check attempt output persistence failed', {
+            attempt_run_id: record.attemptRunId,
+            error: error instanceof Error ? error.message : String(error),
+        });
+    });
+};
+const appendProcessOutput = (record, text) => {
+    if (!text) {
+        return;
+    }
+    record.pendingOutput += text;
+    record.pendingOutputBytes += Buffer.byteLength(text, 'utf8');
+    if (record.pendingOutputBytes >= OUTPUT_FLUSH_BYTES) {
+        requestOutputFlush(record);
+        return;
+    }
+    if (!record.flushTimer) {
+        record.flushTimer = setTimeout(() => requestOutputFlush(record), OUTPUT_FLUSH_DELAY_MS);
+    }
+};
+const signalProcessGroup = (child, signal) => {
+    if (child.pid && process.platform !== 'win32') {
+        try {
+            process.kill(-child.pid, signal);
+            return;
+        }
+        catch (error) {
+            if (error.code === 'ESRCH') {
+                return;
+            }
+        }
+    }
+    try {
+        child.kill(signal);
+    }
+    catch {
+        // The process may have exited between the liveness check and the signal.
+    }
+};
+const terminateProcess = (record) => {
+    signalProcessGroup(record.child, 'SIGTERM');
+    if (!record.killTimer) {
+        record.killTimer = setTimeout(() => signalProcessGroup(record.child, 'SIGKILL'), STOP_GRACE_MS);
+    }
+};
+const notifyBatchWaiters = (projectId, batchRunId) => {
+    const details = getCheckBatchDetails(projectId, batchRunId);
+    if (!details || !['completed', 'failed', 'cancelled', 'interrupted'].includes(details.execution_status)) {
+        return;
+    }
+    const waiters = batchWaiters.get(batchRunId);
+    batchWaiters.delete(batchRunId);
+    for (const resolveWaiter of waiters ?? []) {
+        resolveWaiter(details);
+    }
+};
+const completeAttempt = async (entry, outcome, finalIdentity, options = {}) => {
+    completeCheckAttempt({
+        projectId: entry.projectId,
+        runId: entry.attemptRunId,
+        outcome,
+        finalWorkspaceIdentity: finalIdentity,
+        exitCode: options.exitCode,
+        exitSignal: options.exitSignal,
+        exitError: options.exitError,
+    }, entry.emitter);
+    aggregateCheckBatch(entry.projectId, entry.batchRunId, entry.emitter);
+    notifyBatchWaiters(entry.projectId, entry.batchRunId);
+};
+const finalizeProcess = (record, code, signal, error) => {
+    if (record.finalizationPromise) {
+        return record.finalizationPromise;
+    }
+    record.finalized = true;
+    if (record.timeoutTimer) {
+        clearTimeout(record.timeoutTimer);
+    }
+    if (record.killTimer) {
+        clearTimeout(record.killTimer);
+    }
+    record.finalizationPromise = (async () => {
+        appendProcessOutput(record, record.stdoutDecoder.end());
+        appendProcessOutput(record, record.stderrDecoder.end());
+        await flushOutput(record);
+        const outcome = record.terminalCause
+            ?? (error ? 'spawn_failure' : code === 0 ? 'passed' : 'command_failure');
+        const finalIdentity = await captureCheckWorkspaceIdentity(record.cwd);
+        await completeAttempt(record, outcome, finalIdentity, {
+            exitCode: code,
+            exitSignal: signal,
+            exitError: error?.message ?? (outcome === 'timeout' ? `Check timed out after ${record.definition.timeout_ms}ms` : null),
+        });
+        processes.delete(record.attemptRunId);
+    })();
+    return record.finalizationPromise;
+};
+const failQueuedBatch = async (batchRunId, outcome, identity) => {
+    const removed = queue.filter(entry => entry.batchRunId === batchRunId);
+    for (let index = queue.length - 1; index >= 0; index -= 1) {
+        if (queue[index]?.batchRunId === batchRunId) {
+            queue.splice(index, 1);
+        }
+    }
+    for (const entry of removed) {
+        await completeAttempt(entry, outcome, identity, {
+            exitError: outcome === 'workspace_changed_before_start'
+                ? 'Workspace identity changed before this check could start'
+                : null,
+        });
+    }
+};
+const runQueuedAttempt = async (entry) => {
+    const identity = await captureCheckWorkspaceIdentity(entry.cwd);
+    const forcedOutcome = forcedBatchOutcomes.get(entry.batchRunId);
+    if (forcedOutcome) {
+        await completeAttempt(entry, forcedOutcome, identity, {
+            exitError: forcedOutcome === 'shutdown_loss'
+                ? 'The server shut down before this check attempt completed'
+                : 'The server restarted before this check attempt completed',
+        });
+        return;
+    }
+    if (stoppedBatches.has(entry.batchRunId)) {
+        await completeAttempt(entry, 'user_stop', identity);
+        return;
+    }
+    if (invalidatedBatches.has(entry.batchRunId) || !identitiesMatch(entry.initialIdentity, identity)) {
+        invalidatedBatches.add(entry.batchRunId);
+        await completeAttempt(entry, 'workspace_changed_before_start', identity, {
+            exitError: 'Workspace identity changed before this check could start',
+        });
+        await failQueuedBatch(entry.batchRunId, 'workspace_changed_before_start', identity);
+        return;
+    }
+    const spawnCwd = resolveDefinitionCwd(entry.cwd, entry.definition);
+    startCheckAttempt({
+        projectId: entry.projectId,
+        runId: entry.attemptRunId,
+    }, entry.emitter);
+    await new Promise((resolveAttempt, rejectAttempt) => {
+        let child;
+        try {
+            child = spawn(resolveShell(), ['-i', '-c', entry.definition.command], {
+                cwd: spawnCwd,
+                detached: process.platform !== 'win32',
+                env: createShellEnvironment(),
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+        }
+        catch (error) {
+            const spawnError = error instanceof Error ? error : new Error(String(error));
+            void captureCheckWorkspaceIdentity(entry.cwd)
+                .then(finalIdentity => completeAttempt(entry, 'spawn_failure', finalIdentity, { exitError: spawnError.message }))
+                .then(resolveAttempt, rejectAttempt);
+            return;
+        }
+        const record = {
+            ...entry,
+            child,
+            stdoutDecoder: new StringDecoder('utf8'),
+            stderrDecoder: new StringDecoder('utf8'),
+            pendingOutput: '',
+            pendingOutputBytes: 0,
+            flushTimer: null,
+            outputFlushPromise: null,
+            timeoutTimer: null,
+            killTimer: null,
+            terminalCause: null,
+            spawned: false,
+            finalized: false,
+            finalizationPromise: null,
+        };
+        processes.set(entry.attemptRunId, record);
+        child.once('spawn', () => {
+            record.spawned = true;
+        });
+        child.stdout?.on('data', chunk => appendProcessOutput(record, record.stdoutDecoder.write(Buffer.from(chunk))));
+        child.stderr?.on('data', chunk => appendProcessOutput(record, record.stderrDecoder.write(Buffer.from(chunk))));
+        child.on('error', error => {
+            if (!record.spawned) {
+                record.terminalCause ??= 'spawn_failure';
+                void finalizeProcess(record, null, null, error).then(resolveAttempt, rejectAttempt);
+            }
+        });
+        child.on('close', (code, signal) => {
+            void finalizeProcess(record, code, signal).then(resolveAttempt, rejectAttempt);
+        });
+        record.timeoutTimer = setTimeout(() => {
+            record.terminalCause ??= 'timeout';
+            terminateProcess(record);
+        }, entry.definition.timeout_ms);
+    });
+};
+const pumpScheduler = () => {
+    while (admittedAttempts < MAX_CONCURRENT_CHECK_ATTEMPTS && queue.length > 0) {
+        const entry = queue.shift();
+        if (!entry) {
+            return;
+        }
+        admittedAttempts += 1;
+        admittedEntries.set(entry.attemptRunId, entry);
+        void runQueuedAttempt(entry)
+            .catch(async (error) => {
+            console.error('Check attempt scheduler failure', {
+                attempt_run_id: entry.attemptRunId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            const attempt = listCheckAttemptDetails(entry.projectId, entry.batchRunId)
+                .find(candidate => candidate.execution_run_id === entry.attemptRunId);
+            if (!attempt?.outcome) {
+                const finalIdentity = await captureCheckWorkspaceIdentity(entry.cwd);
+                await completeAttempt(entry, 'spawn_failure', finalIdentity, {
+                    exitError: error instanceof Error ? error.message : String(error),
+                });
+            }
+            else {
+                aggregateCheckBatch(entry.projectId, entry.batchRunId, entry.emitter);
+                notifyBatchWaiters(entry.projectId, entry.batchRunId);
+            }
+            processes.delete(entry.attemptRunId);
+        })
+            .catch(error => {
+            console.error('Check attempt scheduler recovery failed', {
+                attempt_run_id: entry.attemptRunId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+        })
+            .finally(() => {
+            admittedEntries.delete(entry.attemptRunId);
+            admittedAttempts -= 1;
+            pumpScheduler();
+        });
+    }
+};
+const enqueueCheckAttempts = (input) => {
+    queue.push(...input.attempts.map(attempt => ({
+        projectId: input.projectId,
+        batchRunId: input.batchRunId,
+        attemptRunId: attempt.runId,
+        cwd: input.cwd,
+        definition: attempt.definition,
+        initialIdentity: input.initialIdentity,
+        emitter: input.emitter,
+    })));
+    pumpScheduler();
+};
+export const startCheckBatch = async (input, emitter) => withCheckAdmission(async () => {
+    const targetBundleId = input.targetKind === 'bundle' ? input.targetBundleId ?? null : null;
+    if (input.targetKind === 'bundle' && !targetBundleId) {
+        throw new Error('Bundle check batches require a target bundle ID');
+    }
+    if (!existsSync(input.cwd) || !lstatSync(input.cwd).isDirectory()) {
+        throw new Error('Check target workspace is unavailable');
+    }
+    validateDefinitions(input.selectedDefinitions, input.availableDefinitions ?? input.selectedDefinitions);
+    for (const definition of input.selectedDefinitions) {
+        resolveDefinitionCwd(input.cwd, definition);
+    }
+    const conflict = findActiveCheckBatchForTarget(input.projectId, input.targetKind, targetBundleId);
+    if (conflict) {
+        throw new Error(`Check batch ${conflict} is already active for this target`);
+    }
+    const initialIdentity = await captureCheckWorkspaceIdentity(input.cwd);
+    if (initialIdentity.state !== 'available') {
+        throw new Error(`Check workspace identity is unavailable: ${initialIdentity.message}`);
+    }
+    const currentDefinitions = getProjectCheckDefinitions(getProjectById(input.projectId));
+    validateDefinitions(input.selectedDefinitions, currentDefinitions);
+    for (const definition of input.selectedDefinitions) {
+        resolveDefinitionCwd(input.cwd, definition);
+    }
+    assertAcceptingAdmissions();
+    const batchRunId = randomUUID();
+    const attempts = input.selectedDefinitions.map(definition => ({ runId: randomUUID(), definition }));
+    const checkTarget = input.targetKind === 'bundle'
+        ? { targetKind: 'bundle', targetBundleId: targetBundleId }
+        : { targetKind: 'repo_root', targetBundleId: null };
+    const scopeLinks = targetBundleId
+        ? [{ entityType: 'bundle', entityId: targetBundleId, relationship: 'context' }]
+        : [];
+    withExecutionRunTransaction(transaction => {
+        const transactionConflict = findActiveCheckBatchForTarget(input.projectId, input.targetKind, targetBundleId);
+        if (transactionConflict) {
+            throw new Error(`Check batch ${transactionConflict} is already active for this target`);
+        }
+        transaction.createExecutionRun({
+            id: batchRunId,
+            projectId: input.projectId,
+            kind: 'check_batch',
+            initiatedBy: input.automationParentRunId ? 'automation' : 'user',
+            parentRunId: input.automationParentRunId ?? null,
+            domainStatus: 'queued',
+            sourceReference: input.sourceReference ?? null,
+            links: scopeLinks,
+            eventType: 'check_batch_created',
+            checkBatchDetails: {
+                ...checkTarget,
+                targetLabel: input.targetLabel,
+                selectedDefinitions: input.selectedDefinitions,
+                initialWorkspaceIdentity: initialIdentity,
+                rerunOfBatchRunId: input.rerunOfBatchRunId,
+            },
+        });
+        for (const attempt of attempts) {
+            transaction.createExecutionRun({
+                id: attempt.runId,
+                projectId: input.projectId,
+                kind: 'check_attempt',
+                parentRunId: batchRunId,
+                sourceReference: { kind: 'project', id: input.projectId },
+                links: scopeLinks,
+                domainStatus: 'queued',
+                eventType: 'check_attempt_created',
+                checkAttemptDetails: {
+                    batchRunId,
+                    definition: attempt.definition,
+                    initialWorkspaceIdentity: initialIdentity,
+                    rerunOfAttemptRunId: input.rerunAttemptRunIds?.[attempt.definition.id] ?? null,
+                },
+            });
+        }
+        transaction.transitionExecutionRun({
+            projectId: input.projectId,
+            runId: batchRunId,
+            status: 'running',
+            domainStatus: 'running',
+            eventType: 'check_batch_started',
+        });
+    }, emitter);
+    enqueueCheckAttempts({
+        projectId: input.projectId,
+        batchRunId,
+        cwd: input.cwd,
+        attempts,
+        initialIdentity,
+        emitter,
+    });
+    return getCheckBatchDetails(input.projectId, batchRunId);
+});
+export const rerunCheckBatchAndSchedule = async (input, emitter) => withCheckAdmission(async () => {
+    if (!existsSync(input.cwd) || !lstatSync(input.cwd).isDirectory()) {
+        throw new Error('Check target workspace is unavailable');
+    }
+    const sourceAttempts = listCheckAttemptDetails(input.projectId, input.sourceBatchRunId);
+    const selectedDefinitionIds = input.selection.mode === 'all'
+        ? sourceAttempts.map(attempt => attempt.definition_id)
+        : input.selection.mode === 'failed'
+            ? sourceAttempts.filter(attempt => attempt.outcome !== 'passed').map(attempt => attempt.definition_id)
+            : input.selection.definitionIds;
+    const selectedDefinitionIdSet = new Set(selectedDefinitionIds);
+    for (const definition of input.availableDefinitions.filter(candidate => selectedDefinitionIdSet.has(candidate.id))) {
+        resolveDefinitionCwd(input.cwd, definition);
+    }
+    const initialIdentity = await captureCheckWorkspaceIdentity(input.cwd);
+    if (initialIdentity.state !== 'available') {
+        throw new Error(`Check workspace identity is unavailable: ${initialIdentity.message}`);
+    }
+    const currentDefinitions = getProjectCheckDefinitions(getProjectById(input.projectId));
+    const currentById = new Map(currentDefinitions.map(definition => [definition.id, definition]));
+    for (const definitionId of selectedDefinitionIds) {
+        const definition = currentById.get(definitionId);
+        if (definition) {
+            resolveDefinitionCwd(input.cwd, definition);
+        }
+    }
+    assertAcceptingAdmissions();
+    const created = withExecutionRunTransaction(transaction => rerunCheckBatch({
+        transaction,
+        projectId: input.projectId,
+        sourceBatchRunId: input.sourceBatchRunId,
+        selection: input.selection,
+        availableDefinitions: currentDefinitions,
+        initialWorkspaceIdentity: initialIdentity,
+    }), emitter);
+    enqueueCheckAttempts({
+        projectId: input.projectId,
+        batchRunId: created.batchRunId,
+        cwd: input.cwd,
+        attempts: created.attempts,
+        initialIdentity,
+        emitter,
+    });
+    return {
+        batch: created.batch,
+        attempts: listCheckAttemptDetails(input.projectId, created.batchRunId),
+    };
+});
+export const stopCheckBatch = async (projectId, batchRunId, emitter) => {
+    const current = getCheckBatchDetails(projectId, batchRunId);
+    if (!current) {
+        throw new Error(`Check batch ${batchRunId} was not found in project ${projectId}`);
+    }
+    if (['completed', 'failed', 'cancelled', 'interrupted'].includes(current.execution_status)) {
+        return current;
+    }
+    const batch = setCheckBatchStopRequested(projectId, batchRunId);
+    stoppedBatches.add(batchRunId);
+    const identity = await captureCheckWorkspaceIdentity([...processes.values()].find(record => record.batchRunId === batchRunId)?.cwd
+        ?? queue.find(entry => entry.batchRunId === batchRunId)?.cwd
+        ?? '');
+    await failQueuedBatch(batchRunId, 'user_stop', identity);
+    for (const record of processes.values()) {
+        if (record.batchRunId === batchRunId && !record.finalized) {
+            record.terminalCause ??= 'user_stop';
+            terminateProcess(record);
+        }
+    }
+    aggregateCheckBatch(projectId, batchRunId, emitter);
+    notifyBatchWaiters(projectId, batchRunId);
+    return getCheckBatchDetails(projectId, batchRunId);
+};
+export const waitForCheckBatch = (projectId, batchRunId) => {
+    const details = getCheckBatchDetails(projectId, batchRunId);
+    if (!details) {
+        return Promise.reject(new Error(`Check batch ${batchRunId} was not found in project ${projectId}`));
+    }
+    if (['completed', 'failed', 'cancelled', 'interrupted'].includes(details.execution_status)) {
+        return Promise.resolve(details);
+    }
+    return new Promise(resolveWaiter => {
+        const waiters = batchWaiters.get(batchRunId) ?? new Set();
+        waiters.add(resolveWaiter);
+        batchWaiters.set(batchRunId, waiters);
+    });
+};
+export const getCheckRunnerState = () => ({
+    active_attempt_count: admittedAttempts,
+    queued_attempt_count: queue.length,
+    process_attempt_ids: [...processes.keys()],
+});
+export const shutdownCheckRunner = async () => {
+    closeCheckRunnerAdmissions();
+    await waitForCheckAdmissions();
+    const batchIds = new Set([
+        ...queue.map(entry => entry.batchRunId),
+        ...[...admittedEntries.values()].map(entry => entry.batchRunId),
+        ...[...processes.values()].map(record => record.batchRunId),
+    ]);
+    try {
+        for (const batchRunId of batchIds) {
+            forcedBatchOutcomes.set(batchRunId, 'shutdown_loss');
+        }
+        const queuedEntries = queue.splice(0);
+        for (const entry of queuedEntries) {
+            const finalIdentity = await captureCheckWorkspaceIdentity(entry.cwd);
+            await completeAttempt(entry, 'shutdown_loss', finalIdentity, {
+                exitError: 'The server shut down before this check attempt started',
+            });
+        }
+        for (const record of processes.values()) {
+            record.terminalCause ??= 'shutdown_loss';
+            terminateProcess(record);
+        }
+        await Promise.all([...batchIds].map(batchRunId => {
+            const projectId = [...admittedEntries.values()].find(entry => entry.batchRunId === batchRunId)?.projectId
+                ?? [...processes.values()].find(record => record.batchRunId === batchRunId)?.projectId
+                ?? queuedEntries.find(entry => entry.batchRunId === batchRunId)?.projectId;
+            return projectId ? waitForCheckBatch(projectId, batchRunId) : Promise.resolve(null);
+        }));
+    }
+    finally {
+        for (const batchRunId of batchIds) {
+            forcedBatchOutcomes.delete(batchRunId);
+        }
+    }
+};
+export const getCheckBatchAttempts = (projectId, batchRunId) => (listCheckAttemptDetails(projectId, batchRunId));
+//# sourceMappingURL=checkRunner.js.map