@straiffi/archon 1.8.20 → 1.8.22

package/dist/server/lib/checkRunner.js

@@ -1,15 +1,20 @@
 import { randomUUID } from 'crypto';
 import { spawn } from 'child_process';
+import db from '../db.js';
 import { existsSync, lstatSync, realpathSync } from 'fs';
-import { relative, resolve } from 'path';
+import { basename, relative, resolve } from 'path';
 import { StringDecoder } from 'string_decoder';
-import { findActiveCheckBatchForTarget, findActiveCheckAutoFixForTarget, canAutoFixStartCheckBatch, getCheckBatchDetails, inspectCheckBatchAggregation, listCheckAttemptDetails, rerunCheckBatch, setCheckBatchStopRequested, } from './checkRuns.js';
+import { findActiveCheckBatchForTarget, findActiveCheckAutoFixForTarget, canAutoFixStartCheckBatch, getCheckBatchDetails, getCheckAttemptDetails, inspectCheckBatchAggregation, listCheckAttemptDetails, rerunCheckBatch, setCheckBatchStopRequested, } from './checkRuns.js';
 import { getCheckDefinitionFingerprint } from './checkDefinitions.js';
 import { captureCheckWorkspaceIdentity } from './checkWorkspaceIdentity.js';
 import { getProjectById, getProjectCheckDefinitions } from './projects.js';
 import { aggregateCheckBatch, completeCheckAttempt, startCheckAttempt, withExecutionRunTransaction, } from './executionRuns.js';
 import { resolveCommandWorkspacePath } from './testCommandRunner.js';
 import { createWorkspaceProcessEnvironment } from './workspaceProcessEnvironment.js';
+import { createGovernedHostRuntimeEnvironment } from './workspaceProcessEnvironment.js';
+import { cancelApprovalRequest, consumeApprovalRequest, createApprovalRequestInTransaction, emitApprovalRequestCreated, getApprovalRequest, invalidateApprovalRequest, registerApprovalRequestTerminalHandler, } from './approvalRequests.js';
+import { fingerprintConsequentialAction } from './consequentialActions.js';
+import { buildProductionCheckAction, PRODUCTION_CHECK_APPROVAL_TTL_MS } from './productionCheckApprovals.js';
 import { appendCheckAttemptEvidence, appendCheckDiagnostics, completeCheckDiagnosticParser, initializeCheckDiagnosticParser, } from './checkDiagnostics.js';
 import { createCheckDiagnosticParserSession } from './checkDiagnosticParsers.js';
 const MAX_CONCURRENT_CHECK_ATTEMPTS = 2;
@@ -35,7 +40,7 @@ const processes = new Map();
 const admittedEntries = new Map();
 const invalidatedBatches = new Set();
 const stoppedBatches = new Set();
-const forcedBatchOutcomes = new Map();
+const forcedAttemptOutcomes = new Map();
 const batchWaiters = new Map();
 let admittedAttempts = 0;
 let acceptingAdmissions = true;
@@ -72,8 +77,26 @@ export const openCheckRunnerAdmissions = () => {
     acceptingAdmissions = true;
 };
 const resolveShell = () => process.env.SHELL?.trim() || '/bin/zsh';
-const createShellEnvironment = (cwd) => {
-    const env = createWorkspaceProcessEnvironment(cwd);
+export const getCheckShellArguments = (shell, command, governedProductionEnvironment = false) => {
+    if (!governedProductionEnvironment) {
+        return ['-i', '-c', command];
+    }
+    const shellName = basename(shell);
+    if (shellName === 'zsh') {
+        return ['-f', '-c', command];
+    }
+    if (shellName === 'bash') {
+        return ['--noprofile', '--norc', '-c', command];
+    }
+    if (shellName === 'fish') {
+        return ['--no-config', '-c', command];
+    }
+    return ['-c', command];
+};
+const createShellEnvironment = (cwd, definition, governedProductionEnvironment = false) => {
+    const env = definition.consequence === 'production' && governedProductionEnvironment
+        ? createGovernedHostRuntimeEnvironment(cwd).variables
+        : createWorkspaceProcessEnvironment(cwd);
     delete env.PORT;
     return env;
 };
@@ -84,7 +107,7 @@ const isPathInside = (root, candidate) => {
     const pathFromRoot = relative(root, candidate).replaceAll('\\', '/');
     return pathFromRoot === '' || (!pathFromRoot.startsWith('..') && !pathFromRoot.includes('/../'));
 };
-const resolveDefinitionCwd = (workspaceCwd, definition) => {
+export const resolveDefinitionCwd = (workspaceCwd, definition) => {
     const workspacePath = resolveCommandWorkspacePath(definition);
     if (!workspacePath) {
         return workspaceCwd;
@@ -300,6 +323,26 @@ const terminateProcess = (record) => {
         record.killTimer = setTimeout(() => signalProcessGroup(record.child, 'SIGKILL'), STOP_GRACE_MS);
     }
 };
+const waitForProcessFinalization = (record) => {
+    if (record.finalizationPromise) {
+        return record.finalizationPromise;
+    }
+    return new Promise((resolve, reject) => {
+        const waitForFinalization = () => {
+            const finalizationPromise = record.finalizationPromise;
+            if (!finalizationPromise) {
+                reject(new Error(`Check attempt ${record.attemptRunId} closed without starting finalization`));
+                return;
+            }
+            void finalizationPromise.then(resolve, reject);
+        };
+        record.child.once('close', waitForFinalization);
+        if (record.finalizationPromise) {
+            record.child.off('close', waitForFinalization);
+            void record.finalizationPromise.then(resolve, reject);
+        }
+    });
+};
 const notifyBatchWaiters = (projectId, batchRunId) => {
     const details = getCheckBatchDetails(projectId, batchRunId);
     if (!details || !['completed', 'failed', 'cancelled', 'interrupted'].includes(details.execution_status)) {
@@ -418,7 +461,7 @@ const failQueuedBatch = async (batchRunId, outcome, identity) => {
 };
 const runQueuedAttempt = async (entry) => {
     const identity = await captureCheckWorkspaceIdentity(entry.cwd);
-    const forcedOutcome = forcedBatchOutcomes.get(entry.batchRunId);
+    const forcedOutcome = forcedAttemptOutcomes.get(entry.attemptRunId);
     if (forcedOutcome) {
         await completeAttempt(entry, forcedOutcome, identity, {
             exitError: forcedOutcome === 'shutdown_loss'
@@ -440,6 +483,30 @@ const runQueuedAttempt = async (entry) => {
         return;
     }
     const spawnCwd = resolveDefinitionCwd(entry.cwd, entry.definition);
+    if (entry.approvalRequestId) {
+        const request = getApprovalRequest(entry.projectId, entry.approvalRequestId, { emitter: entry.emitter });
+        const batch = getCheckBatchDetails(entry.projectId, entry.batchRunId);
+        if (!request || request.status !== 'consumed' || !batch) {
+            throw new Error('Production check approval is no longer consumable');
+        }
+        const currentDefinitions = getProjectCheckDefinitions(getProjectById(entry.projectId));
+        const currentDefinition = currentDefinitions.find(definition => definition.id === entry.definition.id);
+        if (!currentDefinition || getCheckDefinitionFingerprint(currentDefinition) !== getCheckDefinitionFingerprint(entry.definition)) {
+            throw new Error('Production check definition changed before spawn');
+        }
+        const current = buildProductionCheckAction({
+            projectId: entry.projectId,
+            workspaceCwd: entry.cwd,
+            effectiveCwd: spawnCwd,
+            definition: currentDefinition,
+            batch,
+            workspaceIdentity: identity,
+            expiresAt: request.expires_at,
+        });
+        if (fingerprintConsequentialAction(current.action) !== request.action_fingerprint) {
+            throw new Error('Production check action changed before spawn');
+        }
+    }
     initializeCheckDiagnosticParser(entry.projectId, entry.attemptRunId);
     startCheckAttempt({
         projectId: entry.projectId,
@@ -448,10 +515,11 @@ const runQueuedAttempt = async (entry) => {
     await new Promise((resolveAttempt, rejectAttempt) => {
         let child;
         try {
-            child = spawn(resolveShell(), ['-i', '-c', entry.definition.command], {
+            const shell = resolveShell();
+            child = spawn(shell, getCheckShellArguments(shell, entry.definition.command, entry.governedProductionEnvironment), {
                 cwd: spawnCwd,
                 detached: process.platform !== 'win32',
-                env: createShellEnvironment(entry.cwd),
+                env: createShellEnvironment(entry.cwd, entry.definition, entry.governedProductionEnvironment),
                 stdio: ['ignore', 'pipe', 'pipe'],
             });
         }
@@ -559,6 +627,8 @@ const enqueueCheckAttempts = (input) => {
         definition: attempt.definition,
         initialIdentity: input.initialIdentity,
         emitter: input.emitter,
+        approvalRequestId: attempt.approvalRequestId,
+        governedProductionEnvironment: attempt.governedProductionEnvironment,
     })));
     pumpScheduler();
 };
@@ -608,6 +678,7 @@ export const startCheckBatch = async (input, emitter) => withCheckAdmission(asyn
     assertAcceptingAdmissions();
     const batchRunId = randomUUID();
     const attempts = input.selectedDefinitions.map(definition => ({ runId: randomUUID(), definition }));
+    const createdApprovals = [];
     const checkTarget = input.targetKind === 'bundle'
         ? { targetKind: 'bundle', targetBundleId: targetBundleId }
         : { targetKind: 'repo_root', targetBundleId: null };
@@ -665,6 +736,26 @@ export const startCheckBatch = async (input, emitter) => withCheckAdmission(asyn
                 },
             });
         }
+        const batch = getCheckBatchDetails(input.projectId, batchRunId);
+        for (const attempt of attempts.filter(candidate => candidate.definition.consequence === 'production')) {
+            const expiresAt = new Date(Date.now() + PRODUCTION_CHECK_APPROVAL_TTL_MS).toISOString();
+            const approval = buildProductionCheckAction({
+                projectId: input.projectId,
+                workspaceCwd: input.cwd,
+                effectiveCwd: resolveDefinitionCwd(input.cwd, attempt.definition),
+                definition: attempt.definition,
+                batch,
+                workspaceIdentity: initialIdentity,
+                expiresAt,
+            });
+            createdApprovals.push(createApprovalRequestInTransaction({
+                projectId: input.projectId,
+                executionRunId: attempt.runId,
+                action: approval.action,
+                authority: approval.authority,
+                expiresAt,
+            }, transaction));
+        }
         input.onBatchCreated?.(batchRunId);
         transaction.transitionExecutionRun({
             projectId: input.projectId,
@@ -674,11 +765,15 @@ export const startCheckBatch = async (input, emitter) => withCheckAdmission(asyn
             eventType: 'check_batch_started',
         });
     }, emitter);
+    for (const approval of createdApprovals) {
+        emitApprovalRequestCreated(emitter, approval);
+    }
+    const ordinaryAttempts = attempts.filter(attempt => attempt.definition.consequence !== 'production');
     enqueueCheckAttempts({
         projectId: input.projectId,
         batchRunId,
         cwd: input.cwd,
-        attempts,
+        attempts: ordinaryAttempts,
         initialIdentity,
         emitter,
     });
@@ -711,19 +806,47 @@ export const rerunCheckBatchAndSchedule = async (input, emitter) => withCheckAdm
         }
     }
     assertAcceptingAdmissions();
-    const created = withExecutionRunTransaction(transaction => rerunCheckBatch({
-        transaction,
-        projectId: input.projectId,
-        sourceBatchRunId: input.sourceBatchRunId,
-        selection: input.selection,
-        availableDefinitions: currentDefinitions,
-        initialWorkspaceIdentity: initialIdentity,
-    }), emitter);
+    const createdApprovals = [];
+    const created = withExecutionRunTransaction(transaction => {
+        const result = rerunCheckBatch({
+            transaction,
+            projectId: input.projectId,
+            sourceBatchRunId: input.sourceBatchRunId,
+            selection: input.selection,
+            availableDefinitions: currentDefinitions,
+            initialWorkspaceIdentity: initialIdentity,
+        });
+        const batch = getCheckBatchDetails(input.projectId, result.batchRunId);
+        for (const attempt of result.attempts.filter(candidate => candidate.definition.consequence === 'production')) {
+            const expiresAt = new Date(Date.now() + PRODUCTION_CHECK_APPROVAL_TTL_MS).toISOString();
+            const approval = buildProductionCheckAction({
+                projectId: input.projectId,
+                workspaceCwd: input.cwd,
+                effectiveCwd: resolveDefinitionCwd(input.cwd, attempt.definition),
+                definition: attempt.definition,
+                batch,
+                workspaceIdentity: initialIdentity,
+                expiresAt,
+            });
+            createdApprovals.push(createApprovalRequestInTransaction({
+                projectId: input.projectId,
+                executionRunId: attempt.runId,
+                action: approval.action,
+                authority: approval.authority,
+                expiresAt,
+            }, transaction));
+        }
+        return result;
+    }, emitter);
+    for (const approval of createdApprovals) {
+        emitApprovalRequestCreated(emitter, approval);
+    }
+    const ordinaryAttempts = created.attempts.filter(attempt => attempt.definition.consequence !== 'production');
     enqueueCheckAttempts({
         projectId: input.projectId,
         batchRunId: created.batchRunId,
         cwd: input.cwd,
-        attempts: created.attempts,
+        attempts: ordinaryAttempts,
         initialIdentity,
         emitter,
     });
@@ -742,6 +865,21 @@ export const stopCheckBatch = async (projectId, batchRunId, emitter) => {
     }
     const batch = setCheckBatchStopRequested(projectId, batchRunId);
     stoppedBatches.add(batchRunId);
+    const approvalRows = db.prepare(`
+    SELECT request.id
+    FROM approval_requests request
+    JOIN check_attempt_details attempt ON attempt.execution_run_id = request.execution_run_id
+    WHERE request.project_id = ? AND attempt.batch_run_id = ?
+      AND request.status IN ('pending', 'approved')
+  `).all(projectId, batchRunId);
+    for (const row of approvalRows) {
+        cancelApprovalRequest({
+            projectId,
+            requestId: row.id,
+            reason: 'check_batch_stopped',
+            emitter,
+        });
+    }
     const identity = await captureCheckWorkspaceIdentity([...processes.values()].find(record => record.batchRunId === batchRunId)?.cwd
         ?? queue.find(entry => entry.batchRunId === batchRunId)?.cwd
         ?? '');
@@ -775,30 +913,169 @@ export const getCheckRunnerState = () => ({
     queued_attempt_count: queue.length,
     process_attempt_ids: [...processes.keys()],
 });
+export const resumeProductionCheckApproval = async (projectId, requestId, emitter) => withCheckAdmission(async () => {
+    const request = getApprovalRequest(projectId, requestId, { emitter });
+    if (!request || request.status !== 'approved' || request.action.kind !== 'custom_check') {
+        return false;
+    }
+    const attempt = getCheckAttemptDetails(projectId, request.execution_run_id);
+    if (!attempt || attempt.outcome) {
+        return false;
+    }
+    const batch = getCheckBatchDetails(projectId, attempt.batch_run_id);
+    if (!batch || batch.stop_requested || ['completed', 'failed', 'cancelled', 'interrupted'].includes(batch.execution_status)) {
+        invalidateApprovalRequest({ projectId, requestId, reason: 'check_batch_no_longer_active', emitter });
+        return false;
+    }
+    const workspaceCwd = request.action.workspace_git.workspace_path;
+    let currentIdentity;
+    let currentDefinition;
+    try {
+        currentIdentity = await captureCheckWorkspaceIdentity(workspaceCwd);
+        currentDefinition = getProjectCheckDefinitions(getProjectById(projectId))
+            .find(definition => definition.id === attempt.definition_id);
+    }
+    catch {
+        invalidateApprovalRequest({ projectId, requestId, reason: 'production_check_revalidation_failed', emitter });
+        return false;
+    }
+    if (!currentDefinition
+        || currentDefinition.category !== 'custom'
+        || currentDefinition.consequence !== 'production'
+        || getCheckDefinitionFingerprint(currentDefinition) !== attempt.definition_fingerprint) {
+        consumeApprovalRequest({
+            projectId,
+            requestId,
+            currentAction: {
+                ...request.action,
+                target: { ...request.action.target, check_definition_fingerprint: '0'.repeat(64) },
+            },
+            emitter,
+        });
+        return false;
+    }
+    let current;
+    try {
+        current = buildProductionCheckAction({
+            projectId,
+            workspaceCwd,
+            effectiveCwd: resolveDefinitionCwd(workspaceCwd, currentDefinition),
+            definition: currentDefinition,
+            batch,
+            workspaceIdentity: currentIdentity,
+            expiresAt: request.expires_at,
+        });
+    }
+    catch {
+        invalidateApprovalRequest({ projectId, requestId, reason: 'production_check_revalidation_failed', emitter });
+        return false;
+    }
+    const consumed = consumeApprovalRequest({ projectId, requestId, currentAction: current.action, emitter });
+    if (!consumed.ok) {
+        if (consumed.request && ['expired', 'invalidated'].includes(consumed.request.status)) {
+            await finalizeProductionCheckApproval(projectId, requestId, emitter);
+        }
+        return false;
+    }
+    enqueueCheckAttempts({
+        projectId,
+        batchRunId: batch.execution_run_id,
+        cwd: workspaceCwd,
+        attempts: [{
+                runId: attempt.execution_run_id,
+                definition: currentDefinition,
+                approvalRequestId: requestId,
+                governedProductionEnvironment: request.authority.selected_profile.compatibility === 'governed',
+            }],
+        initialIdentity: attempt.initial_workspace_identity,
+        emitter,
+    });
+    return true;
+});
+export const finalizeProductionCheckApproval = (projectId, requestId, emitter) => {
+    const request = getApprovalRequest(projectId, requestId, { emitter });
+    if (!request || request.action.kind !== 'custom_check' || !['denied', 'cancelled', 'expired', 'invalidated'].includes(request.status)) {
+        return;
+    }
+    const attempt = getCheckAttemptDetails(projectId, request.execution_run_id);
+    if (!attempt) {
+        return;
+    }
+    aggregateCheckBatch(projectId, attempt.batch_run_id, emitter);
+    notifyBatchWaiters(projectId, attempt.batch_run_id);
+};
+registerApprovalRequestTerminalHandler((request, emitter) => {
+    if (request.action_kind === 'custom_check') {
+        finalizeProductionCheckApproval(request.project_id, request.id, emitter);
+    }
+});
 export const shutdownCheckRunner = async () => {
     closeCheckRunnerAdmissions();
     await waitForCheckAdmissions();
+    const hasApprovalRequests = Boolean(db.prepare(`
+    SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'approval_requests'
+  `).get());
+    const durableAttempts = db.prepare(`
+    SELECT run.project_id, run.id AS attempt_run_id, run.parent_run_id AS batch_run_id,
+      ${hasApprovalRequests ? 'approval.id' : 'NULL'} AS approval_request_id,
+      ${hasApprovalRequests ? 'approval.status' : 'NULL'} AS approval_status
+    FROM execution_runs run
+    ${hasApprovalRequests ? `LEFT JOIN approval_requests approval
+      ON approval.execution_run_id = run.id AND approval.project_id = run.project_id` : ''}
+    WHERE run.kind = 'check_attempt' AND run.status IN ('queued', 'waiting', 'running')
+  `).all();
     const batchIds = new Set([
         ...queue.map(entry => entry.batchRunId),
         ...[...admittedEntries.values()].map(entry => entry.batchRunId),
         ...[...processes.values()].map(record => record.batchRunId),
+        ...durableAttempts.map(attempt => attempt.batch_run_id),
     ]);
+    const approvalHeldBatchIds = new Set(durableAttempts
+        .filter(attempt => attempt.approval_status === 'pending' || attempt.approval_status === 'approved')
+        .map(attempt => attempt.batch_run_id));
+    const activeRecords = [...processes.values()];
     try {
-        for (const batchRunId of batchIds) {
-            forcedBatchOutcomes.set(batchRunId, 'shutdown_loss');
+        for (const entry of admittedEntries.values()) {
+            forcedAttemptOutcomes.set(entry.attemptRunId, entry.approvalRequestId ? 'ambiguous_process_loss' : 'shutdown_loss');
         }
         const queuedEntries = queue.splice(0);
         for (const entry of queuedEntries) {
             const finalIdentity = await captureCheckWorkspaceIdentity(entry.cwd);
-            await completeAttempt(entry, 'shutdown_loss', finalIdentity, {
-                exitError: 'The server shut down before this check attempt started',
+            const outcome = entry.approvalRequestId ? 'ambiguous_process_loss' : 'shutdown_loss';
+            await completeAttempt(entry, outcome, finalIdentity, {
+                exitError: entry.approvalRequestId
+                    ? 'The consumed production command process identity was lost during shutdown'
+                    : 'The server shut down before this check attempt started',
             });
         }
-        for (const record of processes.values()) {
-            record.terminalCause ??= 'shutdown_loss';
+        for (const attempt of durableAttempts) {
+            if (attempt.approval_request_id && (attempt.approval_status === 'pending' || attempt.approval_status === 'approved')) {
+                continue;
+            }
+            if (admittedEntries.has(attempt.attempt_run_id)) {
+                continue;
+            }
+            const details = getCheckAttemptDetails(attempt.project_id, attempt.attempt_run_id);
+            if (details && !details.outcome) {
+                completeCheckAttempt({
+                    projectId: attempt.project_id,
+                    runId: attempt.attempt_run_id,
+                    outcome: attempt.approval_status === 'consumed' ? 'ambiguous_process_loss' : 'shutdown_loss',
+                    finalWorkspaceIdentity: details.initial_workspace_identity,
+                    exitError: attempt.approval_status === 'consumed'
+                        ? 'The consumed production command process identity was lost during shutdown'
+                        : 'The server shut down before this check attempt completed',
+                });
+                aggregateCheckBatch(attempt.project_id, attempt.batch_run_id);
+                notifyBatchWaiters(attempt.project_id, attempt.batch_run_id);
+            }
+        }
+        for (const record of activeRecords) {
+            record.terminalCause ??= record.approvalRequestId ? 'ambiguous_process_loss' : 'shutdown_loss';
             terminateProcess(record);
         }
-        await Promise.all([...batchIds].map(batchRunId => {
+        await Promise.all(activeRecords.map(waitForProcessFinalization));
+        await Promise.all([...batchIds].filter(batchRunId => !approvalHeldBatchIds.has(batchRunId)).map(batchRunId => {
             const projectId = [...admittedEntries.values()].find(entry => entry.batchRunId === batchRunId)?.projectId
                 ?? [...processes.values()].find(record => record.batchRunId === batchRunId)?.projectId
                 ?? queuedEntries.find(entry => entry.batchRunId === batchRunId)?.projectId;
@@ -807,7 +1084,9 @@ export const shutdownCheckRunner = async () => {
     }
     finally {
         for (const batchRunId of batchIds) {
-            forcedBatchOutcomes.delete(batchRunId);
+            for (const attempt of durableAttempts.filter(candidate => candidate.batch_run_id === batchRunId)) {
+                forcedAttemptOutcomes.delete(attempt.attempt_run_id);
+            }
         }
     }
 };