@straiffi/archon 1.8.15 → 1.8.16

package/dist/server/lib/executionEnvironmentReconciliation.js

@@ -0,0 +1,782 @@
+import db from '../db.js';
+import { randomUUID } from 'crypto';
+import { createDockerSandboxAdapter, createDockerSandboxName, createSbxCommandRunner, DOCKER_SANDBOX_PROVIDER, QUALIFIED_DOCKER_SANDBOX_TEMPLATES, DockerSandboxCollisionFailure, DockerSandboxStopVerificationFailure, SbxCommandFailure, } from './dockerSandboxExecutionEnvironment.js';
+import { appendExecutionEnvironmentEvent, claimExecutionEnvironmentsForCleanup, completeExecutionEnvironmentCleanup, failExecutionEnvironmentCleanup, getExecutionEnvironment, releaseExecutionEnvironmentLease, transitionExecutionEnvironment, } from './executionEnvironments.js';
+import { getExecutionRun, transitionExecutionRun } from './executionRuns.js';
+const VALIDATION_CLEANUP_TIMEOUT_MS = 30_000;
+const VALIDATION_CLEANUP_REMEDIATION = 'Verify the persisted sandbox identity and stop it manually before retrying validation.';
+export class ExecutionEnvironmentCleanupError extends Error {
+    constructor(message, options = {}) {
+        super(message, options);
+        this.name = 'ExecutionEnvironmentCleanupError';
+    }
+}
+export const createDefaultDockerSandboxAdapter = () => (createDockerSandboxAdapter(createSbxCommandRunner()));
+export const resolveExpectedDockerSandbox = (environment, requestedAgent) => {
+    if (environment.provider_kind !== DOCKER_SANDBOX_PROVIDER) {
+        throw new Error(`Execution environment ${environment.id} is not a Docker Sandbox environment`);
+    }
+    if (!environment.agent_identity || !environment.template_identity) {
+        throw new Error(`Execution environment ${environment.id} has incomplete persisted sandbox ownership evidence`);
+    }
+    if (requestedAgent && environment.agent_identity !== requestedAgent) {
+        throw new Error(`Execution environment ${environment.id} is not owned by the requested agent template`);
+    }
+    return {
+        name: environment.external_id ?? createDockerSandboxName({
+            projectId: environment.project_id,
+            targetKey: environment.target_key,
+            environmentRevision: environment.environment_revision,
+            validationFingerprint: environment.environment_validation_fingerprint,
+        }),
+        agent: environment.agent_identity,
+        workspacePath: environment.target_workspace_path,
+        template: environment.template_identity,
+    };
+};
+const validationCleanupFailure = (error, timedOut) => {
+    if (timedOut || (error instanceof SbxCommandFailure && error.kind === 'timeout')) {
+        return { code: 'validation_cleanup_stop_timeout', message: 'Docker Sandbox cleanup timed out before stopped state was verified.' };
+    }
+    if (error instanceof DockerSandboxCollisionFailure) {
+        return { code: 'validation_cleanup_identity_mismatch', message: error.message };
+    }
+    if (error instanceof DockerSandboxStopVerificationFailure) {
+        return { code: 'validation_cleanup_unverified_running', message: error.message };
+    }
+    return {
+        code: 'validation_cleanup_provider_failure',
+        message: error instanceof Error ? error.message : 'Docker Sandbox cleanup could not verify stopped state.',
+    };
+};
+export const stopAndRecordEnvironmentValidationCleanup = async (input) => {
+    const expected = resolveExpectedDockerSandbox(input.environment, input.requestedAgent);
+    const controller = new AbortController();
+    let timedOut = false;
+    const timeoutFailure = new Promise((_resolve, reject) => {
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+            reject(new Error('Docker Sandbox cleanup timed out.'));
+        }, VALIDATION_CLEANUP_TIMEOUT_MS);
+        timeout.unref?.();
+        controller.signal.addEventListener('abort', () => clearTimeout(timeout), { once: true });
+    });
+    try {
+        await Promise.race([
+            input.adapter.stop(expected, { signal: controller.signal, timeoutMs: VALIDATION_CLEANUP_TIMEOUT_MS }),
+            timeoutFailure,
+        ]);
+        const current = getExecutionEnvironment(input.environment.project_id, input.environment.id);
+        if (!current || current.lifecycle_state === 'retired' || current.lifecycle_state === 'failed') {
+            return;
+        }
+        if (current.lifecycle_state === 'stopped') {
+            appendExecutionEnvironmentEvent({
+                projectId: current.project_id,
+                environmentId: current.id,
+                eventType: 'environment_validation_cleanup_completed',
+                fromState: 'stopped',
+                toState: 'stopped',
+                message: 'Docker Sandbox stopped state was verified after validation.',
+                metadata: { executionRunId: input.executionRunId, sandboxName: expected.name },
+            });
+            return;
+        }
+        transitionExecutionEnvironment({
+            projectId: current.project_id,
+            environmentId: current.id,
+            state: 'stopped',
+            eventType: 'environment_validation_cleanup_completed',
+            message: 'Docker Sandbox stopped state was verified after validation.',
+            metadata: { executionRunId: input.executionRunId, sandboxName: expected.name },
+        });
+    }
+    catch (error) {
+        const failure = validationCleanupFailure(error, timedOut);
+        const current = getExecutionEnvironment(input.environment.project_id, input.environment.id);
+        if (!current || current.lifecycle_state === 'retired' || current.lifecycle_state === 'failed') {
+            return;
+        }
+        const metadata = {
+            executionRunId: input.executionRunId,
+            sandboxName: expected.name,
+            remediation: VALIDATION_CLEANUP_REMEDIATION,
+        };
+        if (current.lifecycle_state === 'unavailable') {
+            appendExecutionEnvironmentEvent({
+                projectId: current.project_id,
+                environmentId: current.id,
+                eventType: 'environment_validation_cleanup_failed',
+                fromState: 'unavailable',
+                toState: 'unavailable',
+                message: failure.message,
+                metadata: { ...metadata, failureCode: failure.code },
+            });
+            return;
+        }
+        transitionExecutionEnvironment({
+            projectId: current.project_id,
+            environmentId: current.id,
+            state: 'unavailable',
+            eventType: 'environment_validation_cleanup_failed',
+            failure: { code: failure.code, message: failure.message, details: { remediation: VALIDATION_CLEANUP_REMEDIATION } },
+            metadata,
+        });
+    }
+    finally {
+        controller.abort();
+    }
+};
+export const resetExecutionEnvironment = async (input) => {
+    const environment = getExecutionEnvironment(input.projectId, input.environmentId);
+    if (!environment) {
+        throw new ExecutionEnvironmentCleanupError(`Execution environment ${input.environmentId} was not found.`);
+    }
+    const activeBinding = db.prepare(`
+    SELECT execution_run_id
+    FROM execution_environment_run_bindings
+    WHERE project_id = ? AND environment_id = ? AND released_at IS NULL
+    LIMIT 1
+  `).get(input.projectId, input.environmentId);
+    if (activeBinding) {
+        throw new ExecutionEnvironmentCleanupError(`Execution environment ${input.environmentId} is still used by active run ${activeBinding.execution_run_id}.`);
+    }
+    if (environment.lifecycle_state === 'retired') {
+        return environment;
+    }
+    transitionExecutionEnvironment({
+        projectId: input.projectId,
+        environmentId: input.environmentId,
+        state: 'retiring',
+        eventType: 'environment_reset_started',
+        message: 'Docker Sandbox reset started by the user.',
+    });
+    const expected = resolveExpectedDockerSandbox(environment);
+    try {
+        const observed = await input.adapter.inspect(expected.name);
+        if (observed && !inspectionMatches(expected, observed)) {
+            throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} does not match Archon's persisted external identity`);
+        }
+        if (observed?.state === 'running') {
+            await input.adapter.stop(expected);
+        }
+        const afterStop = await input.adapter.inspect(expected.name);
+        if (afterStop && !inspectionMatches(expected, afterStop)) {
+            throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} was replaced during reset`);
+        }
+        if (afterStop) {
+            await input.adapter.remove(expected);
+        }
+        const remaining = await input.adapter.inspect(expected.name);
+        if (remaining) {
+            throw new ExecutionEnvironmentCleanupError(`Sandbox ${expected.name} still exists after reset.`);
+        }
+        transitionExecutionEnvironment({
+            projectId: input.projectId,
+            environmentId: input.environmentId,
+            state: 'cleanup_pending',
+            eventType: 'environment_reset_cleanup_started',
+        });
+        return transitionExecutionEnvironment({
+            projectId: input.projectId,
+            environmentId: input.environmentId,
+            state: 'retired',
+            eventType: 'environment_reset_completed',
+            message: 'Docker Sandbox was removed and verified absent.',
+        });
+    }
+    catch (error) {
+        const current = getExecutionEnvironment(input.projectId, input.environmentId);
+        if (current && (current.lifecycle_state === 'retiring' || current.lifecycle_state === 'cleanup_pending')) {
+            transitionExecutionEnvironment({
+                projectId: input.projectId,
+                environmentId: input.environmentId,
+                state: 'unavailable',
+                eventType: 'environment_reset_failed',
+                failure: {
+                    code: error instanceof DockerSandboxCollisionFailure ? 'identity_mismatch' : 'reset_failed',
+                    message: error instanceof Error ? error.message : 'Docker Sandbox reset failed.',
+                },
+            });
+        }
+        throw error;
+    }
+};
+export const stopAndRecordEnvironmentExecutionCleanup = async (input) => {
+    const expected = resolveExpectedDockerSandbox(input.environment);
+    try {
+        const before = await input.adapter.inspect(expected.name);
+        if (before && !inspectionMatches(expected, before)) {
+            throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} does not match Archon's persisted external identity`);
+        }
+        if (before?.state === 'running') {
+            await input.adapter.stop(expected, { timeoutMs: VALIDATION_CLEANUP_TIMEOUT_MS });
+        }
+        const after = await input.adapter.inspect(expected.name);
+        if (after && !inspectionMatches(expected, after)) {
+            throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} was replaced during execution cleanup`);
+        }
+        if (after?.state === 'running') {
+            throw new DockerSandboxStopVerificationFailure(`Sandbox ${expected.name} remained running`);
+        }
+        const current = getExecutionEnvironment(input.environment.project_id, input.environment.id);
+        if (!current || current.lifecycle_state === 'retired') {
+            return;
+        }
+        const metadata = { executionRunId: input.executionRunId, sandboxName: expected.name };
+        if (!after) {
+            if (current.lifecycle_state === 'unavailable' || current.lifecycle_state === 'failed') {
+                appendExecutionEnvironmentEvent({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    eventType: 'environment_execution_cleanup_absence_verified',
+                    fromState: current.lifecycle_state,
+                    toState: current.lifecycle_state,
+                    message: 'Docker Sandbox absence was verified after execution interruption.',
+                    metadata,
+                });
+            }
+            else {
+                transitionExecutionEnvironment({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    state: 'unavailable',
+                    eventType: 'environment_execution_cleanup_absence_verified',
+                    failure: { code: 'sandbox_missing', message: 'Docker Sandbox absence was verified after execution interruption.' },
+                    metadata,
+                });
+            }
+            return;
+        }
+        if (current.lifecycle_state === 'stopped') {
+            appendExecutionEnvironmentEvent({
+                projectId: current.project_id,
+                environmentId: current.id,
+                eventType: 'environment_execution_cleanup_completed',
+                fromState: 'stopped',
+                toState: 'stopped',
+                message: 'Docker Sandbox stopped state was verified after execution interruption.',
+                metadata,
+            });
+        }
+        else if (current.lifecycle_state !== 'failed' && current.lifecycle_state !== 'unavailable') {
+            transitionExecutionEnvironment({
+                projectId: current.project_id,
+                environmentId: current.id,
+                state: 'stopped',
+                eventType: 'environment_execution_cleanup_completed',
+                message: 'Docker Sandbox stopped state was verified after execution interruption.',
+                metadata,
+            });
+        }
+    }
+    catch (error) {
+        const current = getExecutionEnvironment(input.environment.project_id, input.environment.id);
+        const message = error instanceof Error ? error.message : 'Docker Sandbox execution cleanup could not verify stopped state.';
+        if (current && current.lifecycle_state !== 'retired') {
+            const metadata = { executionRunId: input.executionRunId, sandboxName: expected.name };
+            if (current.lifecycle_state === 'failed' || current.lifecycle_state === 'unavailable') {
+                appendExecutionEnvironmentEvent({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    eventType: 'environment_execution_cleanup_failed',
+                    fromState: current.lifecycle_state,
+                    toState: current.lifecycle_state,
+                    message,
+                    metadata,
+                });
+            }
+            else {
+                transitionExecutionEnvironment({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    state: 'unavailable',
+                    eventType: 'environment_execution_cleanup_failed',
+                    failure: { code: 'execution_termination_unverified', message },
+                    metadata,
+                });
+            }
+        }
+        throw new ExecutionEnvironmentCleanupError(`Unable to prove termination of execution run ${input.executionRunId} in sandbox ${expected.name}`, { cause: error });
+    }
+};
+const inspectionMatches = (expected, observed) => (expected.name === observed.name
+    && expected.agent === observed.agent
+    && expected.workspacePath === observed.workspacePath
+    && expected.template === observed.template);
+const parseProductionAuthority = (binding) => {
+    if (!binding.authority_json) {
+        return null;
+    }
+    try {
+        const authority = JSON.parse(binding.authority_json);
+        if (authority?.schema_version !== 1
+            || authority.provider !== DOCKER_SANDBOX_PROVIDER
+            || authority.environment_id !== binding.environment_id
+            || authority.environment_revision !== binding.environment_revision
+            || authority.environment_validation_fingerprint !== binding.environment_validation_fingerprint
+            || (authority.agent !== 'opencode' && authority.agent !== 'claude')
+            || typeof authority.target?.workspace_path !== 'string'
+            || typeof authority.target?.key !== 'string') {
+            return null;
+        }
+        return authority;
+    }
+    catch {
+        return null;
+    }
+};
+const parseProductionPlacement = (binding) => {
+    if (!binding.placement_json) {
+        return null;
+    }
+    try {
+        const placement = JSON.parse(binding.placement_json);
+        if (placement?.provider !== DOCKER_SANDBOX_PROVIDER
+            || placement.environmentId !== binding.environment_id
+            || placement.environmentRevision !== binding.environment_revision
+            || placement.executionRunId !== binding.execution_run_id
+            || placement.scope?.kind !== 'exact-sandbox'
+            || typeof placement.scope.externalId !== 'string') {
+            return null;
+        }
+        return placement;
+    }
+    catch {
+        return null;
+    }
+};
+const interruptProductionRunAfterCleanupFailure = (binding, message) => {
+    const run = getExecutionRun(binding.project_id, binding.execution_run_id);
+    if (!run || !['queued', 'running', 'waiting'].includes(run.status)) {
+        return;
+    }
+    transitionExecutionRun({
+        projectId: binding.project_id,
+        runId: binding.execution_run_id,
+        status: 'interrupted',
+        domainStatus: 'interrupted',
+        failure: {
+            code: 'production_environment_termination_unverified',
+            message: `Startup could not verify termination of the production Docker Sandbox placement: ${message}`,
+            details: {
+                environmentId: binding.environment_id,
+                environmentRevision: binding.environment_revision,
+                placementRecorded: binding.placement_json !== null,
+            },
+        },
+        eventType: 'production_environment_reconciliation_failed',
+        eventMessage: 'The production environment lease was retained because termination could not be verified.',
+        eventMetadata: {
+            environmentId: binding.environment_id,
+            environmentRevision: binding.environment_revision,
+            leaseRetained: true,
+            placementRecorded: binding.placement_json !== null,
+        },
+    });
+};
+export const reconcileProductionExecutionEnvironmentBindings = async (adapter = createDefaultDockerSandboxAdapter()) => {
+    const bindings = db.prepare(`
+    SELECT *
+    FROM execution_environment_run_bindings
+    WHERE relationship = 'execution' AND released_at IS NULL
+    ORDER BY bound_at ASC, execution_run_id ASC
+  `).all();
+    const summary = { cleaned: 0, failed: 0 };
+    for (const binding of bindings) {
+        const environment = getExecutionEnvironment(binding.project_id, binding.environment_id);
+        const authority = parseProductionAuthority(binding);
+        const parsedPlacement = parseProductionPlacement(binding);
+        try {
+            if (!environment) {
+                throw new Error(`Execution environment ${binding.environment_id} is missing`);
+            }
+            if (!authority) {
+                throw new Error('The immutable execution authority is malformed or does not match its lease');
+            }
+            if (binding.placement_json !== null && !parsedPlacement) {
+                throw new Error('The immutable execution placement is malformed or does not match its lease');
+            }
+            const placement = parsedPlacement
+                && parsedPlacement.providerVersion === authority.provider_version
+                && parsedPlacement.targetKey === authority.target.key
+                ? parsedPlacement
+                : null;
+            if (parsedPlacement && !placement) {
+                throw new Error('The immutable execution placement does not match its authority');
+            }
+            const externalId = placement?.scope.kind === 'exact-sandbox'
+                ? placement.scope.externalId
+                : environment.external_id;
+            if (!externalId) {
+                throw new Error('No exact persisted sandbox identity is available');
+            }
+            if (placement?.scope.kind === 'exact-sandbox' && environment.external_id && externalId !== environment.external_id) {
+                throw new Error('The immutable placement and live sandbox identity disagree');
+            }
+            const expected = {
+                name: externalId,
+                agent: authority.agent,
+                workspacePath: authority.target.workspace_path,
+                template: QUALIFIED_DOCKER_SANDBOX_TEMPLATES[authority.agent],
+            };
+            const before = await adapter.inspect(expected.name);
+            if (before && !inspectionMatches(expected, before)) {
+                throw new DockerSandboxCollisionFailure(expected.name, 'The persisted production sandbox identity does not match provider inspection');
+            }
+            if (before?.state === 'running') {
+                await adapter.stop(expected, { timeoutMs: VALIDATION_CLEANUP_TIMEOUT_MS });
+            }
+            const after = await adapter.inspect(expected.name);
+            if (after && !inspectionMatches(expected, after)) {
+                throw new DockerSandboxCollisionFailure(expected.name, 'The production sandbox identity changed before termination was verified');
+            }
+            if (after?.state === 'running') {
+                throw new DockerSandboxStopVerificationFailure(`Sandbox ${expected.name} remained running`);
+            }
+            const state = after ? 'stopped' : 'absent';
+            const cleanupCompletion = state === 'stopped'
+                ? {
+                    state,
+                    eventType: 'production_environment_reconciliation_completed',
+                    message: 'Production Docker Sandbox termination was positively verified before its lease was released.',
+                    metadata: { executionRunId: binding.execution_run_id, sandboxName: externalId, providerState: state },
+                }
+                : {
+                    state,
+                    eventType: 'production_environment_reconciliation_sandbox_missing',
+                    failure: { code: 'sandbox_missing', message: 'The exact production Docker Sandbox is absent.' },
+                    metadata: { executionRunId: binding.execution_run_id, sandboxName: externalId, providerState: state },
+                };
+            if (placement) {
+                releaseExecutionEnvironmentLease({
+                    projectId: binding.project_id,
+                    environmentId: binding.environment_id,
+                    executionRunId: binding.execution_run_id,
+                    proof: { kind: 'placement_terminated', inspection: { identity: placement, state } },
+                    cleanupCompletion,
+                });
+            }
+            else {
+                releaseExecutionEnvironmentLease({
+                    projectId: binding.project_id,
+                    environmentId: binding.environment_id,
+                    executionRunId: binding.execution_run_id,
+                    proof: { kind: 'exact_sandbox_stopped', externalId, state },
+                    cleanupCompletion,
+                });
+            }
+            summary.cleaned += 1;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : 'Production environment termination could not be verified.';
+            const current = environment && getExecutionEnvironment(binding.project_id, binding.environment_id);
+            if (current && !['unavailable', 'retired'].includes(current.lifecycle_state)) {
+                const failureState = current.lifecycle_state === 'failed' ? 'cleanup_pending' : 'unavailable';
+                transitionExecutionEnvironment({
+                    projectId: binding.project_id,
+                    environmentId: binding.environment_id,
+                    state: failureState,
+                    eventType: 'production_environment_reconciliation_failed',
+                    failure: { code: 'production_termination_unverified', message },
+                    metadata: { executionRunId: binding.execution_run_id, leaseRetained: true },
+                });
+            }
+            else if (current) {
+                appendExecutionEnvironmentEvent({
+                    projectId: binding.project_id,
+                    environmentId: binding.environment_id,
+                    eventType: 'production_environment_reconciliation_failed',
+                    fromState: current.lifecycle_state,
+                    toState: current.lifecycle_state,
+                    message,
+                    metadata: { executionRunId: binding.execution_run_id, leaseRetained: true },
+                });
+            }
+            interruptProductionRunAfterCleanupFailure(binding, message);
+            summary.failed += 1;
+        }
+    }
+    return summary;
+};
+const recordObservation = (environment, observed) => {
+    const targetState = observed.state === 'stopped' ? 'stopped' : 'ready';
+    const allowed = ((targetState === 'stopped' && ['preparing', 'ready', 'running'].includes(environment.lifecycle_state))
+        || (targetState === 'ready' && ['preparing', 'running', 'stopped'].includes(environment.lifecycle_state)));
+    if (environment.lifecycle_state !== targetState && allowed) {
+        transitionExecutionEnvironment({
+            projectId: environment.project_id,
+            environmentId: environment.id,
+            state: targetState,
+            eventType: 'environment_reconciled',
+            message: `Docker Sandbox was observed ${observed.state} during startup reconciliation.`,
+            metadata: { providerState: observed.state },
+        });
+        return;
+    }
+    appendExecutionEnvironmentEvent({
+        projectId: environment.project_id,
+        environmentId: environment.id,
+        eventType: 'environment_reconciled',
+        fromState: environment.lifecycle_state,
+        toState: environment.lifecycle_state,
+        message: `Docker Sandbox was observed ${observed.state} during startup reconciliation.`,
+        metadata: { providerState: observed.state },
+    });
+};
+export const reconcileExecutionEnvironments = async (adapter = createDefaultDockerSandboxAdapter()) => {
+    const rows = db.prepare(`
+    SELECT id, project_id
+    FROM execution_environments
+    WHERE lifecycle_state <> 'retired'
+    ORDER BY created_at ASC, id ASC
+  `).all();
+    const summary = { observed_running: 0, observed_stopped: 0, observed_missing: 0, unavailable: 0, deferred: 0 };
+    for (const row of rows) {
+        const environment = getExecutionEnvironment(row.project_id, row.id);
+        if (!environment || environment.provider_kind !== DOCKER_SANDBOX_PROVIDER) {
+            summary.deferred += 1;
+            continue;
+        }
+        try {
+            const expected = resolveExpectedDockerSandbox(environment);
+            const observed = await adapter.inspect(expected.name);
+            if (!observed) {
+                if (environment.lifecycle_state === 'failed') {
+                    appendExecutionEnvironmentEvent({
+                        projectId: environment.project_id,
+                        environmentId: environment.id,
+                        eventType: 'environment_reconciliation_missing',
+                        fromState: environment.lifecycle_state,
+                        toState: environment.lifecycle_state,
+                        message: 'The persisted Docker Sandbox was not found during startup reconciliation.',
+                        metadata: { failureCode: 'sandbox_missing' },
+                    });
+                    summary.observed_missing += 1;
+                    continue;
+                }
+                transitionExecutionEnvironment({
+                    projectId: environment.project_id,
+                    environmentId: environment.id,
+                    state: 'unavailable',
+                    eventType: 'environment_reconciliation_missing',
+                    failure: { code: 'sandbox_missing', message: 'The persisted Docker Sandbox was not found during startup reconciliation.' },
+                });
+                summary.observed_missing += 1;
+                continue;
+            }
+            if (!inspectionMatches(expected, observed)) {
+                throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} does not match Archon's persisted external identity`);
+            }
+            recordObservation(environment, observed);
+            summary[observed.state === 'running' ? 'observed_running' : 'observed_stopped'] += 1;
+        }
+        catch (error) {
+            const current = getExecutionEnvironment(environment.project_id, environment.id);
+            const message = error instanceof Error ? error.message : 'Docker Sandbox reconciliation failed.';
+            if (current && !['failed', 'unavailable', 'retired'].includes(current.lifecycle_state)) {
+                transitionExecutionEnvironment({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    state: 'unavailable',
+                    eventType: 'environment_reconciliation_unavailable',
+                    failure: { code: error instanceof DockerSandboxCollisionFailure ? 'identity_mismatch' : 'provider_unavailable', message },
+                });
+            }
+            else if (current) {
+                appendExecutionEnvironmentEvent({
+                    projectId: current.project_id,
+                    environmentId: current.id,
+                    eventType: 'environment_reconciliation_unavailable',
+                    fromState: current.lifecycle_state,
+                    toState: current.lifecycle_state,
+                    message,
+                });
+            }
+            summary.unavailable += 1;
+        }
+    }
+    return summary;
+};
+const listOrphanedExecutionEnvironmentScopes = () => {
+    const rows = db.prepare(`
+  SELECT DISTINCT environment.project_id, environment.target_kind, environment.target_key
+  FROM execution_environments environment
+  INNER JOIN projects project
+    ON project.id = environment.project_id
+   AND project.deletion_started_at IS NULL
+  WHERE environment.lifecycle_state <> 'retired'
+    AND (
+      (
+        environment.target_kind = 'bundle-worktree'
+        AND NOT EXISTS (
+          SELECT 1
+          FROM worktree_bundles bundle
+          WHERE bundle.project_id = environment.project_id
+            AND environment.target_key = 'project:' || environment.project_id || ':bundle:' || bundle.id
+        )
+      )
+      OR (
+        environment.target_kind = 'branch-worktree'
+        AND NOT EXISTS (
+          SELECT 1
+          FROM tickets ticket
+          WHERE ticket.project_id = environment.project_id
+            AND ticket.worktree_bundle_id IS NULL
+            AND ticket.branch IS NOT NULL
+            AND environment.target_key = 'project:' || ticket.project_id || ':branch:' || ticket.branch
+        )
+      )
+    )
+  ORDER BY environment.project_id ASC, environment.target_kind ASC, environment.target_key ASC
+  `).all();
+    return rows.map(row => ({
+        projectId: row.project_id,
+        targetKind: row.target_kind,
+        targetKey: row.target_key,
+    }));
+};
+export const reconcileOrphanedExecutionEnvironments = async (adapter = createDefaultDockerSandboxAdapter()) => {
+    const scopes = listOrphanedExecutionEnvironmentScopes();
+    const summary = { scanned: scopes.length, cleaned: 0, failed: 0 };
+    for (const scope of scopes) {
+        try {
+            const result = await cleanupExecutionEnvironmentsForTarget({ ...scope, adapter });
+            summary.cleaned += result.cleaned;
+        }
+        catch {
+            summary.failed += 1;
+        }
+    }
+    return summary;
+};
+const getCleanupScopeQuery = (scope, alias = '') => {
+    const column = (name) => alias ? `${alias}.${name}` : name;
+    const conditions = [`${column('project_id')} = ?`];
+    const params = [scope.projectId];
+    if (scope.targetKind !== undefined) {
+        conditions.push(`${column('target_kind')} = ?`);
+        params.push(scope.targetKind);
+    }
+    if (scope.targetKey !== undefined) {
+        conditions.push(`${column('target_key')} = ?`);
+        params.push(scope.targetKey);
+    }
+    return { where: conditions.join(' AND '), params };
+};
+const assertNoActiveEnvironmentBindings = (scope) => {
+    const query = getCleanupScopeQuery(scope, 'environment');
+    const active = db.prepare(`
+    SELECT binding.environment_id, binding.execution_run_id
+    FROM execution_environment_run_bindings binding
+    JOIN execution_environments environment
+      ON environment.id = binding.environment_id AND environment.project_id = binding.project_id
+    WHERE ${query.where}
+      AND binding.released_at IS NULL
+    LIMIT 1
+  `).get(...query.params);
+    if (active) {
+        throw new ExecutionEnvironmentCleanupError(`Execution environment ${active.environment_id} is still used by active run ${active.execution_run_id}`);
+    }
+};
+const prepareEnvironmentsForCleanup = (scope, eventType, message) => {
+    const query = getCleanupScopeQuery(scope);
+    const rows = db.prepare(`
+    SELECT id FROM execution_environments
+    WHERE ${query.where} AND lifecycle_state NOT IN ('retiring', 'cleanup_pending', 'retired')
+    ORDER BY created_at ASC, id ASC
+  `).all(...query.params);
+    for (const row of rows) {
+        transitionExecutionEnvironment({
+            projectId: scope.projectId,
+            environmentId: row.id,
+            state: 'retiring',
+            eventType,
+            message,
+        });
+    }
+};
+const cleanupExecutionEnvironments = async (scope, claimId, adapter = createDefaultDockerSandboxAdapter(), options) => {
+    assertNoActiveEnvironmentBindings(scope);
+    prepareEnvironmentsForCleanup(scope, options.startedEventType, options.startedMessage);
+    const staleBefore = new Date().toISOString();
+    let cleaned = 0;
+    while (true) {
+        const claimed = claimExecutionEnvironmentsForCleanup({
+            projectId: scope.projectId,
+            targetKind: scope.targetKind,
+            targetKey: scope.targetKey,
+            claimId,
+            limit: 200,
+            staleBefore,
+        });
+        if (claimed.length === 0) {
+            break;
+        }
+        const failures = [];
+        for (const environment of claimed) {
+            try {
+                const expected = resolveExpectedDockerSandbox(environment);
+                const observed = await adapter.inspect(expected.name);
+                if (observed && !inspectionMatches(expected, observed)) {
+                    throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} does not match Archon's persisted external identity`);
+                }
+                if (observed?.state === 'running') {
+                    await adapter.stop(expected);
+                }
+                if (observed) {
+                    await adapter.remove(expected);
+                }
+                const remaining = await adapter.inspect(expected.name);
+                if (remaining && !inspectionMatches(expected, remaining)) {
+                    throw new DockerSandboxCollisionFailure(expected.name, `Sandbox ${expected.name} was replaced before absence could be verified`);
+                }
+                if (remaining) {
+                    throw new Error(`Sandbox ${expected.name} still exists after removal`);
+                }
+                completeExecutionEnvironmentCleanup({ projectId: scope.projectId, environmentId: environment.id, claimId });
+            }
+            catch (error) {
+                const failure = error instanceof Error ? error : new Error(String(error));
+                failExecutionEnvironmentCleanup({
+                    projectId: scope.projectId,
+                    environmentId: environment.id,
+                    claimId,
+                    failure: {
+                        code: error instanceof DockerSandboxCollisionFailure ? 'identity_mismatch' : 'cleanup_failed',
+                        message: failure.message,
+                    },
+                });
+                failures.push(failure);
+            }
+        }
+        if (failures.length > 0) {
+            throw new ExecutionEnvironmentCleanupError(`Failed to clean ${failures.length} execution environment(s): ${failures[0]?.message ?? 'Unknown cleanup failure'}`, { cause: new AggregateError(failures) });
+        }
+        cleaned += claimed.length;
+    }
+    const query = getCleanupScopeQuery(scope);
+    const remaining = db.prepare(`
+    SELECT id FROM execution_environments
+    WHERE ${query.where} AND lifecycle_state <> 'retired'
+    LIMIT 1
+  `).get(...query.params);
+    if (remaining) {
+        throw new ExecutionEnvironmentCleanupError(`Execution environment ${remaining.id} could not be claimed for verified cleanup.`);
+    }
+    return { cleaned };
+};
+export const cleanupExecutionEnvironmentsForTarget = async (input) => cleanupExecutionEnvironments({
+    projectId: input.projectId,
+    targetKind: input.targetKind,
+    targetKey: input.targetKey,
+}, randomUUID(), input.adapter, {
+    startedEventType: 'environment_target_deletion_started',
+    startedMessage: 'Environment cleanup started because its execution target is being deleted.',
+});
+export const cleanupProjectExecutionEnvironments = async (projectId, claimId, adapter = createDefaultDockerSandboxAdapter()) => cleanupExecutionEnvironments({ projectId }, claimId, adapter, {
+    startedEventType: 'environment_project_deletion_started',
+    startedMessage: 'Environment cleanup started for claimed project deletion.',
+});
+//# sourceMappingURL=executionEnvironmentReconciliation.js.map