@straiffi/archon 1.10.9 → 1.10.10

package/dist/server/lib/run.js

@@ -1,23 +1,36 @@
 import { spawn } from "child_process";
-import { existsSync, realpathSync } from "fs";
-import { isAbsolute, relative, resolve } from "path";
+import { assertHostSessionRequest, createHostRunSession, createRunConfigSnapshot, finishHostService, queueHostServices, serviceIdentity } from "./hostRunSession.js";
+import { isProcessAlive, killManagedProcess, runCommand, runWorkspaceSetup, stopRunProcesses, trackExecutionHandle, waitForProcessesToExit } from "./runProcesses.js";
+import { createRunServiceLauncher } from "./runServices.js";
+import { withHostServiceRecovery } from "./hostServiceRunProvider.js";
+import { createHostSessionSettler, finalizeDurableRun } from "./runSessionLifecycle.js";
+import { existsSync } from "fs";
 import { resolveWorktreePath } from "./git.js";
 import { getProjectById, getProjectRunConfig } from "./projects.js";
 import { resolveHelperModelConfig } from "./agentModelConfig.js";
 import config from "./config.js";
-import { applyDetectedPreviewTarget, buildPreviewProxyUrl, createIdlePreviewStatus, createRunPreviewStatus, detectPreviewTargets, inferPreviewTargetsFromLogChunk, isPreviewService, normalizePreviewLogText, resolveReachablePreviewTarget, } from "./preview.js";
-import { deactivatePreviewProxyForProject, ensurePreviewProxyForProject, } from "./previewProxy.js";
-import { getExecutionShellCommand, resolveShell } from "./shell.js";
-import { redactRepositorySecrets } from "./projectMemoryScan.js";
+import { createIdlePreviewStatus, createRunPreviewStatus, isPreviewService, } from "./preview.js";
+import { deactivatePreviewProxyForProject, } from "./previewProxy.js";
+import { resolveShell } from "./shell.js";
 import { isTicketRunSetupReady } from "./ticketDependencies.js";
 import { listTicketsInRunContext, resolveTicketBranch, resolveTicketContextKey, resolveTicketTool, } from "./tickets.js";
 import { createWorkspaceProcessEnvironment } from "./workspaceProcessEnvironment.js";
 import { resolveWorkloadExecutionEnvironment } from "./buildExecutionEnvironment.js";
-import { classifyTicketExecutionTarget, resolveExecutionTarget, } from "./executionTargets.js";
-import { completeServiceRun, createServiceRun, recordServiceRunReleaseFailure, recordServiceRunPlacement, } from "./serviceRuns.js";
-import { transitionExecutionRun } from "./executionRuns.js";
+import { classifyTicketExecutionTarget, resolveExecutionTarget, resolveExecutionTargetLocation, } from "./executionTargets.js";
+import { createServiceRun, markServiceRunAllocationStarted, hasActiveProjectServiceRun, recordServiceRunPlacement, getRecoverableHostServiceRun, retryInterruptedHostServiceRunForContext, } from "./serviceRuns.js";
+import { appendLifecycleEvent, transitionExecutionRun } from "./executionRuns.js";
 const runningProcesses = new Map();
-const MAX_PREVIEW_LOG_BUFFER_CHARS = 16_000;
+const lastRunStatuses = new Map();
+export const getLastRunStatusForProject = (projectId) => lastRunStatuses.get(projectId) ?? null;
+export class RunAdmissionError extends Error {
+    code;
+    status = 409;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "RunAdmissionError";
+    }
+}
 const getRunConfig = (project) => getProjectRunConfig(project);
 export const resolveProjectRunContextKey = (projectId, target = {}) => {
     if (target.kind === "bundle" && target.bundleId) {
@@ -58,7 +71,73 @@ const resolveRunContextKey = (ticketOrKey) => {
 export const hasRunConfig = (project = null) => hasRunnableCommands(getRunConfig(project));
 export const isRunning = (ticketOrKey) => {
     const contextKey = resolveRunContextKey(ticketOrKey);
-    return contextKey ? runningProcesses.has(contextKey) : false;
+    const state = contextKey ? runningProcesses.get(contextKey) : null;
+    if (state?.host_session && state.start_services) {
+        return state.host_session.services.some(service => ["starting", "running", "stopping"].includes(service.status));
+    }
+    return Boolean(state);
+};
+export const validateRunControl = (project, options) => {
+    const active = getActiveRunStatusForProject(project.id);
+    const state = active ? runningProcesses.get(active.context_key) : undefined;
+    if (!state && hasActiveProjectServiceRun(project.id)) {
+        if (options.starting) {
+            return "Durable service authority exists without a live runtime owner. Retry stopping the whole session before starting services.";
+        }
+        const recovered = getRecoverableHostServiceRun(project.id);
+        if (!recovered) {
+            return "Service cleanup requires reconciliation of the recorded execution environment.";
+        }
+        if (options.contextKey && options.contextKey !== recovered.context_key) {
+            return "Another service session is already active for this project";
+        }
+        if (options.sessionId !== undefined && options.sessionId !== recovered.sessionId) {
+            return "The service session has changed. Refresh before retrying.";
+        }
+        return options.serviceId !== undefined ? "Restarted host sessions require whole-session cleanup" : null;
+    }
+    if (state?.admission_pending && options.starting) {
+        return "Service admission is still pending. Retry after admission settles.";
+    }
+    if (state?.cancelled && options.starting) {
+        return "The service session is stopping. Stop the whole session before retrying.";
+    }
+    if (options.contextKey && active && options.contextKey !== active.context_key) {
+        return "Another service session is already active for this project";
+    }
+    if (state && options.starting && options.executionProvider !== undefined) {
+        const requestedProvider = options.executionProvider === "host" ? "host" : project.build_execution_provider ?? "host";
+        if (requestedProvider !== state.execution_provider) {
+            return "The requested execution provider is incompatible with the active service session";
+        }
+    }
+    if (state?.host_session) {
+        if (options.starting && state.host_session.services.some(service => (options.serviceId === undefined || service.service_id === options.serviceId) && service.status === "stopping")) {
+            return "Service process cleanup is pending. Retry stopping the service before restarting it.";
+        }
+        if (state.host_session.setup_error && options.starting) {
+            return "Shared setup failed. Stop the session before retrying.";
+        }
+        try {
+            assertHostSessionRequest(state.host_session, options.serviceId, options.sessionId);
+            return null;
+        }
+        catch (error) {
+            return getErrorMessage(error);
+        }
+    }
+    if (options.sessionId !== undefined) {
+        return "The service session is no longer active";
+    }
+    if (options.serviceId !== undefined) {
+        if (state || (options.executionProvider !== "host" && project.build_execution_provider === "docker_sandbox")) {
+            return "Individual service controls are not supported by Docker Sandbox";
+        }
+        if (!(getRunConfig(project)?.services ?? []).some((service, index) => serviceIdentity(service, index) === options.serviceId)) {
+            return "Unknown service ID";
+        }
+    }
+    return null;
 };
 export const getActiveRunStatus = () => {
     const iterator = runningProcesses.values().next();
@@ -67,6 +146,28 @@ export const getActiveRunStatus = () => {
 export const getActiveRunStatusForContextKey = (contextKey) => {
     return runningProcesses.get(contextKey)?.active_run ?? null;
 };
+export const startActiveRunServices = (contextKey, serviceId, sessionId) => {
+    const state = runningProcesses.get(contextKey);
+    if (!state?.host_session || state.cancelled || state.admission_pending) {
+        throw new RunAdmissionError("service_session_unavailable", "The host service session is no longer active");
+    }
+    assertHostSessionRequest(state.host_session, serviceId, sessionId);
+    if (state.host_session.services.some(service => (serviceId === undefined || service.service_id === serviceId) && service.status === "stopping")) {
+        throw new RunAdmissionError("service_cleanup_pending", "Service process cleanup is pending. Retry stopping the service before restarting it.");
+    }
+    if (state.start_services) {
+        return state.start_services(serviceId).then(() => {
+            if (runningProcesses.get(contextKey) === state && !state.cancelled) {
+                state.on_state_changed?.();
+            }
+        });
+    }
+    else {
+        queueHostServices(state.host_session, serviceId);
+        state.on_state_changed?.();
+    }
+    return Promise.resolve();
+};
 export const getActiveRunStatusForProject = (projectId) => {
     if (!projectId) {
         return null;
@@ -99,38 +200,6 @@ export const getPreviewStatusForProject = (projectId) => {
 const isActiveRunState = (contextKey, state) => {
     return runningProcesses.get(contextKey) === state && !state.cancelled;
 };
-const isProcessAlive = (proc) => proc.exitCode === null && proc.signalCode === null;
-const untrackProcess = (state, proc) => {
-    const index = state.processes.indexOf(proc);
-    if (index >= 0) {
-        state.processes.splice(index, 1);
-    }
-};
-const trackProcess = (state, proc) => {
-    const cleanup = () => {
-        untrackProcess(state, proc);
-    };
-    state.processes.push(proc);
-    proc.on("close", cleanup);
-    proc.on("error", cleanup);
-    return proc;
-};
-const trackExecutionHandle = (state, handle) => {
-    state.execution_handles?.push(handle);
-    return trackProcess(state, handle.child);
-};
-const spawnManagedProcess = (cmd, cwd, stdio, state) => {
-    const env = createWorkspaceProcessEnvironment(cwd);
-    delete env.PORT;
-    const shell = resolveShell();
-    const proc = spawn(shell, ["-i", "-c", cmd], {
-        cwd,
-        stdio,
-        detached: process.platform !== "win32",
-        env,
-    });
-    return trackProcess(state, proc);
-};
 export const openIde = (cwd, ideCommand) => {
     spawn(resolveShell(), ["-i", "-c", `${ideCommand} .`], {
         cwd,
@@ -152,552 +221,494 @@ export const openTicketIdeInWorkspace = (ticket, project = null, workspaceCwd =
     openIde(cwd, runConfig.ide);
     return true;
 };
-const killManagedProcess = (proc, signal) => {
-    if (!isProcessAlive(proc)) {
-        return;
-    }
-    if (proc.pid && process.platform !== "win32") {
-        try {
-            process.kill(-proc.pid, signal);
-            return;
-        }
-        catch (error) {
-            if (error.code === "ESRCH") {
-                return;
-            }
-        }
-    }
-    try {
-        proc.kill(signal);
-    }
-    catch { }
-};
-const waitForProcessesToExit = async (processes, timeoutMs) => {
-    const deadline = Date.now() + timeoutMs;
-    await new Promise((resolve) => {
-        const check = () => {
-            const allDone = processes.every((proc) => !isProcessAlive(proc));
-            if (allDone || Date.now() >= deadline) {
-                resolve();
-            }
-            else {
-                setTimeout(check, 100);
-            }
-        };
-        setTimeout(check, 100);
-    });
-};
-const runCommand = (cmd, cwd, state) => new Promise((resolve, reject) => {
-    const proc = spawnManagedProcess(cmd, cwd, ["ignore", "pipe", "pipe"], state);
-    let output = "";
-    const appendOutput = (chunk) => {
-        output = `${output}${chunk.toString()}`;
-        if (output.length > 8000) {
-            output = output.slice(-8000);
-        }
-    };
-    proc.stdout?.on("data", appendOutput);
-    proc.stderr?.on("data", appendOutput);
-    proc.on("close", (code) => {
-        if (code === 0) {
-            resolve();
-        }
-        else {
-            const suffix = output.trim() ? `\n${output.trim()}` : "";
-            reject(new Error(`Setup command failed (exit ${code}): ${cmd}${suffix}`));
-        }
-    });
-    proc.on("error", (err) => {
-        reject(err);
-    });
-});
-export const runSetupCommandsInWorkspace = async (project, cwd, execution) => {
-    const runConfig = getRunConfig(project);
-    const setupCommands = runConfig?.setup ?? [];
-    if (setupCommands.length === 0) {
-        return;
-    }
-    const state = { processes: [] };
-    for (const cmd of setupCommands) {
-        if (execution?.signal?.aborted) {
-            throw new Error("Build was stopped.");
-        }
-        if (!execution) {
-            await runCommand(cmd, cwd, state);
-            continue;
-        }
-        const shell = getExecutionShellCommand({
-            provider: execution.provider.kind,
-            command: cmd,
-            loadHostStartupFiles: execution.runtimeEnvironment.kind !== "host"
-                || execution.runtimeEnvironment.inheritance !== "governed",
-        });
-        const handle = execution.provider.execute({
-            phase: "setup",
-            target: execution.target,
-            command: shell.command,
-            args: shell.args,
-            cwd,
-            stdio: ["ignore", "pipe", "pipe"],
-            launcherEnvironment: execution.launcherEnvironment,
-            runtimeEnvironment: execution.runtimeEnvironment,
-            detached: process.platform !== "win32",
-        });
-        execution.onHandle?.(handle);
-        try {
-            let output = "";
-            const appendOutput = (chunk) => {
-                output = `${output}${chunk.toString()}`;
-                if (output.length > 8000) {
-                    output = output.slice(-8000);
-                }
-            };
-            handle.child.stdout?.on("data", appendOutput);
-            handle.child.stderr?.on("data", appendOutput);
-            await new Promise((resolve, reject) => {
-                handle.child.once("error", reject);
-                handle.child.once("close", code => {
-                    if (code === 0) {
-                        resolve();
-                        return;
-                    }
-                    const diagnostic = redactRepositorySecrets(output).content.trim();
-                    const suffix = diagnostic ? `\n${diagnostic}` : "";
-                    reject(new Error(`Setup command failed (exit ${code ?? "unknown"}).${suffix}`));
-                });
-            });
-            const placementFailure = await handle.placementFailure;
-            if (placementFailure) {
-                throw placementFailure;
-            }
-        }
-        finally {
-            execution.onHandle?.(null);
-        }
-    }
-};
+export const runSetupCommandsInWorkspace = runWorkspaceSetup;
 const clearRunState = (contextKey, state, onStopped) => {
     if (runningProcesses.get(contextKey) !== state) {
         return;
     }
+    if (state.release_execution_environment || state.finalization_failed) {
+        state.cancelled = true;
+        return;
+    }
     deactivatePreviewProxyForProject(state.active_run.project_id);
     state.preview_log_buffer = "";
+    if (state.active_run.project_id && state.active_run.config_snapshot) {
+        lastRunStatuses.set(state.active_run.project_id, {
+            ...structuredClone(state.active_run),
+            config_snapshot: state.active_run.config_snapshot,
+        });
+    }
     runningProcesses.delete(contextKey);
     onStopped?.();
 };
 const canCreateDurableServiceRun = (project, targetDescriptor) => Boolean(targetDescriptor
     && project.id
     && project.active_execution_profile_revision_id);
-const resolveServiceCwd = (workspacePath, configuredCwd) => {
-    if (!configuredCwd) {
-        return workspacePath;
-    }
-    if (isAbsolute(configuredCwd)) {
-        throw new Error("Service working directories must be relative to the canonical target");
-    }
-    const serviceCwd = resolve(workspacePath, configuredCwd);
-    const relativePath = relative(workspacePath, serviceCwd);
-    if (relativePath === ".." || relativePath.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)) {
-        throw new Error("Service working directories must remain inside the canonical target");
-    }
-    const realWorkspacePath = realpathSync(workspacePath);
-    const realServiceCwd = realpathSync(serviceCwd);
-    const realRelativePath = relative(realWorkspacePath, realServiceCwd);
-    if (realRelativePath === ".." || realRelativePath.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)) {
-        throw new Error("Service working directories must not escape the canonical target through symbolic links");
-    }
-    return serviceCwd;
-};
-const finalizeDurableRun = async (state, status, eventType, message) => {
-    const runId = state.execution_run_id;
-    if (!runId) {
-        return;
-    }
-    const release = state.release_execution_environment;
-    if (release) {
-        try {
-            await release();
-            state.release_execution_environment = null;
+const startRun = ({ activeRun, cwd, executionAgent, executionModel, logLabelPrefix, onStopped, project, runConfig, skipSetup = false, targetDescriptor, profileSelection, serviceId, sessionId, }) => {
+    const contextKey = activeRun.context_key;
+    const admittedProvider = profileSelection?.reason === "explicit_service_host_override"
+        ? "host" : project?.build_execution_provider ?? "host";
+    const existing = runningProcesses.get(contextKey);
+    if (existing) {
+        if (existing.admission_pending) {
+            throw new RunAdmissionError("service_admission_pending", "Service admission is still pending. Retry after admission settles.");
         }
-        catch (error) {
-            recordServiceRunReleaseFailure({
-                projectId: state.execution_project_id,
-                executionRunId: runId,
-                message: getErrorMessage(error),
-                intendedStatus: status,
-                intendedEventType: eventType,
-            });
-            throw error;
+        if (existing.cancelled) {
+            throw new Error("The service session is stopping. Retry the stop before starting services.");
+        }
+        if (existing.execution_provider !== admittedProvider) {
+            throw new RunAdmissionError("service_provider_conflict", "The requested execution provider is incompatible with the active service session");
+        }
+        if (existing.host_session) {
+            return startActiveRunServices(contextKey, serviceId, sessionId);
         }
+        else if (serviceId || sessionId) {
+            throw new Error("Individual service controls are only available for host sessions");
+        }
+        return Promise.resolve();
     }
-    completeServiceRun({
-        projectId: state.execution_project_id,
-        executionRunId: runId,
-        status,
-        code: status === "failed" ? "service_run_failed" : null,
-        message,
-        eventType,
-    });
-};
-const stopRunProcesses = async (state) => {
-    for (const proc of state.processes) {
-        killManagedProcess(proc, "SIGTERM");
+    if (sessionId !== undefined) {
+        throw new Error("The service session is no longer active");
     }
-    await waitForProcessesToExit(state.processes, 1000);
-    for (const proc of state.processes) {
-        if (isProcessAlive(proc)) {
-            killManagedProcess(proc, "SIGKILL");
-        }
+    if (getActiveRunStatusForProject(activeRun.project_id)) {
+        throw new Error("Another service session is already active for this project");
     }
-    await waitForProcessesToExit(state.processes, 1000);
-};
-const startRun = async ({ activeRun, cwd, executionAgent, executionModel, logLabelPrefix, onStopped, project, runConfig, skipSetup = false, targetDescriptor, profileSelection, }) => {
-    const contextKey = activeRun.context_key;
+    if (project?.id && hasActiveProjectServiceRun(project.id)) {
+        throw new RunAdmissionError("service_authority_without_runtime", "Durable service authority exists without a live runtime owner. Reconcile it before starting services.");
+    }
+    if (!existsSync(cwd)) {
+        throw new RunAdmissionError("service_workspace_unavailable", `Service workspace does not exist: ${cwd}`);
+    }
+    runConfig = structuredClone(runConfig);
+    activeRun = { ...activeRun, execution_provider: admittedProvider, config_snapshot: createRunConfigSnapshot(runConfig, {
+            preview_service_name: project?.preview_service_name,
+            preview_path: project?.preview_path,
+            preview_capability_mode: project?.preview_capability_mode,
+        }) };
+    project = structuredClone(project);
+    profileSelection = profileSelection ? structuredClone(profileSelection) : undefined;
     const state = {
+        admission_pending: true,
+        execution_provider: admittedProvider,
         processes: [],
         cancelled: false,
         active_run: activeRun,
-        preview_status: createRunPreviewStatus(project, runConfig),
+        preview_status: createRunPreviewStatus(project, runConfig, admittedProvider),
         preview_log_buffer: "",
         execution_run_id: null,
         execution_handles: [],
         release_execution_environment: null,
         execution_project_id: null,
+        setup_abort_controller: new AbortController(),
+        launch_operations: 0,
+        launch_promises: new Set(),
+        on_state_changed: () => {
+            if (state.execution_run_id && !state.cancelled) {
+                appendLifecycleEvent({
+                    projectId: project.id,
+                    runId: state.execution_run_id,
+                    eventType: "service_status_changed",
+                    metadata: { session_id: state.host_session?.id, services: state.host_session?.services },
+                });
+            }
+            onStopped?.();
+        },
     };
-    runningProcesses.set(contextKey, state);
-    if (!existsSync(cwd)) {
-        console.error(`[run] path does not exist: ${cwd}`);
-        clearRunState(contextKey, state, onStopped);
-        return;
+    const expectsHost = admittedProvider === "host";
+    if (!expectsHost && serviceId !== undefined) {
+        throw new Error("Individual service controls are not supported by Docker Sandbox");
+    }
+    if (expectsHost) {
+        state.host_session = createHostRunSession(runConfig);
+        assertHostSessionRequest(state.host_session, serviceId, state.host_session.id);
+        state.active_run = {
+            ...activeRun,
+            session_id: state.host_session.id,
+            services: state.host_session.services,
+        };
+        queueHostServices(state.host_session, serviceId);
     }
-    let execution = null;
-    let executionTarget = null;
+    runningProcesses.set(contextKey, state);
     if (canCreateDurableServiceRun(project, targetDescriptor)) {
         try {
-            const target = await resolveExecutionTarget({
-                project,
-                descriptor: targetDescriptor,
-                expectedWorkspacePath: cwd,
-            });
-            if (!isActiveRunState(contextKey, state)) {
-                return;
-            }
-            executionTarget = target;
-            const source = activeRun.kind === "ticket"
-                ? {
-                    kind: "ticket",
-                    id: activeRun.ticket_id,
-                    bundleId: target.identity.kind === "bundle-worktree"
-                        ? target.identity.bundleId
-                        : null,
-                }
-                : { kind: "project", id: project.id };
+            const target = resolveExecutionTargetLocation({ project, descriptor: targetDescriptor, expectedWorkspacePath: cwd });
+            cwd = target.workspacePath;
             const durableRun = createServiceRun({
-                projectId: project.id,
-                contextKey,
-                source,
-                target,
-                services: runConfig.services ?? [],
-                profileSelection,
+                projectId: project.id, contextKey, sessionId: state.host_session?.id,
+                source: activeRun.kind === "ticket"
+                    ? { kind: "ticket", id: activeRun.ticket_id, bundleId: target.identity.kind === "bundle-worktree" ? target.identity.bundleId : null }
+                    : { kind: "project", id: project.id },
+                target, services: runConfig.services ?? [], profileSelection,
             });
             state.execution_run_id = durableRun.id;
             state.execution_project_id = project.id;
-            if (!isActiveRunState(contextKey, state)) {
-                await finalizeDurableRun(state, "cancelled", "service_run_cancelled");
-                return;
-            }
-            execution = await resolveWorkloadExecutionEnvironment({
-                project,
-                executionRunId: durableRun.id,
-                agent: executionAgent,
-                model: executionModel,
-                target,
-                workload: "service",
-                onLeaseAcquired: release => {
-                    state.release_execution_environment = release;
-                },
-            });
-            state.release_execution_environment = execution.release;
-            if (!isActiveRunState(contextKey, state)) {
-                await finalizeDurableRun(state, "cancelled", "service_run_cancelled");
-                return;
-            }
-            state.execution_provider = execution.provider.kind === "docker_sandbox"
-                ? "docker_sandbox"
-                : "host";
-            state.preview_status = createRunPreviewStatus(project, runConfig, state.execution_provider);
-            transitionExecutionRun({
-                projectId: project.id,
-                runId: durableRun.id,
-                status: "running",
-                domainStatus: "starting",
-                eventType: "service_run_starting",
-                eventMessage: "Service execution provider and canonical target authority were resolved.",
-            });
         }
         catch (error) {
-            if (state.execution_run_id) {
-                await finalizeDurableRun(state, "failed", "service_run_start_failed", getErrorMessage(error)).catch(releaseError => {
-                    console.error("[run] service environment release failed:", getErrorMessage(releaseError));
-                });
+            if (runningProcesses.get(contextKey) === state) {
+                runningProcesses.delete(contextKey);
             }
-            console.error(`[run] durable service start failed for ${logLabelPrefix}:`, getErrorMessage(error));
-            clearRunState(contextKey, state, onStopped);
-            return;
+            throw new RunAdmissionError("service_admission_rejected", getErrorMessage(error));
         }
     }
-    if (!skipSetup) {
-        try {
-            if (execution) {
-                await runSetupCommandsInWorkspace(project, cwd, {
-                    provider: execution.provider,
-                    target: executionTarget,
-                    launcherEnvironment: execution.launcherEnvironment,
-                    runtimeEnvironment: execution.runtimeEnvironment,
-                    onHandle: handle => {
-                        if (handle) {
-                            trackExecutionHandle(state, handle);
-                        }
+    state.admission_pending = false;
+    state.settle_session = createHostSessionSettler(state, () => runningProcesses.get(contextKey) === state, () => clearRunState(contextKey, state, onStopped));
+    const startup = async () => {
+        if (!isActiveRunState(contextKey, state)) {
+            return;
+        }
+        let execution = null;
+        let executionTarget = null;
+        if (canCreateDurableServiceRun(project, targetDescriptor)) {
+            try {
+                const target = await resolveExecutionTarget({
+                    project,
+                    descriptor: targetDescriptor,
+                    expectedWorkspacePath: cwd,
+                });
+                if (!isActiveRunState(contextKey, state)) {
+                    return;
+                }
+                executionTarget = target;
+                markServiceRunAllocationStarted(project.id, state.execution_run_id);
+                execution = await resolveWorkloadExecutionEnvironment({
+                    project,
+                    executionRunId: state.execution_run_id,
+                    agent: executionAgent,
+                    model: executionModel,
+                    target,
+                    workload: "service",
+                    onLeaseAcquired: release => {
+                        state.release_execution_environment = release;
                     },
                 });
-            }
-            else {
-                for (const cmd of runConfig.setup ?? []) {
-                    await runCommand(cmd, cwd, state);
-                    if (!isActiveRunState(contextKey, state)) {
-                        return;
+                state.release_execution_environment = execution.release;
+                if (!isActiveRunState(contextKey, state)) {
+                    return;
+                }
+                if (execution.provider.kind !== state.execution_provider) {
+                    throw new Error("The resolved execution provider differs from the admitted service provider");
+                }
+                execution = { ...execution, provider: withHostServiceRecovery(execution.provider, project.id, state.execution_run_id) };
+                if (state.execution_provider === "docker_sandbox") {
+                    if (serviceId !== undefined) {
+                        throw new Error("Individual service controls are not supported by Docker Sandbox");
                     }
+                    state.host_session = undefined;
+                    state.active_run = activeRun;
                 }
+                state.preview_status = createRunPreviewStatus(project, runConfig, state.execution_provider);
+                transitionExecutionRun({
+                    projectId: project.id,
+                    runId: state.execution_run_id,
+                    status: "running",
+                    domainStatus: "starting",
+                    eventType: "service_run_starting",
+                    eventMessage: "Service execution provider and canonical target authority were resolved.",
+                });
             }
-        }
-        catch (err) {
-            if (state.cancelled) {
+            catch (error) {
+                if (state.execution_run_id) {
+                    for (const service of state.host_session?.services ?? []) {
+                        if (state.host_session?.pending.has(service.service_id)) {
+                            service.status = "failed";
+                            service.error = getErrorMessage(error);
+                        }
+                    }
+                    state.host_session?.pending.clear();
+                    await finalizeDurableRun(state, "failed", "service_run_start_failed", getErrorMessage(error)).catch(releaseError => {
+                        console.error("[run] service environment release failed:", getErrorMessage(releaseError));
+                    });
+                }
+                console.error(`[run] durable service start failed for ${logLabelPrefix}:`, getErrorMessage(error));
+                clearRunState(contextKey, state, onStopped);
                 return;
             }
-            console.error(`[run] setup failed for ${logLabelPrefix}:`, getErrorMessage(err));
-            await finalizeDurableRun(state, "failed", "service_setup_failed", getErrorMessage(err)).catch(releaseError => console.error("[run] service environment release failed:", getErrorMessage(releaseError)));
-            clearRunState(contextKey, state, onStopped);
-            return;
         }
-    }
-    if (!isActiveRunState(contextKey, state)) {
-        return;
-    }
-    let serviceFailureMessage = null;
-    let startupComplete = false;
-    const cleanupIfAllDead = () => {
-        if (!startupComplete || runningProcesses.get(contextKey) !== state || state.cancelled) {
-            return;
-        }
-        const anyAlive = state.processes.some(isProcessAlive);
-        if (!anyAlive) {
-            void finalizeDurableRun(state, serviceFailureMessage ? "failed" : "completed", serviceFailureMessage ? "service_run_failed" : "service_run_completed", serviceFailureMessage ?? undefined)
-                .catch(error => console.error("[run] service environment release failed:", getErrorMessage(error)))
-                .finally(() => clearRunState(contextKey, state, onStopped));
-        }
-    };
-    let configuredServices;
-    try {
-        configuredServices = (runConfig.services ?? []).map(service => ({
-            service,
-            cwd: resolveServiceCwd(cwd, service.cwd),
-        }));
-    }
-    catch (error) {
-        await finalizeDurableRun(state, "failed", "service_configuration_invalid", getErrorMessage(error)).catch(releaseError => console.error("[run] service environment release failed:", getErrorMessage(releaseError)));
-        clearRunState(contextKey, state, onStopped);
-        return;
-    }
-    try {
-        for (const { service, cwd: serviceCwd } of configuredServices) {
-            const label = service.name ?? service.cmd;
-            let handle = null;
-            const proc = execution
-                ? (() => {
-                    const shell = getExecutionShellCommand({
-                        provider: execution.provider.kind,
-                        command: service.cmd,
-                        loadHostStartupFiles: execution.runtimeEnvironment.kind !== "host"
-                            || execution.runtimeEnvironment.inheritance !== "governed",
-                    });
-                    handle = execution.provider.execute({
-                        phase: "service",
+        if (!skipSetup) {
+            try {
+                if (execution) {
+                    await runSetupCommandsInWorkspace(project, cwd, {
+                        provider: execution.provider,
                         target: executionTarget,
-                        command: shell.command,
-                        args: shell.args,
-                        cwd: serviceCwd,
-                        stdio: ["ignore", "pipe", "pipe"],
                         launcherEnvironment: execution.launcherEnvironment,
                         runtimeEnvironment: execution.runtimeEnvironment,
-                        detached: process.platform !== "win32",
+                        signal: state.setup_abort_controller?.signal,
+                        onHandle: handle => {
+                            if (handle) {
+                                trackExecutionHandle(state, handle);
+                                if (state.cancelled) {
+                                    killManagedProcess(handle.child, "SIGTERM");
+                                }
+                            }
+                        },
                     });
-                    return trackExecutionHandle(state, handle);
-                })()
-                : spawnManagedProcess(service.cmd, serviceCwd, ["ignore", "pipe", "pipe"], state);
-            const handleLogChunk = (data) => {
-                if (!isPreviewService(service, state.preview_status)) {
-                    return;
                 }
-                state.preview_log_buffer = normalizePreviewLogText(`${state.preview_log_buffer}${data.toString()}`).slice(-MAX_PREVIEW_LOG_BUFFER_CHARS);
-                const detectedTargets = detectPreviewTargets(state.preview_log_buffer, state.preview_status.capability);
-                const previewTargets = detectedTargets.length > 0
-                    ? detectedTargets
-                    : inferPreviewTargetsFromLogChunk(state.preview_log_buffer, project, service, state.preview_status.capability);
-                if (previewTargets.length === 0) {
-                    return;
-                }
-                void (async () => {
-                    for (const detectedTarget of previewTargets) {
-                        const reachableTarget = await resolveReachablePreviewTarget(detectedTarget, project.preview_path);
-                        if (!reachableTarget || !isActiveRunState(contextKey, state)) {
-                            continue;
-                        }
-                        if (state.preview_status.detected_url === reachableTarget.url &&
-                            state.preview_status.proxy_url) {
+                else {
+                    for (const cmd of runConfig.setup ?? []) {
+                        await runCommand(cmd, cwd, state);
+                        if (!isActiveRunState(contextKey, state)) {
                             return;
                         }
-                        try {
-                            const proxyOrigin = await ensurePreviewProxyForProject(project.id, reachableTarget.url);
-                            state.preview_status = {
-                                ...applyDetectedPreviewTarget(state.preview_status, reachableTarget),
-                                proxy_url: buildPreviewProxyUrl(proxyOrigin, reachableTarget.url),
-                            };
-                        }
-                        catch (error) {
-                            state.preview_status = {
-                                ...state.preview_status,
-                                state: "error",
-                                proxy_url: null,
-                                message: getErrorMessage(error) || "Preview proxy failed",
-                            };
+                    }
+                }
+            }
+            catch (err) {
+                if (state.cancelled) {
+                    return;
+                }
+                console.error(`[run] setup failed for ${logLabelPrefix}:`, getErrorMessage(err));
+                await finalizeDurableRun(state, "failed", "service_setup_failed", getErrorMessage(err)).catch(releaseError => console.error("[run] service environment release failed:", getErrorMessage(releaseError)));
+                if (state.host_session) {
+                    state.host_session.setup_error = getErrorMessage(err);
+                    for (const service of state.host_session.services) {
+                        if (state.host_session.pending.has(service.service_id)) {
+                            service.status = "failed";
+                            service.error = getErrorMessage(err);
                         }
-                        return;
                     }
-                })();
-            };
-            proc.stdout?.on("data", (data) => {
-                console.log(`[run:${logLabelPrefix}:${label}] ${data.toString().trimEnd()}`);
-                handleLogChunk(data);
-            });
-            proc.stderr?.on("data", (data) => {
-                console.error(`[run:${logLabelPrefix}:${label}] ${data.toString().trimEnd()}`);
-                handleLogChunk(data);
-            });
-            proc.on("close", (code) => {
-                console.log(`[run:${logLabelPrefix}:${label}] exited with code ${code}`);
-                if (code !== 0 && !state.cancelled) {
-                    serviceFailureMessage = `Service ${label} exited with code ${code ?? "unknown"}.`;
+                    state.host_session.pending.clear();
                 }
+                clearRunState(contextKey, state, onStopped);
+                return;
+            }
+        }
+        if (!isActiveRunState(contextKey, state)) {
+            return;
+        }
+        let serviceFailureMessage = null;
+        let startupComplete = false;
+        const cleanupIfAllDead = () => {
+            if (state.host_session) {
+                void state.settle_session?.().catch(error => console.error("[run] service finalization failed:", error));
+                return;
+            }
+            if (!startupComplete || runningProcesses.get(contextKey) !== state || state.cancelled) {
+                return;
+            }
+            const anyAlive = state.processes.some(isProcessAlive);
+            if (!anyAlive) {
+                state.cancelled = true;
+                void finalizeDurableRun(state, serviceFailureMessage ? "failed" : "completed", serviceFailureMessage ? "service_run_failed" : "service_run_completed", serviceFailureMessage ?? undefined)
+                    .catch(error => console.error("[run] service environment release failed:", getErrorMessage(error)))
+                    .finally(() => clearRunState(contextKey, state, onStopped));
+            }
+        };
+        const launchServices = createRunServiceLauncher({
+            state, project, runConfig, cwd, execution, executionTarget, logLabelPrefix,
+            isActive: () => isActiveRunState(contextKey, state),
+            onExit: failure => {
+                serviceFailureMessage = failure ?? serviceFailureMessage;
                 cleanupIfAllDead();
-            });
-            proc.on("error", (err) => {
-                console.error(`[run:${logLabelPrefix}:${label}] error: ${getErrorMessage(err)}`);
-                if (!state.cancelled) {
-                    serviceFailureMessage = `Service ${label} failed: ${getErrorMessage(err)}`;
+                if (state.host_session && isActiveRunState(contextKey, state)) {
+                    state.on_state_changed?.();
                 }
-                cleanupIfAllDead();
+            },
+        });
+        try {
+            state.start_services = requestedId => {
+                state.launch_operations = (state.launch_operations ?? 0) + 1;
+                const launching = (async () => {
+                    try {
+                        await launchServices(requestedId);
+                    }
+                    finally {
+                        state.launch_operations -= 1;
+                        await state.settle_session?.();
+                    }
+                })();
+                state.launch_promises.add(launching);
+                const remove = () => { state.launch_promises.delete(launching); };
+                void launching.then(remove, remove);
+                return launching;
+            };
+            if (state.host_session) {
+                const pending = [...state.host_session.pending];
+                await Promise.all(pending.map(id => state.start_services(id)));
+            }
+            else {
+                await launchServices();
+            }
+        }
+        catch (error) {
+            await stopRunProcesses(state);
+            await finalizeDurableRun(state, "failed", "service_placement_failed", getErrorMessage(error)).catch(releaseError => console.error("[run] service environment release failed:", getErrorMessage(releaseError)));
+            clearRunState(contextKey, state, onStopped);
+            return;
+        }
+        if (state.execution_run_id && execution && (runConfig.services ?? []).length === 0) {
+            recordServiceRunPlacement({
+                projectId: project.id,
+                executionRunId: state.execution_run_id,
+                provider: state.execution_provider,
+                environmentId: execution.environment?.id ?? null,
+                environmentRevision: execution.environment?.environment_revision ?? null,
             });
-            if (handle) {
-                const placementFailure = await handle.placementFailure;
-                if (!isActiveRunState(contextKey, state)) {
+        }
+        if (!isActiveRunState(contextKey, state)) {
+            return;
+        }
+        if (state.execution_run_id) {
+            transitionExecutionRun({
+                projectId: project.id,
+                runId: state.execution_run_id,
+                status: "running",
+                domainStatus: "active",
+                eventType: "service_run_active",
+                eventMessage: state.execution_provider === "docker_sandbox"
+                    ? "Services are active in a shared Docker Sandbox environment; interruption applies to the whole environment."
+                    : "Services are active as verified host processes.",
+            });
+        }
+        if ((runConfig.services ?? []).length === 0) {
+            await finalizeDurableRun(state, "completed", "service_run_completed").catch(error => console.error("[run] service environment release failed:", getErrorMessage(error)));
+            clearRunState(contextKey, state, onStopped);
+            return;
+        }
+        startupComplete = true;
+        cleanupIfAllDead();
+    };
+    state.startup_promise = startup().finally(async () => {
+        state.startup_complete = true;
+        if (!state.finalization_failed) {
+            await state.settle_session?.();
+        }
+    });
+    return state.startup_promise;
+};
+export const stopAllTickets = async () => {
+    const contextKeys = [...runningProcesses.keys()];
+    await Promise.all(contextKeys.map(contextKey => stopTicket(contextKey)));
+};
+export const stopTicket = async (ticketOrKey, options = {}) => {
+    let contextKey = resolveRunContextKey(ticketOrKey);
+    if (options.sessionId && ticketOrKey && typeof ticketOrKey !== "string") {
+        const frozen = [...runningProcesses.values()].find(state => state.host_session?.id === options.sessionId
+            && state.active_run.kind === "ticket" && state.active_run.ticket_id === ticketOrKey.id);
+        contextKey = frozen?.active_run.context_key ?? contextKey;
+    }
+    if (!contextKey) {
+        return;
+    }
+    const state = runningProcesses.get(contextKey);
+    if (!state) {
+        if (retryInterruptedHostServiceRunForContext(contextKey, options)) {
+            return;
+        }
+        if (options.sessionId) {
+            throw new Error("The service session is no longer active");
+        }
+        return;
+    }
+    if (state.host_session) {
+        assertHostSessionRequest(state.host_session, options.serviceId, options.sessionId);
+        if (options.serviceId !== undefined) {
+            const session = state.host_session;
+            session.pending.delete(options.serviceId);
+            const status = session.services.find(service => service.service_id === options.serviceId);
+            const proc = session.processes.get(options.serviceId);
+            if (!proc) {
+                status.status = "stopped";
+                if (!state.start_services && session.pending.size === 0 && session.processes.size === 0) {
+                    await stopTicket(contextKey, { sessionId: session.id });
                     return;
                 }
-                if (placementFailure) {
-                    throw placementFailure;
+                await state.settle_session?.();
+                return;
+            }
+            state.launch_operations = (state.launch_operations ?? 0) + 1;
+            try {
+                status.status = "stopping";
+                const configuredService = session.config.services.find((service, index) => serviceIdentity(service, index) === options.serviceId);
+                if (configuredService && isPreviewService(configuredService, state.preview_status)) {
+                    deactivatePreviewProxyForProject(state.active_run.project_id);
+                    state.preview_status = { ...state.preview_status, state: "idle", detected_url: null, detected_origin: null, detected_path: null, proxy_url: null };
+                    state.preview_log_buffer = "";
                 }
-                const placement = await handle.placementIdentity;
-                if (!isActiveRunState(contextKey, state)) {
-                    return;
+                killManagedProcess(proc, "SIGTERM");
+                await waitForProcessesToExit([proc], 1000);
+                killManagedProcess(proc, "SIGKILL");
+                await waitForProcessesToExit([proc], 1000);
+                if (isProcessAlive(proc)) {
+                    throw new Error("The service did not exit. Retry stopping it before restarting.");
                 }
-                recordServiceRunPlacement({
-                    projectId: project.id,
-                    executionRunId: state.execution_run_id,
-                    provider: state.execution_provider,
-                    environmentId: execution?.environment?.id ?? null,
-                    environmentRevision: execution?.environment?.environment_revision ?? null,
-                    placement,
-                });
+                finishHostService(session, options.serviceId, proc, proc.exitCode);
+                state.on_state_changed?.();
+            }
+            finally {
+                state.launch_operations -= 1;
+                await state.settle_session?.();
             }
+            return;
         }
     }
-    catch (error) {
-        await stopRunProcesses(state);
-        await finalizeDurableRun(state, "failed", "service_placement_failed", getErrorMessage(error)).catch(releaseError => console.error("[run] service environment release failed:", getErrorMessage(releaseError)));
-        clearRunState(contextKey, state, onStopped);
-        return;
+    else if (options.serviceId !== undefined || options.sessionId !== undefined) {
+        throw new Error("Individual service controls are not supported by Docker Sandbox");
     }
-    if (state.execution_run_id && execution && (runConfig.services ?? []).length === 0) {
-        recordServiceRunPlacement({
-            projectId: project.id,
-            executionRunId: state.execution_run_id,
-            provider: state.execution_provider,
-            environmentId: execution.environment?.id ?? null,
-            environmentRevision: execution.environment?.environment_revision ?? null,
-        });
+    const stoppingWork = !state.finalization_intent && (!state.startup_complete
+        || Boolean(state.launch_operations) || Boolean(state.host_session?.pending.size)
+        || state.processes.some(isProcessAlive));
+    state.cancelled = true;
+    state.setup_abort_controller?.abort();
+    state.host_session?.pending.clear();
+    if (state.host_session && stoppingWork) {
+        state.host_session.stop_requested = true;
+    }
+    for (const service of state.host_session?.services ?? []) {
+        if (["starting", "running"].includes(service.status)) {
+            service.status = "stopping";
+        }
     }
-    if (!isActiveRunState(contextKey, state)) {
+    deactivatePreviewProxyForProject(state.active_run.project_id);
+    await stopRunProcesses(state);
+    await state.startup_promise?.catch(error => {
+        console.error("[run] startup settled during stop:", error);
+    });
+    await Promise.allSettled([...(state.launch_promises ?? [])]);
+    if (runningProcesses.get(contextKey) !== state) {
         return;
     }
-    if (state.execution_run_id) {
-        transitionExecutionRun({
-            projectId: project.id,
-            runId: state.execution_run_id,
-            status: "running",
-            domainStatus: "active",
-            eventType: "service_run_active",
-            eventMessage: state.execution_provider === "docker_sandbox"
-                ? "Services are active in a shared Docker Sandbox environment; interruption applies to the whole environment."
-                : "Services are active as verified host processes.",
-        });
-    }
-    if ((runConfig.services ?? []).length === 0) {
-        await finalizeDurableRun(state, "completed", "service_run_completed").catch(error => console.error("[run] service environment release failed:", getErrorMessage(error)));
-        clearRunState(contextKey, state, onStopped);
+    if (state.finalization) {
+        await state.finalization;
         return;
     }
-    startupComplete = true;
-    cleanupIfAllDead();
-};
-export const stopAllTickets = async () => {
-    const contextKeys = [...runningProcesses.keys()];
-    await Promise.all(contextKeys.map(stopTicket));
-};
-export const stopTicket = async (ticketOrKey) => {
-    const contextKey = resolveRunContextKey(ticketOrKey);
-    if (!contextKey) {
-        return;
+    await stopRunProcesses(state);
+    for (const [id, proc] of state.host_session?.processes ?? []) {
+        finishHostService(state.host_session, id, proc, proc.exitCode);
     }
-    const state = runningProcesses.get(contextKey);
-    if (!state) {
+    for (const service of state.host_session?.services ?? []) {
+        if (service.status === "stopping") {
+            service.status = "stopped";
+        }
+    }
+    await state.settle_session?.();
+    if (state.finalization) {
+        await state.finalization;
         return;
     }
-    state.cancelled = true;
-    await stopRunProcesses(state);
-    await finalizeDurableRun(state, "cancelled", "service_run_cancelled", state.execution_provider === "docker_sandbox"
+    state.finalization ??= finalizeDurableRun(state, "cancelled", "service_run_cancelled", state.execution_provider === "docker_sandbox"
         ? "The service run was stopped. Docker Sandbox interruption stops the whole environment and every attached service."
-        : "The service run host processes were stopped.").catch(error => console.error("[run] service environment release failed:", getErrorMessage(error)));
-    if (runningProcesses.get(contextKey) === state) {
-        deactivatePreviewProxyForProject(state.active_run.project_id);
-        state.preview_log_buffer = "";
-        runningProcesses.delete(contextKey);
-    }
+        : "The service run host processes were stopped.").then(() => clearRunState(contextKey, state)).catch(error => {
+        state.finalization = null;
+        throw error;
+    });
+    await state.finalization;
 };
 export const runTicket = async (ticket, io, project = null) => {
     return runTicketInWorkspace(ticket, io, project, null);
 };
-export const runTicketInWorkspace = async (ticket, io, project = null, workspaceCwd = null, profileSelection) => {
+export const runTicketInWorkspace = (ticket, io, project = null, workspaceCwd = null, profileSelection, controls = {}) => {
     const runConfig = getRunConfig(project);
     if (!runConfig || !hasRunnableCommands(runConfig)) {
-        return;
+        return Promise.resolve();
     }
     const contextKey = resolveTicketContextKey(ticket);
     const branch = resolveTicketBranch(ticket);
     if (!contextKey || (!branch && !workspaceCwd)) {
-        return;
+        return Promise.resolve();
     }
     const cwd = workspaceCwd ?? resolveWorktreePath(branch, project);
     const executionAgent = resolveTicketTool(ticket, config.tool ?? "opencode");
-    await startRun({
+    return startRun({
         activeRun: {
             kind: "ticket",
             context_key: contextKey,
@@ -713,7 +724,7 @@ export const runTicketInWorkspace = async (ticket, io, project = null, workspace
             preferWorkflow: true,
         }).model,
         logLabelPrefix: `ticket:${ticket.id}`,
-        onStopped: () => emitRunStateForContext(io, ticket, false),
+        onStopped: () => emitRunStateForContext(io, ticket, isRunning(ticket)),
         project: project,
         runConfig,
         skipSetup: isTicketRunSetupReady(ticket),
@@ -725,17 +736,18 @@ export const runTicketInWorkspace = async (ticket, io, project = null, workspace
             })
             : undefined,
         profileSelection,
+        ...controls,
     });
 };
-export const runProject = async (project, options = {}) => {
+export const runProject = (project, options = {}) => {
     const runConfig = getRunConfig(project);
     if (!runConfig || !hasRunnableCommands(runConfig)) {
-        return;
+        return Promise.resolve();
     }
     const targetKind = options.targetKind ?? "repo_root";
     const targetBundleId = targetKind === "bundle" ? (options.targetBundleId ?? null) : null;
     const executionAgent = config.tool ?? "opencode";
-    await startRun({
+    return startRun({
         activeRun: {
             kind: "project",
             context_key: resolveProjectRunContextKey(project.id, {
@@ -767,9 +779,16 @@ export const runProject = async (project, options = {}) => {
                 branch: options.targetBranch ?? null,
             },
         profileSelection: options.profileSelection,
+        serviceId: options.serviceId,
+        sessionId: options.sessionId,
     });
 };
 export const stopProjectRun = async (projectId, target = {}) => {
-    await stopTicket(resolveProjectRunContextKey(projectId, target));
+    const active = getActiveRunStatusForProject(projectId);
+    const recovered = active ? null : getRecoverableHostServiceRun(projectId);
+    if (!active && !recovered && hasActiveProjectServiceRun(projectId)) {
+        throw new Error("Service cleanup requires reconciliation of the recorded execution environment.");
+    }
+    await stopTicket(active?.context_key ?? recovered?.context_key ?? resolveProjectRunContextKey(projectId, target), target);
 };
 //# sourceMappingURL=run.js.map