@roj-ai/sdk 0.1.36 → 0.2.2

package/dist/plugins/services/service.js

@@ -46,19 +46,43 @@ const PORT_CONFLICT_PATTERN = /EADDRINUSE|address already in use/i;
  * `failed` instead of looping forever.
  */
 const MAX_PORT_CONFLICT_RETRIES = 3;
+/**
+ * Deadline for embedder callbacks and PID-registry writes inside a transition.
+ *
+ * A transition holds the runtime lease that idle eviction waits on, so a resolver
+ * or a registry write that never settles pins the session resident forever and
+ * defeats the bound the lease exists to enforce. None of them is meant to take
+ * 30 s — the deadline only turns "wedged" into "failed".
+ */
+const DEFAULT_SERVICE_HOOK_TIMEOUT_MS = 30_000;
 /** `restartPolicy` defaults — see ServiceConfig.restartPolicy for the rationale. */
 const DEFAULT_MAX_RESTART_RETRIES = 3;
 const DEFAULT_RESTART_DELAY_MS = 1000;
 const DEFAULT_MAX_RESTART_DELAY_MS = 30_000;
 const DEFAULT_RESTART_HEALTHY_AFTER_MS = 60_000;
+let serviceExecutorObserverForTesting;
+/** Observe executor creation in integration tests without exposing lifecycle methods as plugin API. */
+export function setServiceExecutorObserverForTesting(observer) {
+    const previous = serviceExecutorObserverForTesting;
+    serviceExecutorObserverForTesting = observer;
+    return () => {
+        if (serviceExecutorObserverForTesting === observer)
+            serviceExecutorObserverForTesting = previous;
+    };
+}
 export class ServiceExecutor {
     services = new Map();
     allocatedPorts = new Map();
     waiters = new Map();
     /** Per-type lock collapsing concurrent start() calls onto a single in-flight start. */
     startInFlight = new Map();
+    /** Per-type stop operation awaited by runtime disposal. */
+    stopInFlight = new Map();
+    /** Registry deletions that must settle before a replacement can record its PID. */
+    pidForgets = new Map();
     /** Per-type counter bounding automatic EADDRINUSE port re-allocation retries. */
     portConflictRetries = new Map();
+    portConflictTimers = new Map();
     /** Per-type counter bounding `restartPolicy` revivals after an unexpected exit. */
     restartRetries = new Map();
     /** Pending `restartPolicy` timers, cancelled when someone stops or starts the service by hand. */
@@ -68,14 +92,44 @@ export class ServiceExecutor {
     fs;
     processRunner;
     pidRegistry;
+    killProcess;
+    hookTimeoutMs;
+    closing = false;
+    closePromise;
     /** Optional callback invoked on every service status change */
     onStatusChanged;
+    /** Invoked after every owned start attempt, including deferred retries. */
+    onStartSettled;
     constructor(logger, portPool, deps) {
         this.logger = logger;
         this.portPool = portPool;
         this.fs = deps.fs;
         this.processRunner = deps.process;
         this.pidRegistry = deps.pidRegistry;
+        this.killProcess = deps.kill ?? process.kill;
+        this.hookTimeoutMs = deps.hookTimeoutMs ?? DEFAULT_SERVICE_HOOK_TIMEOUT_MS;
+        serviceExecutorObserverForTesting?.(this);
+    }
+    /**
+     * Reject when `operation` outlives {@link DEFAULT_SERVICE_HOOK_TIMEOUT_MS}.
+     *
+     * The operation itself cannot be cancelled — an embedder callback keeps running —
+     * but the transition stops waiting on it, which is what releases the lease.
+     */
+    async withDeadline(label, operation) {
+        let timer;
+        try {
+            return await Promise.race([
+                (async () => await operation())(),
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => reject(new Error(`${label} did not settle within ${this.hookTimeoutMs}ms`)), this.hookTimeoutMs);
+                }),
+            ]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
     }
     notifyStatusChanged(sessionId, serviceType, status, details = {}) {
         this.onStatusChanged?.(sessionId, serviceType, status, details);
@@ -87,7 +141,7 @@ export class ServiceExecutor {
         if (!config.availableWhen)
             return Ok(true);
         try {
-            return Ok(await config.availableWhen({ sessionId: String(sessionId), workspaceDir }));
+            return Ok(await this.withDeadline(`Service '${config.type}' availability check`, () => config.availableWhen?.({ sessionId: String(sessionId), workspaceDir }) ?? true));
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -103,7 +157,8 @@ export class ServiceExecutor {
                         recoverable: true,
                     });
                 }
-                const raw = await config.cwd({ sessionId: String(sessionId), workspaceDir });
+                const cwdResolver = config.cwd;
+                const raw = await this.withDeadline(`Service '${config.type}' cwd resolver`, () => cwdResolver({ sessionId: String(sessionId), workspaceDir }));
                 return Ok(resolve(workspaceDir, raw));
             }
             if (config.cwd !== undefined) {
@@ -172,6 +227,8 @@ export class ServiceExecutor {
      * Deduping here guarantees exactly one process per type.
      */
     async start(config, sessionId, workspaceDir, preferredPort) {
+        if (this.closing)
+            return Err({ message: 'Service executor is closing', recoverable: false });
         const inFlight = this.startInFlight.get(config.type);
         if (inFlight)
             return inFlight;
@@ -182,16 +239,43 @@ export class ServiceExecutor {
         }
         finally {
             this.startInFlight.delete(config.type);
+            this.onStartSettled?.(config.type);
         }
     }
     async startInternal(config, sessionId, workspaceDir, preferredPort) {
+        if (this.closing)
+            return Err({ message: 'Service executor is closing', recoverable: false });
         // This start supersedes any revival the policy had queued.
         this.cancelPendingRestart(config.type);
         const existing = this.services.get(config.type);
         if (existing && (existing.status === 'starting' || existing.status === 'ready')) {
             return Ok(undefined);
         }
+        if (existing?.status === 'stopping') {
+            return Err({ message: `Service '${config.type}' is stopping, cannot start`, recoverable: true });
+        }
+        if (existing) {
+            // Whatever sits under this type is about to be overwritten — in this map and in
+            // the pid registry, both keyed by (session, type) — and to lose its port to the
+            // process spawned below, since a restart deliberately reuses it. A `failed`
+            // entry can still own a live process group, so reclaim it before its handle is
+            // dropped or nothing will ever be able to find it again.
+            const reaped = await this.reapEntry(existing, 'superseded by a new start');
+            if (!reaped.ok) {
+                this.logger.warn('Could not reclaim the superseded service process group', {
+                    serviceType: config.type,
+                    pid: existing.pid,
+                    error: reaped.error.message,
+                });
+            }
+            await existing.forgetPid();
+        }
+        await this.pidForgets.get(config.type);
+        if (this.closing)
+            return Err({ message: 'Service executor is closing', recoverable: false });
         const availability = await this.isAvailable(config, sessionId, workspaceDir);
+        if (this.closing)
+            return Err({ message: 'Service executor is closing', recoverable: false });
         if (!availability.ok)
             return availability;
         if (!availability.value) {
@@ -219,9 +303,29 @@ export class ServiceExecutor {
             return cwdResult;
         }
         const cwd = cwdResult.value;
-        const logBufferSize = config.logBufferSize ?? 200;
+        if (this.closing) {
+            if (allocatedNow) {
+                this.allocatedPorts.delete(config.type);
+                this.portPool.release(port);
+            }
+            return Err({ message: 'Service executor is closing', recoverable: false });
+        }
+        const logBufferSize = Math.max(1, config.logBufferSize ?? 200);
         const startupTimeoutMs = config.startupTimeoutMs ?? 30_000;
         const readyRegex = config.readyPattern ? new RegExp(config.readyPattern) : undefined;
+        const logs = new RingBuffer(logBufferSize);
+        const maxDiagnosticLength = 16_384;
+        const startTime = Date.now();
+        let portConflictDetected = false;
+        let setupComplete = false;
+        let retryPortConflictDuringSetup = false;
+        let readyDetectedDuringSetup = false;
+        let readyLineDuringSetup;
+        let stdoutPartial = '';
+        let stderrPartial = '';
+        let outputSequence = 0;
+        let stdoutPartialSequence = 0;
+        let stderrPartialSequence = 0;
         const startArgs = {
             port,
             sessionId: String(sessionId),
@@ -231,12 +335,14 @@ export class ServiceExecutor {
         let command;
         let serviceEnv;
         try {
-            command = typeof config.command === 'function'
-                ? await config.command(startArgs)
-                : config.command;
-            serviceEnv = typeof config.env === 'function'
-                ? await config.env(startArgs)
-                : config.env;
+            const commandResolver = config.command;
+            command = typeof commandResolver === 'function'
+                ? await this.withDeadline(`Service '${config.type}' command resolver`, () => commandResolver(startArgs))
+                : commandResolver;
+            const envResolver = config.env;
+            serviceEnv = typeof envResolver === 'function'
+                ? await this.withDeadline(`Service '${config.type}' env resolver`, () => envResolver(startArgs))
+                : envResolver;
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
@@ -248,6 +354,13 @@ export class ServiceExecutor {
             this.notifyStatusChanged(sessionId, config.type, 'failed', { port, cwd, error: errorMessage });
             return Err({ message: errorMessage, recoverable: true });
         }
+        if (this.closing) {
+            if (allocatedNow) {
+                this.allocatedPorts.delete(config.type);
+                this.portPool.release(port);
+            }
+            return Err({ message: 'Service executor is closing', recoverable: false });
+        }
         const shell = process.platform === 'win32' ? 'cmd.exe' : '/bin/sh';
         const shellFlag = process.platform === 'win32' ? '/c' : '-c';
         const child = this.processRunner.spawn(shell, [shellFlag, command], {
@@ -263,10 +376,18 @@ export class ServiceExecutor {
             const current = this.services.get(config.type);
             if (current && current.process === child && current.status !== 'stopped' && current.status !== 'failed') {
                 current.status = 'failed';
+                // The pid rides along on purpose — see the startup timeout below.
                 this.notifyStatusChanged(sessionId, config.type, 'failed', {
                     error: error.message,
                     cwd,
                     command,
+                    pid: current.pid,
+                    pidStartTime: current.pidStartTime,
+                });
+                void this.reapEntry(current, 'process error').then((reaped) => {
+                    if (!reaped.ok) {
+                        this.logger.error('Could not kill errored service process group', reaped.error, { serviceType: config.type, pid: current.pid });
+                    }
                 });
             }
         });
@@ -278,29 +399,137 @@ export class ServiceExecutor {
             this.notifyStatusChanged(sessionId, config.type, 'failed', { port, cwd, command, error: 'Failed to spawn process' });
             return Err({ message: 'Failed to spawn service process', recoverable: true });
         }
+        // Bind the narrowed pid: the deferred registry call below cannot carry the
+        // narrowing across the closure boundary.
+        const pid = child.pid;
+        const matches = (regex, content) => {
+            regex.lastIndex = 0;
+            const matched = regex.test(content);
+            regex.lastIndex = 0;
+            return matched;
+        };
+        const recordDiagnostic = (line, prefix) => {
+            const maxContentLength = Math.max(0, maxDiagnosticLength - prefix.length);
+            const bounded = line.length > maxContentLength ? line.slice(-maxContentLength) : line;
+            logs.push(`${prefix}${bounded}`);
+        };
+        // Detection sees the full incoming content before diagnostics are truncated.
+        const appendOutput = (partial, data, prefix) => {
+            const combined = partial + data.toString();
+            if (matches(PORT_CONFLICT_PATTERN, combined))
+                portConflictDetected = true;
+            const lines = combined.split('\n');
+            const nextPartial = lines.pop() ?? '';
+            const readyMatched = readyRegex ? matches(readyRegex, combined) || lines.some((line) => matches(readyRegex, line)) : false;
+            if (readyMatched && !setupComplete) {
+                readyDetectedDuringSetup = true;
+                readyLineDuringSetup = combined.slice(-maxDiagnosticLength);
+            }
+            for (const line of lines)
+                recordDiagnostic(line, prefix);
+            if (setupComplete) {
+                const current = this.services.get(config.type);
+                if (current?.process === child && current.status === 'starting') {
+                    this.logger.debug('Service output', { serviceType: config.type });
+                    if (readyMatched)
+                        markReady(combined.slice(-maxDiagnosticLength));
+                }
+                void checkReadyWhen();
+            }
+            // Retain enough suffix for markers split across chunks, never an entire line.
+            return nextPartial.length > maxDiagnosticLength ? nextPartial.slice(-maxDiagnosticLength) : nextPartial;
+        };
+        const onStdout = (data) => {
+            stdoutPartial = appendOutput(stdoutPartial, data, '');
+            stdoutPartialSequence = stdoutPartial ? ++outputSequence : 0;
+        };
+        const onStderr = (data) => {
+            stderrPartial = appendOutput(stderrPartial, data, '[stderr] ');
+            stderrPartialSequence = stderrPartial ? ++outputSequence : 0;
+        };
+        let exitDuringSetup;
+        let forgetPromise;
+        const forgetPid = () => {
+            if (forgetPromise)
+                return forgetPromise;
+            const previous = this.pidForgets.get(config.type);
+            const runForget = () => this.withDeadline(`Service '${config.type}' PID forget`, () => this.pidRegistry?.forget(String(sessionId), config.type) ?? Promise.resolve());
+            const operation = previous ? previous.then(runForget, runForget) : runForget();
+            forgetPromise = operation;
+            this.pidForgets.set(config.type, operation);
+            void operation.then(() => {
+                if (this.pidForgets.get(config.type) === operation)
+                    this.pidForgets.delete(config.type);
+            }, () => {
+                if (this.pidForgets.get(config.type) === operation)
+                    this.pidForgets.delete(config.type);
+                if (forgetPromise === operation)
+                    forgetPromise = undefined;
+            });
+            return operation;
+        };
+        const bufferClose = (code) => {
+            exitDuringSetup = { code };
+        };
+        child.stdout?.on('data', onStdout);
+        child.stderr?.on('data', onStderr);
+        child.on('close', bufferClose);
         // Capture start time immediately so a later PID-reuse check can distinguish
         // "our process" from "an unrelated process that grabbed this PID after ours died"
         const pidStartTime = await getProcessStartTime(this.fs, child.pid);
         // Durably record the process so a future agent can reap it if we die before the
         // `close` handler gets to forget it. The session projection cannot serve that
         // purpose — see ServicePidRegistry.
-        await this.pidRegistry?.record({ sessionId: String(sessionId), serviceType: config.type, pid: child.pid, pidStartTime, command });
+        try {
+            await this.withDeadline(`Service '${config.type}' PID record`, () => this.pidRegistry?.record({ sessionId: String(sessionId), serviceType: config.type, pid, pidStartTime, command })
+                ?? Promise.resolve());
+        }
+        catch (error) {
+            // The child is already running but unrecorded, so nothing could reap it
+            // after a crash — tear it down here rather than leak it.
+            const message = error instanceof Error ? error.message : String(error);
+            const errorMessage = `Service '${config.type}' could not record its PID: ${message}`;
+            try {
+                this.killProcess(-child.pid, 'SIGKILL');
+            }
+            catch {
+                // The process may already be gone.
+            }
+            if (allocatedNow) {
+                this.allocatedPorts.delete(config.type);
+                this.portPool.release(port);
+            }
+            this.notifyStatusChanged(sessionId, config.type, 'failed', { port, cwd, command, error: errorMessage });
+            return Err({ message: errorMessage, recoverable: true });
+        }
+        if (this.closing) {
+            try {
+                this.killProcess(-child.pid, 'SIGKILL');
+            }
+            catch {
+                // The process may already be gone.
+            }
+            await forgetPid();
+            if (allocatedNow) {
+                this.allocatedPorts.delete(config.type);
+                this.portPool.release(port);
+            }
+            return Err({ message: 'Service executor is closing', recoverable: false });
+        }
         // Emit starting event with PID, port, resolved cwd/command, and start time.
         this.notifyStatusChanged(sessionId, config.type, 'starting', { port, pid: child.pid, pidStartTime, cwd, command });
-        const logs = new RingBuffer(logBufferSize);
-        const startTime = Date.now();
-        // Set when the child reports a port bind-conflict; read by the close handler
-        // to decide between a fresh-port retry and a terminal failure.
-        let portConflictDetected = false;
         const entry = {
             config,
             process: child,
             pid: child.pid,
+            pidStartTime,
             status: 'starting',
             port,
             cwd,
             command,
             logs,
+            forgetPid,
+            exited: false,
         };
         this.services.set(config.type, entry);
         let startupTimer;
@@ -359,24 +588,6 @@ export class ServiceExecutor {
                 readyCheckInFlight = false;
             }
         };
-        const processLine = (line) => {
-            logs.push(line);
-            if (PORT_CONFLICT_PATTERN.test(line)) {
-                portConflictDetected = true;
-            }
-            const current = this.services.get(config.type);
-            if (!current || current.process !== child)
-                return;
-            if (current.status === 'starting') {
-                this.logger.debug('Service output', { serviceType: config.type, line });
-            }
-            if (readyRegex && current.status === 'starting') {
-                if (readyRegex.test(line)) {
-                    markReady(line);
-                }
-            }
-            void checkReadyWhen();
-        };
         // Startup timeout — mark as failed if not ready in time
         if (readyRegex || config.readyWhen) {
             startupTimer = setTimeout(() => {
@@ -387,19 +598,22 @@ export class ServiceExecutor {
                 const errorMsg = `Service startup timed out after ${startupTimeoutMs}ms`;
                 this.logger.error(errorMsg, undefined, { serviceType: config.type });
                 clearReadinessTimers();
+                // The pid rides along on purpose: should this agent die before the reap
+                // below completes, the recorded pid is all a later boot has to find the
+                // survivor. The child's exit takes the claim away again.
                 this.notifyStatusChanged(sessionId, config.type, 'failed', {
                     port: current.port,
                     cwd: current.cwd,
                     command: current.command,
                     error: errorMsg,
+                    pid: current.pid,
+                    pidStartTime: current.pidStartTime,
+                });
+                void this.reapEntry(current, 'startup timeout').then((reaped) => {
+                    if (!reaped.ok) {
+                        this.logger.error('Could not kill timed-out service process group', reaped.error, { serviceType: config.type, pid: current.pid });
+                    }
                 });
-                // Kill the timed-out process
-                try {
-                    process.kill(-current.pid, 'SIGKILL');
-                }
-                catch {
-                    // Already gone
-                }
             }, startupTimeoutMs);
         }
         if (config.readyWhen) {
@@ -407,52 +621,38 @@ export class ServiceExecutor {
             readyCheckTimer = setInterval(() => {
                 void checkReadyWhen();
             }, intervalMs);
-            void checkReadyWhen();
         }
-        // Pipe stdout/stderr line by line
-        let stdoutPartial = '';
-        child.stdout?.on('data', (data) => {
-            stdoutPartial += data.toString();
-            const lines = stdoutPartial.split('\n');
-            stdoutPartial = lines.pop();
-            for (const line of lines) {
-                processLine(line);
-            }
-        });
-        let stderrPartial = '';
-        child.stderr?.on('data', (data) => {
-            stderrPartial += data.toString();
-            const lines = stderrPartial.split('\n');
-            stderrPartial = lines.pop();
-            for (const line of lines) {
-                processLine(`[stderr] ${line}`);
-            }
-        });
-        // Handle unexpected exit
-        child.on('close', (code) => {
+        // Handle unexpected exit. Named and guarded because it also has to be
+        // replayable — see the exit-during-setup check after markReady() below.
+        let closeHandled = false;
+        const handleClose = (code) => {
+            if (closeHandled)
+                return;
+            closeHandled = true;
+            entry.exited = true;
             clearReadinessTimers();
-            // The process is gone, so its durable record has nothing left to reap.
-            void this.pidRegistry?.forget(String(sessionId), config.type);
-            // Flush remaining partial lines
-            if (stdoutPartial) {
-                processLine(stdoutPartial);
-                stdoutPartial = '';
-            }
-            if (stderrPartial) {
-                processLine(`[stderr] ${stderrPartial}`);
-                stderrPartial = '';
+            // A replacement must not be recorded until this deletion finishes.
+            void forgetPid();
+            const partials = [
+                { content: stdoutPartial, prefix: '', sequence: stdoutPartialSequence },
+                { content: stderrPartial, prefix: '[stderr] ', sequence: stderrPartialSequence },
+            ];
+            for (const partial of partials.filter(({ content }) => content).sort((left, right) => left.sequence - right.sequence)) {
+                recordDiagnostic(partial.content, partial.prefix);
             }
+            stdoutPartial = '';
+            stderrPartial = '';
             const current = this.services.get(config.type);
             if (!current || current.process !== child)
                 return;
             if (current.status === 'stopping') {
-                // Expected stop
+                // Expected stop — the child's exit, not stopInternal, is what publishes it.
                 current.status = 'stopped';
-                this.notifyStatusChanged(sessionId, config.type, 'stopped');
+                this.notifyStatusChanged(sessionId, config.type, 'stopped', { stoppedBy: current.stopSource ?? 'agent' });
             }
             else if (current.status === 'starting' || current.status === 'ready') {
                 const retries = this.portConflictRetries.get(config.type) ?? 0;
-                if (portConflictDetected && retries < MAX_PORT_CONFLICT_RETRIES) {
+                if (!this.closing && current.status === 'starting' && portConflictDetected && retries < MAX_PORT_CONFLICT_RETRIES) {
                     // The chosen port is held by a foreign process (e.g. a service
                     // leaked from another session, or a survivor of a previous boot).
                     // Re-allocate a fresh port and retry so the service recovers on a
@@ -469,7 +669,18 @@ export class ServiceExecutor {
                     });
                     this.allocatedPorts.delete(config.type);
                     this.services.delete(config.type);
-                    void this.start(config, sessionId, workspaceDir);
+                    if (!setupComplete) {
+                        retryPortConflictDuringSetup = true;
+                    }
+                    else {
+                        const retryTimer = setTimeout(() => {
+                            this.portConflictTimers.delete(config.type);
+                            if (!this.closing)
+                                void this.start(config, sessionId, workspaceDir);
+                        }, 0);
+                        retryTimer.unref?.();
+                        this.portConflictTimers.set(config.type, retryTimer);
+                    }
                     return;
                 }
                 // Unexpected exit
@@ -491,7 +702,16 @@ export class ServiceExecutor {
                     code,
                 });
             }
-        });
+            else if (current.status === 'failed') {
+                // The process this entry still claimed is now confirmed gone — a startup
+                // timeout or a process error published the pid without being able to
+                // confirm the kill. Take the claim away so no later boot goes looking for
+                // it; the status and the error that explain the failure stay as they are.
+                this.notifyStatusChanged(sessionId, config.type, 'failed');
+            }
+        };
+        child.on('close', handleClose);
+        child.off('close', bufferClose);
         this.logger.info('Service starting', {
             serviceType: config.type,
             pid: child.pid,
@@ -501,10 +721,37 @@ export class ServiceExecutor {
             readyPattern: config.readyPattern,
             startupTimeoutMs,
         });
-        // If no ready pattern, immediately mark as ready
-        if (!readyRegex && !config.readyWhen) {
+        // A service that died inside the bookkeeping above closed while only the
+        // setup close collector was listening, so replay that close now.
+        // `alreadyReaped` covers the in-between state: the exit is recorded but
+        // 'close' has not fired yet because it waits for the stdio EOF. The listener
+        // above is guaranteed to receive it, so calling handleClose here would only
+        // drop the tail of the log the failure has to explain — while marking such a
+        // child ready would be a lie.
+        const alreadyReaped = child.exitCode != null || child.signalCode != null;
+        if (exitDuringSetup) {
+            handleClose(exitDuringSetup.code);
+        }
+        setupComplete = true;
+        if (retryPortConflictDuringSetup) {
+            await forgetPid();
+            if (this.closing)
+                return Err({ message: 'Service executor is closing', recoverable: false });
+            return this.startInternal(config, sessionId, workspaceDir);
+        }
+        if (exitDuringSetup || alreadyReaped) {
+            return Ok(undefined);
+        }
+        if (readyDetectedDuringSetup) {
+            markReady(readyLineDuringSetup);
+        }
+        else if (!readyRegex && !config.readyWhen) {
+            // If no ready condition is configured, a live child is ready immediately.
             markReady();
         }
+        else if (config.readyWhen) {
+            void checkReadyWhen();
+        }
         return Ok(undefined);
     }
     /**
@@ -515,6 +762,8 @@ export class ServiceExecutor {
      * change, or undefined when the service is left failed for good.
      */
     scheduleRestart(config, sessionId, workspaceDir, uptimeMs, exitCode) {
+        if (this.closing)
+            return undefined;
         const policy = config.restartPolicy;
         if (!policy)
             return undefined;
@@ -546,7 +795,8 @@ export class ServiceExecutor {
         });
         const timer = setTimeout(() => {
             this.restartTimers.delete(config.type);
-            void this.start(config, sessionId, workspaceDir);
+            if (!this.closing)
+                void this.start(config, sessionId, workspaceDir);
         }, delayMs);
         // A pending revival must not keep the process alive on its own.
         timer.unref?.();
@@ -562,69 +812,379 @@ export class ServiceExecutor {
         this.restartTimers.delete(serviceType);
         return true;
     }
-    /** Whether a `restartPolicy` revival is queued — the service is down but coming back. */
+    /**
+     * Whether the service is down but coming back on its own — a `restartPolicy`
+     * revival or a queued port-conflict retry. Both are timers this executor
+     * owns, so both need the runtime to outlive the call that armed them.
+     */
     hasScheduledRestart(serviceType) {
-        return this.restartTimers.has(serviceType);
+        return this.restartTimers.has(serviceType) || this.portConflictTimers.has(serviceType);
+    }
+    /**
+     * Stop accepting work and drain all runtime-owned processes and timers.
+     *
+     * `reason: 'evicted'` only parks the session: every service keeps its full
+     * graceful window and is recorded as stopped by the runtime, which is what
+     * lets autoStart bring it back once the runtime is rebuilt. The other
+     * reasons end the session, so a process group that ignores SIGTERM is taken
+     * down without waiting.
+     */
+    close(sessionId, reason = 'closed') {
+        this.closing = true;
+        this.closePromise ??= this.performClose(sessionId, reason).finally(() => {
+            this.onStatusChanged = undefined;
+            this.onStartSettled = undefined;
+        });
+        return this.closePromise;
+    }
+    async performClose(sessionId, reason) {
+        this.cancelPortConflictTimers();
+        await Promise.allSettled([...this.startInFlight.values()]);
+        this.cancelPortConflictTimers();
+        const concurrentStopTypes = new Set(this.stopInFlight.keys());
+        await Promise.allSettled([...this.stopInFlight.values()]);
+        // One service that cannot be signalled must not strand the others, and must not
+        // skip the tail — portPool is process-global, so a skipped release loses those
+        // ports for the life of the server.
+        const failures = [];
+        const parking = reason === 'evicted';
+        try {
+            for (const [serviceType, entry] of [...this.services]) {
+                if (entry.status === 'stopped')
+                    continue;
+                if (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'stopping' || this.restartTimers.has(serviceType)) {
+                    const forcedGraceMs = parking ? entry.config.gracefulStopMs ?? 5000 : 0;
+                    try {
+                        if (concurrentStopTypes.has(serviceType)) {
+                            await this.forceStopForClose(serviceType, entry, sessionId, forcedGraceMs);
+                            continue;
+                        }
+                        const stopped = await this.stop(serviceType, sessionId, 'eviction');
+                        if (!stopped.ok)
+                            await this.forceStopForClose(serviceType, entry, sessionId, forcedGraceMs);
+                    }
+                    catch (error) {
+                        failures.push(error);
+                    }
+                }
+            }
+        }
+        finally {
+            this.cancelAllDeferredStarts();
+            await Promise.allSettled([...this.pidForgets.values()]);
+            for (const port of this.allocatedPorts.values())
+                this.portPool.release(port);
+            this.allocatedPorts.clear();
+            this.services.clear();
+            for (const serviceType of this.waiters.keys())
+                this.resolveWaiters(serviceType, 'stopped');
+        }
+        // Still reject — a close that could not terminate a process is not a clean close.
+        if (failures.length === 1)
+            throw failures[0];
+        if (failures.length > 1) {
+            throw new AggregateError(failures, `Failed to stop ${failures.length} services during close`);
+        }
+    }
+    async forceStopForClose(serviceType, entry, sessionId, gracefulStopMs) {
+        const previousStatus = entry.status;
+        this.recordStopSource(entry, 'eviction');
+        if (entry.status !== 'stopping') {
+            entry.status = 'stopping';
+            this.notifyStatusChanged(sessionId, serviceType, 'stopping');
+        }
+        const stopped = await this.terminateProcessGroup(entry.pid, gracefulStopMs);
+        if (!stopped.ok) {
+            // Mirror stopInternal: no recovery path matches a stranded `stopping`, so a
+            // force-stop that failed has to leave a status the next runtime can reap.
+            if (entry.status === 'stopping' && previousStatus !== 'stopping') {
+                entry.status = previousStatus;
+                this.notifyStatusChanged(sessionId, serviceType, previousStatus, {
+                    port: entry.port,
+                    cwd: entry.cwd,
+                    command: entry.command,
+                    error: stopped.error.message,
+                });
+            }
+            throw stopped.error;
+        }
+        // The group is confirmed gone, so nothing may offer this pid up for a reap again.
+        entry.exited = true;
+        await entry.forgetPid();
+        if (this.services.get(serviceType) === entry) {
+            entry.status = 'stopped';
+            this.notifyStatusChanged(sessionId, serviceType, 'stopped', { stoppedBy: entry.stopSource ?? 'eviction' });
+        }
+    }
+    /** An agent stop stays the agent's decision even when a lifecycle stop is what finishes it. */
+    recordStopSource(entry, stoppedBy) {
+        if (stoppedBy === 'agent' || entry.stopSource === undefined)
+            entry.stopSource = stoppedBy;
+    }
+    cancelAllDeferredStarts() {
+        for (const timer of this.restartTimers.values())
+            clearTimeout(timer);
+        this.restartTimers.clear();
+        this.cancelPortConflictTimers();
+    }
+    cancelPortConflictTimers() {
+        for (const timer of this.portConflictTimers.values())
+            clearTimeout(timer);
+        this.portConflictTimers.clear();
+    }
+    errorWithContext(error, context) {
+        const cause = error instanceof Error ? error.message : String(error);
+        return new Error(`${context}: ${cause}`);
+    }
+    hasErrorCode(error, code) {
+        return error instanceof Error && 'code' in error && error.code === code;
+    }
+    async hasNonZombieProcessGroupMember(pid) {
+        if (process.platform !== 'linux')
+            return true;
+        let entries;
+        try {
+            entries = await this.fs.readdir('/proc');
+        }
+        catch {
+            return true;
+        }
+        let foundGroupMember = false;
+        for (const entry of entries) {
+            if (!/^\d+$/.test(entry))
+                continue;
+            try {
+                const stat = await this.fs.readFile(`/proc/${entry}/stat`, 'utf-8');
+                const rparen = stat.lastIndexOf(')');
+                if (rparen === -1)
+                    continue;
+                const fields = stat.slice(rparen + 2).split(' ');
+                if (Number(fields[2]) !== pid)
+                    continue;
+                foundGroupMember = true;
+                if (fields[0] !== 'Z')
+                    return true;
+            }
+            catch {
+                // Processes can disappear while /proc is scanned.
+            }
+        }
+        // A successful group probe with no readable member is still evidence of life.
+        return !foundGroupMember;
+    }
+    async probeProcessGroup(pid) {
+        try {
+            this.killProcess(-pid, 0);
+        }
+        catch (error) {
+            if (this.hasErrorCode(error, 'ESRCH'))
+                return { state: 'gone' };
+            if (this.hasErrorCode(error, 'EPERM'))
+                return { state: 'alive' };
+            return { state: 'error', error: this.errorWithContext(error, `Failed to probe process group ${pid}`) };
+        }
+        return await this.hasNonZombieProcessGroupMember(pid) ? { state: 'alive' } : { state: 'gone' };
+    }
+    async signalProcessGroup(pid, signal) {
+        try {
+            this.killProcess(-pid, signal);
+            return { state: 'alive' };
+        }
+        catch (error) {
+            if (this.hasErrorCode(error, 'ESRCH'))
+                return { state: 'gone' };
+            return { state: 'error', error: this.errorWithContext(error, `Failed to send ${signal} to process group ${pid}`) };
+        }
+    }
+    async waitForProcessGroupExit(pid, timeoutMs) {
+        const initial = await this.probeProcessGroup(pid);
+        if (initial.state !== 'alive')
+            return initial;
+        return new Promise((resolve) => {
+            let settled = false;
+            const finish = (result) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearInterval(checkInterval);
+                clearTimeout(timeout);
+                resolve(result);
+            };
+            const checkInterval = setInterval(() => {
+                void this.probeProcessGroup(pid).then((result) => {
+                    if (result.state !== 'alive')
+                        finish(result);
+                });
+            }, 50);
+            const timeout = setTimeout(() => {
+                void this.probeProcessGroup(pid).then(finish);
+            }, timeoutMs);
+        });
+    }
+    /**
+     * Reclaim a service process group we spawned, unless the kernel recycled its pid.
+     *
+     * Every path that gives up on a service comes through here, because ending the
+     * *wait* is not ending the *process*: a dev server that missed its readiness
+     * window keeps its several hundred MB of RSS until somebody kills it, which on a
+     * memory-starved box is what makes the next startup miss its window too.
+     *
+     * `pidStartTime` was captured at spawn, so a different current start time means
+     * the pid now belongs to an unrelated process and killing it would be disastrous.
+     * An unknown start time (non-Linux, no /proc) leaves only the kill-and-hope path.
+     */
+    async reapProcessGroup(target, context = {}) {
+        const currentStartTime = await getProcessStartTime(this.fs, target.pid);
+        if (target.pidStartTime !== undefined && currentStartTime !== undefined && currentStartTime !== target.pidStartTime) {
+            this.logger.warn('PID reuse detected — refusing to kill service process group', {
+                ...context,
+                pid: target.pid,
+                storedStartTime: target.pidStartTime,
+                currentStartTime,
+            });
+            return Ok('pid-reused');
+        }
+        const killed = this.killProcessGroup(target.pid);
+        if (!killed.ok)
+            return killed;
+        return Ok('killed');
+    }
+    /**
+     * Reclaim the process group of an entry we are giving up on or replacing.
+     *
+     * A no-op once the child's `close` has fired: there is nothing left to reclaim,
+     * and skipping it also keeps the non-Linux kill-and-hope path away from a pid the
+     * kernel may since have handed to somebody else. A settled reap ends our claim,
+     * so the entry is never offered up twice.
+     */
+    async reapEntry(entry, reason) {
+        if (entry.exited)
+            return Ok(undefined);
+        const reaped = await this.reapProcessGroup({ pid: entry.pid, pidStartTime: entry.pidStartTime }, { serviceType: entry.config.type, reason });
+        if (!reaped.ok)
+            return reaped;
+        entry.exited = true;
+        return Ok(undefined);
+    }
+    /**
+     * SIGKILL a process group through the injectable kill seam, without waiting
+     * for it to go. Reached through {@link reapProcessGroup} wherever a recorded
+     * pid could have been recycled.
+     */
+    killProcessGroup(pid) {
+        try {
+            this.killProcess(-pid, 'SIGKILL');
+            return Ok(undefined);
+        }
+        catch (error) {
+            if (this.hasErrorCode(error, 'ESRCH'))
+                return Ok(undefined); // Already gone.
+            return Err(this.errorWithContext(error, `Failed to SIGKILL process group ${pid}`));
+        }
+    }
+    async terminateProcessGroup(pid, gracefulStopMs) {
+        const initial = await this.probeProcessGroup(pid);
+        if (initial.state === 'gone')
+            return Ok(undefined);
+        if (initial.state === 'error')
+            return Err(initial.error);
+        const term = await this.signalProcessGroup(pid, 'SIGTERM');
+        if (term.state === 'gone')
+            return Ok(undefined);
+        if (term.state === 'error')
+            return Err(term.error);
+        const graceful = await this.waitForProcessGroupExit(pid, gracefulStopMs);
+        if (graceful.state === 'gone')
+            return Ok(undefined);
+        if (graceful.state === 'error')
+            return Err(graceful.error);
+        const kill = await this.signalProcessGroup(pid, 'SIGKILL');
+        if (kill.state === 'gone')
+            return Ok(undefined);
+        if (kill.state === 'error')
+            return Err(kill.error);
+        const forced = await this.waitForProcessGroupExit(pid, 5000);
+        if (forced.state === 'gone')
+            return Ok(undefined);
+        if (forced.state === 'error')
+            return Err(forced.error);
+        return Err(new Error(`Process group ${pid} survived SIGKILL`));
     }
     /**
      * Stop a running service gracefully.
      * Port is NOT released — kept for session-level stability across restarts.
      */
-    async stop(serviceType, sessionId) {
+    async stop(serviceType, sessionId, stoppedBy = 'agent') {
+        const inFlight = this.stopInFlight.get(serviceType);
+        if (inFlight)
+            return inFlight;
+        let operation;
+        operation = this.stopInternal(serviceType, sessionId, stoppedBy).finally(() => {
+            if (this.stopInFlight.get(serviceType) === operation)
+                this.stopInFlight.delete(serviceType);
+        });
+        this.stopInFlight.set(serviceType, operation);
+        return operation;
+    }
+    async stopInternal(serviceType, sessionId, stoppedBy) {
         const hadPendingRestart = this.cancelPendingRestart(serviceType);
         this.restartRetries.delete(serviceType);
         const entry = this.services.get(serviceType);
         if (!entry) {
             return Err({ message: `Service '${serviceType}' not found`, recoverable: false });
         }
-        if (entry.status !== 'starting' && entry.status !== 'ready' && entry.status !== 'paused') {
-            // A failed service with a revival queued is really "about to restart",
-            // and calling that off is a legitimate stop rather than an error.
-            if (hadPendingRestart) {
+        if (entry.status !== 'starting' && entry.status !== 'ready') {
+            // `failed` says the start gave up, not that the process did — and refusing to
+            // stop it left the platform-side reaper, the one caller that can free a leaked
+            // dev server's memory, with no way to do so. A failed service with a revival
+            // queued is on top of that really "about to restart", and calling that off is a
+            // legitimate stop rather than an error.
+            if (entry.status === 'failed' || hadPendingRestart) {
+                const reaped = await this.reapEntry(entry, 'stopped while failed');
+                if (!reaped.ok)
+                    return Err({ message: reaped.error.message, recoverable: true });
+                try {
+                    await entry.forgetPid();
+                }
+                catch (error) {
+                    return Err({ message: this.errorWithContext(error, `Failed to forget PID for service '${serviceType}'`).message, recoverable: true });
+                }
                 entry.status = 'stopped';
-                this.notifyStatusChanged(sessionId, serviceType, 'stopped');
+                this.notifyStatusChanged(sessionId, serviceType, 'stopped', { stoppedBy });
                 return Ok(undefined);
             }
             return Err({ message: `Service '${serviceType}' is ${entry.status}, cannot stop`, recoverable: false });
         }
+        const previousStatus = entry.status;
+        this.recordStopSource(entry, stoppedBy);
         entry.status = 'stopping';
         this.notifyStatusChanged(sessionId, serviceType, 'stopping');
         const gracefulStopMs = entry.config.gracefulStopMs ?? 5000;
-        // Send SIGTERM
+        const stopped = await this.terminateProcessGroup(entry.pid, gracefulStopMs);
+        if (!stopped.ok) {
+            if (entry.status === 'stopping') {
+                entry.status = previousStatus;
+                this.notifyStatusChanged(sessionId, serviceType, previousStatus, {
+                    port: entry.port,
+                    cwd: entry.cwd,
+                    command: entry.command,
+                    error: stopped.error.message,
+                });
+            }
+            return Err({ message: stopped.error.message, recoverable: true });
+        }
+        // The group is confirmed gone, so nothing may offer this pid up for a reap again.
+        entry.exited = true;
         try {
-            process.kill(-entry.pid, 'SIGTERM');
+            await entry.forgetPid();
         }
-        catch {
-            // Process already gone
+        catch (error) {
+            return Err({ message: this.errorWithContext(error, `Failed to forget PID for service '${serviceType}'`).message, recoverable: true });
+        }
+        if (entry.status === 'stopping') {
             entry.status = 'stopped';
-            this.notifyStatusChanged(sessionId, serviceType, 'stopped');
-            return Ok(undefined);
+            this.notifyStatusChanged(sessionId, serviceType, 'stopped', { stoppedBy: entry.stopSource ?? stoppedBy });
         }
-        // Wait for graceful shutdown, then SIGKILL
-        await new Promise((resolve) => {
-            const checkInterval = setInterval(() => {
-                try {
-                    // Check if process is still alive (signal 0 doesn't kill, just checks)
-                    process.kill(entry.pid, 0);
-                }
-                catch {
-                    // Process gone
-                    clearInterval(checkInterval);
-                    clearTimeout(killTimeout);
-                    resolve();
-                }
-            }, 200);
-            const killTimeout = setTimeout(() => {
-                clearInterval(checkInterval);
-                try {
-                    process.kill(-entry.pid, 'SIGKILL');
-                }
-                catch {
-                    // Already gone
-                }
-                resolve();
-            }, gracefulStopMs);
-        });
         this.logger.info('Service stopped', { serviceType });
         return Ok(undefined);
     }
@@ -633,61 +1193,13 @@ export class ServiceExecutor {
      */
     async restart(config, sessionId, workspaceDir, preferredPort) {
         const entry = this.services.get(config.type);
-        if (entry && (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'paused')) {
+        if (entry && (entry.status === 'starting' || entry.status === 'ready')) {
             const stopResult = await this.stop(config.type, sessionId);
             if (!stopResult.ok)
                 return stopResult;
         }
         return this.start(config, sessionId, workspaceDir, preferredPort);
     }
-    /**
-     * Pause a running service (SIGSTOP).
-     */
-    async pause(serviceType, sessionId) {
-        const entry = this.services.get(serviceType);
-        if (!entry) {
-            return Err({ message: `Service '${serviceType}' not found`, recoverable: false });
-        }
-        if (entry.status !== 'ready') {
-            return Err({ message: `Service '${serviceType}' is ${entry.status}, cannot pause`, recoverable: false });
-        }
-        try {
-            process.kill(entry.pid, 'SIGSTOP');
-        }
-        catch {
-            return Err({ message: `Failed to pause service '${serviceType}'`, recoverable: false });
-        }
-        entry.status = 'paused';
-        this.notifyStatusChanged(sessionId, serviceType, 'paused');
-        this.logger.info('Service paused', { serviceType });
-        return Ok(undefined);
-    }
-    /**
-     * Resume a paused service (SIGCONT).
-     */
-    async resume(config, sessionId, _workspaceDir) {
-        const entry = this.services.get(config.type);
-        if (!entry) {
-            return Err({ message: `Service '${config.type}' not found`, recoverable: false });
-        }
-        if (entry.status !== 'paused') {
-            return Err({ message: `Service '${config.type}' is ${entry.status}, cannot resume`, recoverable: false });
-        }
-        try {
-            process.kill(entry.pid, 'SIGCONT');
-        }
-        catch {
-            return Err({ message: `Failed to resume service '${config.type}'`, recoverable: false });
-        }
-        entry.status = 'ready';
-        this.notifyStatusChanged(sessionId, config.type, 'ready', {
-            port: entry.port,
-            cwd: entry.cwd,
-            command: entry.command,
-        });
-        this.logger.info('Service resumed', { serviceType: config.type });
-        return Ok(undefined);
-    }
     /**
      * Get recent log lines for a service.
      */
@@ -711,67 +1223,5 @@ export class ServiceExecutor {
         const status = this.getStatus(serviceType);
         return status === 'starting' || status === 'ready';
     }
-    /**
-     * Shutdown all services and release all ports back to pool.
-     * Called on session close.
-     */
-    async shutdown() {
-        // Nothing queued may spawn after the executor is gone.
-        for (const serviceType of [...this.restartTimers.keys()])
-            this.cancelPendingRestart(serviceType);
-        this.restartRetries.clear();
-        const promises = [];
-        for (const [serviceType, entry] of this.services) {
-            if (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'paused') {
-                const gracefulStopMs = entry.config.gracefulStopMs ?? 5000;
-                const killPromise = new Promise((resolve) => {
-                    try {
-                        process.kill(-entry.pid, 'SIGTERM');
-                    }
-                    catch {
-                        resolve();
-                        return;
-                    }
-                    const killTimeout = setTimeout(() => {
-                        try {
-                            process.kill(-entry.pid, 'SIGKILL');
-                        }
-                        catch {
-                            // Already gone
-                        }
-                        resolve();
-                    }, gracefulStopMs);
-                    const checkInterval = setInterval(() => {
-                        try {
-                            process.kill(entry.pid, 0);
-                        }
-                        catch {
-                            clearInterval(checkInterval);
-                            clearTimeout(killTimeout);
-                            resolve();
-                        }
-                    }, 200);
-                });
-                promises.push(killPromise);
-                entry.status = 'stopping';
-                this.logger.info('Shutting down service', { serviceType });
-            }
-        }
-        await Promise.all(promises);
-        this.services.clear();
-        // Drain all waiters with error
-        for (const [serviceType, pending] of this.waiters) {
-            for (const waiter of pending) {
-                clearTimeout(waiter.timer);
-                waiter.resolve(Err({ message: `Service '${serviceType}' shut down`, recoverable: false }));
-            }
-        }
-        this.waiters.clear();
-        // Release all allocated ports back to pool
-        for (const port of this.allocatedPorts.values()) {
-            this.portPool.release(port);
-        }
-        this.allocatedPorts.clear();
-    }
 }
 //# sourceMappingURL=service.js.map