@runcore-sh/runcore 0.1.2

package/dist/agents/runtime/manager.js

@@ -0,0 +1,947 @@
+/**
+ * Agent Runtime Environment — RuntimeManager.
+ *
+ * The central orchestrator that ties together lifecycle management,
+ * resource allocation, the event bus, registry persistence, and the
+ * agent driver. Provides the high-level API for spawning, pausing,
+ * resuming, and terminating agent instances.
+ *
+ * Usage:
+ *   const manager = new RuntimeManager(driver, configOverrides);
+ *   await manager.init();
+ *   const instance = await manager.spawn({ taskId, label, prompt, origin });
+ *   await manager.pause(instance.id);
+ *   await manager.resume(instance.id);
+ *   await manager.terminate(instance.id);
+ *   await manager.shutdown();
+ */
+import { RuntimeError, ErrorCodes } from "./errors.js";
+import { loadRuntimeConfig, resolveInstanceConfig, resolveResources } from "./config.js";
+import { transition, shouldRetry, prepareRetry, isTerminal } from "./lifecycle.js";
+import { ResourcePool } from "./resources.js";
+import { RuntimeBus } from "./bus.js";
+import { AgentRegistry } from "./registry.js";
+import { logActivity } from "../../activity/log.js";
+import { createLogger } from "../../utils/logger.js";
+import { rememberTaskOutcome } from "../memory.js";
+import { readTask, readTaskOutput, updateTask } from "../store.js";
+import { updateBoardTaskState } from "../spawn.js";
+const log = createLogger("agent-runtime");
+// ---------------------------------------------------------------------------
+// RuntimeManager
+// ---------------------------------------------------------------------------
+export class RuntimeManager {
+    config;
+    bus;
+    resources;
+    registry;
+    driver;
+    monitorTimer = null;
+    shutdownRequested = false;
+    // ── Retry loop guards ──────────────────────────────────────────────────
+    /** Prevents concurrent handleExit calls for the same instance. */
+    handleExitGuard = new Set();
+    /** Tracks instances with a pending retry timeout (prevents duplicate scheduling). */
+    retryPending = new Set();
+    // ── Global retry budget ────────────────────────────────────────────────
+    /** Rolling window of retry timestamps across all instances. */
+    globalRetryTimestamps = [];
+    /**
+     * Max retries across all instances within the budget window.
+     * Scaled to 3× maxConcurrentAgents to avoid budget exhaustion when a full
+     * batch fails (each agent gets maxRetries=2, so 5 agents = 10 retries
+     * which previously exhausted the entire budget in one batch).
+     */
+    globalRetryBudget;
+    /** Rolling window for the global retry budget (ms). */
+    static GLOBAL_RETRY_WINDOW_MS = 5 * 60_000; // 5 minutes
+    constructor(driver, configOverrides) {
+        this.config = loadRuntimeConfig(configOverrides);
+        this.driver = driver;
+        this.bus = new RuntimeBus();
+        this.resources = new ResourcePool(this.config);
+        this.registry = new AgentRegistry(this.config.persistDir);
+        // Scale retry budget: 3× max concurrent agents ensures one bad batch
+        // doesn't exhaust the entire budget (5 agents × 2 retries = 10, budget = 15)
+        this.globalRetryBudget = Math.max(10, this.config.maxConcurrentAgents * 3);
+    }
+    // -------------------------------------------------------------------------
+    // Initialization & shutdown
+    // -------------------------------------------------------------------------
+    /** Initialize the runtime: load registry, recover agents, start monitor. */
+    async init() {
+        const t0 = Date.now();
+        log.info("Initializing agent runtime", { driver: this.driver.name, maxAgents: this.config.maxConcurrentAgents });
+        const t1 = Date.now();
+        await this.registry.init();
+        const registryMs = Date.now() - t1;
+        const t2 = Date.now();
+        await this.recoverAgents();
+        const recoveryMs = Date.now() - t2;
+        this.startMonitor();
+        const totalMs = Date.now() - t0;
+        logActivity({
+            source: "agent",
+            summary: "Agent runtime initialized",
+            detail: `Driver: ${this.driver.name}, max agents: ${this.config.maxConcurrentAgents}, init ${totalMs}ms [registry:${registryMs}ms recovery:${recoveryMs}ms]`,
+        });
+    }
+    /** Graceful shutdown: terminate active agents, stop monitor, clean up. */
+    async shutdown(reason = "Runtime shutdown") {
+        if (this.shutdownRequested)
+            return;
+        log.info("Shutting down agent runtime", { reason });
+        this.shutdownRequested = true;
+        this.bus.emitShutdown(reason);
+        this.stopMonitor();
+        // Terminate all active agents
+        const active = this.registry.listActive();
+        const terminations = active.map((inst) => this.terminate(inst.id, reason).catch(() => { }));
+        await Promise.allSettled(terminations);
+        // Clean up resources, retry guards, and bus
+        this.resources.clear();
+        this.handleExitGuard.clear();
+        this.retryPending.clear();
+        this.globalRetryTimestamps.length = 0;
+        this.bus.destroy();
+        logActivity({
+            source: "agent",
+            summary: "Agent runtime shut down",
+            detail: `Reason: ${reason}, terminated ${active.length} agents`,
+        });
+    }
+    // -------------------------------------------------------------------------
+    // Core lifecycle operations
+    // -------------------------------------------------------------------------
+    /** Spawn a new agent instance. */
+    async spawn(request) {
+        log.info(`Spawning agent instance: ${request.label}`, { taskId: request.taskId, origin: request.origin });
+        if (this.shutdownRequested) {
+            throw new RuntimeError(ErrorCodes.SHUTDOWN_IN_PROGRESS, "Runtime is shutting down");
+        }
+        const instanceConfig = resolveInstanceConfig(this.config, request.config);
+        const resourceAlloc = resolveResources(this.config, request.resources);
+        // Check resource availability — queue if not available
+        if (!this.resources.canAllocate(resourceAlloc)) {
+            logActivity({
+                source: "agent",
+                summary: `Agent queued (resources exhausted): ${request.label}`,
+                detail: `Active: ${this.resources.activeCount}/${this.config.maxConcurrentAgents}`,
+            });
+            await this.resources.enqueue(request, instanceConfig.priority);
+        }
+        // Create the instance
+        const now = new Date().toISOString();
+        const instance = {
+            id: this.registry.generateId(),
+            taskId: request.taskId,
+            state: "initializing",
+            config: instanceConfig,
+            resources: resourceAlloc,
+            metadata: {
+                label: request.label,
+                origin: request.origin,
+                parentId: request.parentId,
+                tags: request.tags ?? [],
+            },
+            retryCount: 0,
+            createdAt: now,
+            updatedAt: now,
+        };
+        // Store the prompt on the instance for the driver
+        instance._prompt = request.prompt;
+        // Register and allocate resources
+        await this.registry.register(instance);
+        this.resources.allocate(instance.id, resourceAlloc);
+        logActivity({
+            source: "agent",
+            summary: `Spawning agent: ${request.label}`,
+            detail: `Instance ${instance.id}, task ${request.taskId}`,
+        });
+        // Attempt to spawn via driver
+        try {
+            const pid = await this.driver.spawn(instance);
+            instance.pid = pid;
+            const event = transition(instance, "running", "Spawned successfully");
+            await this.registry.update(instance.id, {
+                state: instance.state,
+                pid,
+                updatedAt: instance.updatedAt,
+            });
+            this.bus.emitLifecycle(event);
+            this.bus.emitSpawned(instance.id, pid);
+            // Register exit handler with driver
+            if ("onExit" in this.driver && typeof this.driver.onExit === "function") {
+                this.driver
+                    .onExit(instance.id, (code) => this.handleExit(instance.id, code));
+            }
+            logActivity({
+                source: "agent",
+                summary: `Agent running: ${request.label}`,
+                detail: `PID ${pid}, instance ${instance.id}`,
+            });
+            return instance;
+        }
+        catch (err) {
+            const error = this.toAgentError(err, ErrorCodes.SPAWN_FAILED, true);
+            const event = transition(instance, "failed", "Spawn failed", error);
+            await this.registry.update(instance.id, {
+                state: "failed",
+                error,
+                updatedAt: instance.updatedAt,
+            });
+            this.resources.release(instance.id);
+            this.bus.emitLifecycle(event);
+            // Check if retry is possible BEFORE emitting terminal failure.
+            // Same fix as handleExit: prevent external listeners from firing
+            // prematurely while retries are still pending.
+            const retryDelay = shouldRetry(instance);
+            if (retryDelay !== null) {
+                logActivity({
+                    source: "agent",
+                    summary: `Agent spawn failed, will retry: ${request.label}`,
+                    detail: `Attempt ${instance.retryCount + 1}/${instance.config.maxRetries}, backoff ${retryDelay}ms`,
+                });
+                await this.maybeRetry(instance);
+            }
+            else {
+                // Terminal failure — notify external listeners
+                this.bus.emitFailed(instance.id, error);
+                logActivity({
+                    source: "agent",
+                    summary: `Agent spawn failed (terminal): ${request.label}`,
+                    detail: err.message,
+                });
+            }
+            return instance;
+        }
+    }
+    /** Pause a running agent. */
+    async pause(instanceId, reason) {
+        const instance = this.requireInstance(instanceId);
+        this.assertState(instance, "running");
+        let checkpoint;
+        try {
+            checkpoint = await this.driver.pause(instance);
+        }
+        catch (err) {
+            logActivity({
+                source: "agent",
+                summary: `Agent pause failed: ${instance.metadata.label}`,
+                detail: err.message,
+            });
+            throw err;
+        }
+        if (checkpoint) {
+            instance.checkpointData = checkpoint;
+        }
+        const event = transition(instance, "paused", reason ?? "Paused by request");
+        await this.registry.update(instanceId, {
+            state: "paused",
+            checkpointData: instance.checkpointData,
+            pausedAt: instance.pausedAt,
+            updatedAt: instance.updatedAt,
+        });
+        this.resources.release(instanceId);
+        this.bus.emitLifecycle(event);
+        logActivity({
+            source: "agent",
+            summary: `Agent paused: ${instance.metadata.label}`,
+            detail: checkpoint ? "Checkpoint saved" : "No checkpoint",
+        });
+        return instance;
+    }
+    /** Resume a paused agent. */
+    async resume(instanceId) {
+        const instance = this.requireInstance(instanceId);
+        this.assertState(instance, "paused");
+        // Check resource availability
+        if (!this.resources.canAllocate(instance.resources)) {
+            throw new RuntimeError(ErrorCodes.RESOURCE_EXHAUSTED, "Cannot resume: resources unavailable", true);
+        }
+        const resumeEvent = transition(instance, "resuming", "Resuming from pause");
+        this.bus.emitLifecycle(resumeEvent);
+        // Re-allocate resources
+        this.resources.allocate(instanceId, instance.resources);
+        try {
+            const pid = await this.driver.resume(instance, instance.checkpointData);
+            instance.pid = pid;
+            instance.pausedAt = undefined;
+            const runEvent = transition(instance, "running", "Resumed successfully");
+            await this.registry.update(instanceId, {
+                state: "running",
+                pid,
+                pausedAt: undefined,
+                updatedAt: instance.updatedAt,
+            });
+            this.bus.emitLifecycle(runEvent);
+            this.bus.emitSpawned(instanceId, pid);
+            // Re-register exit handler
+            if ("onExit" in this.driver && typeof this.driver.onExit === "function") {
+                this.driver
+                    .onExit(instanceId, (code) => this.handleExit(instanceId, code));
+            }
+            logActivity({
+                source: "agent",
+                summary: `Agent resumed: ${instance.metadata.label}`,
+                detail: `PID ${pid}, instance ${instanceId}`,
+            });
+            return instance;
+        }
+        catch (err) {
+            const error = this.toAgentError(err, ErrorCodes.RESUME_FAILED, true);
+            const failEvent = transition(instance, "failed", "Resume failed", error);
+            await this.registry.update(instanceId, {
+                state: "failed",
+                error,
+                updatedAt: instance.updatedAt,
+            });
+            this.resources.release(instanceId);
+            this.bus.emitLifecycle(failEvent);
+            this.bus.emitFailed(instanceId, error);
+            logActivity({
+                source: "agent",
+                summary: `Agent resume failed: ${instance.metadata.label}`,
+                detail: err.message,
+            });
+            return instance;
+        }
+    }
+    /** Terminate an agent (from any active state). */
+    async terminate(instanceId, reason) {
+        const instance = this.requireInstance(instanceId);
+        if (isTerminal(instance.state)) {
+            return instance; // Already done
+        }
+        // Transition to terminating
+        try {
+            const termEvent = transition(instance, "terminating", reason ?? "Terminated by request");
+            this.bus.emitLifecycle(termEvent);
+        }
+        catch {
+            // If transition isn't valid from current state, force it
+            instance.state = "terminating";
+            instance.updatedAt = new Date().toISOString();
+        }
+        // Kill the process
+        try {
+            await this.driver.terminate(instance);
+        }
+        catch {
+            // Best effort — process may already be dead
+        }
+        // Transition to terminated
+        const finalEvent = transition(instance, "terminated", reason ?? "Terminated");
+        // Skip disk persist — GC will delete this file within gcTtlMs.
+        // On restart, driver.isAlive() returns false → recovery handles correctly.
+        await this.registry.update(instanceId, {
+            state: "terminated",
+            terminatedAt: instance.terminatedAt,
+            updatedAt: instance.updatedAt,
+        }, true);
+        this.resources.release(instanceId);
+        this.bus.unsubscribe(instanceId);
+        this.bus.emitLifecycle(finalEvent);
+        // Update linked AgentTask
+        await updateTask(instance.taskId, {
+            status: "cancelled",
+            finishedAt: new Date().toISOString(),
+        }).catch(() => { });
+        logActivity({
+            source: "agent",
+            summary: `Agent terminated: ${instance.metadata.label}`,
+            detail: `Reason: ${reason ?? "requested"}, instance ${instanceId}`,
+        });
+        return instance;
+    }
+    // -------------------------------------------------------------------------
+    // Queries
+    // -------------------------------------------------------------------------
+    /** Get an agent instance by ID. */
+    getInstance(id) {
+        return this.registry.get(id);
+    }
+    /** Get an instance by its linked task ID. */
+    getByTaskId(taskId) {
+        return this.registry.getByTaskId(taskId);
+    }
+    /** List all instances, optionally filtered. */
+    listInstances(filter) {
+        if (filter?.states) {
+            return this.registry.list({ states: filter.states });
+        }
+        return this.registry.list();
+    }
+    /** List all active (non-terminal) instances. */
+    listActive() {
+        return this.registry.listActive();
+    }
+    /** Get a snapshot of current resource usage. */
+    getResourceSnapshot() {
+        return this.resources.snapshot();
+    }
+    /** Get instance counts by state. */
+    getStateCounts() {
+        return this.registry.countByState();
+    }
+    /** Remove a terminated instance from the registry and disk. */
+    async removeInstance(id) {
+        return this.registry.remove(id);
+    }
+    /** Remove multiple terminated instances in parallel. */
+    async removeInstances(ids) {
+        return this.registry.removeMany(ids);
+    }
+    /**
+     * Remove multiple instances from in-memory registry only (no file I/O).
+     * Returns the count removed and their persist file paths for caller-managed
+     * batch deletion.
+     */
+    removeInstancesInMemory(ids) {
+        return this.registry.removeManyInMemory(ids);
+    }
+    // -------------------------------------------------------------------------
+    // Inter-agent messaging
+    // -------------------------------------------------------------------------
+    /** Send a message between agents. */
+    sendMessage(from, to, type, payload) {
+        return this.bus.send({ from, to, type, payload });
+    }
+    /** Send a request and await a correlated response. */
+    async requestResponse(from, to, type, payload, timeoutMs) {
+        return this.bus.request({ from, to, type, payload }, timeoutMs);
+    }
+    /** Subscribe an agent to receive messages. */
+    subscribeAgent(agentId, handler) {
+        this.bus.subscribe(agentId, handler);
+    }
+    // -------------------------------------------------------------------------
+    // Event subscriptions (for external consumers)
+    // -------------------------------------------------------------------------
+    /** Subscribe to lifecycle events. */
+    onLifecycle(handler) {
+        this.bus.on("agent:lifecycle", handler);
+    }
+    /** Subscribe to agent error events. */
+    onError(handler) {
+        this.bus.on("agent:error", handler);
+    }
+    /** Subscribe to resource warning events. */
+    onResourceWarning(handler) {
+        this.bus.on("runtime:resource-warning", handler);
+    }
+    // -------------------------------------------------------------------------
+    // Internal: process exit handling
+    // -------------------------------------------------------------------------
+    async handleExit(instanceId, code) {
+        const instance = this.registry.get(instanceId);
+        if (!instance || isTerminal(instance.state))
+            return;
+        // Guard: prevent concurrent handleExit for the same instance.
+        // Monitor poll + driver exit handler can race, causing duplicate retry scheduling.
+        if (this.handleExitGuard.has(instanceId)) {
+            logActivity({
+                source: "agent",
+                summary: `handleExit skipped (concurrent call): ${instance.metadata.label}`,
+                detail: `Instance ${instanceId}, exit code ${code} — another handleExit is already processing`,
+            });
+            return;
+        }
+        this.handleExitGuard.add(instanceId);
+        try {
+            await this.handleExitInner(instanceId, instance, code);
+        }
+        finally {
+            this.handleExitGuard.delete(instanceId);
+        }
+    }
+    async handleExitInner(instanceId, instance, code) {
+        const output = await readTaskOutput(instance.taskId).catch(() => "");
+        const resultSummary = output.trim().slice(0, 1000) || undefined;
+        // Null exit code (signal/restart) with substantial output is treated as success,
+        // matching the same logic in spawn.ts. Without this, RuntimeManager retries
+        // agents that spawn.ts already marked as completed.
+        const hasSubstantialOutput = output.trim().length > 100;
+        const success = code === 0 || (code == null && hasSubstantialOutput);
+        if (success) {
+            const event = transition(instance, "completed", `Exited with code ${code}`);
+            // Skip disk persist — GC will delete this file within gcTtlMs.
+            // On restart, driver.isAlive() returns false → recovery handles correctly.
+            await this.registry.update(instanceId, {
+                state: "completed",
+                updatedAt: instance.updatedAt,
+                terminatedAt: instance.terminatedAt,
+            }, true);
+            this.resources.release(instanceId);
+            this.bus.emitLifecycle(event);
+            this.bus.emitCompleted(instanceId, code ?? undefined);
+        }
+        else {
+            // Classify recoverability based on exit code and context:
+            // - null + near timeout: timeout kill — not recoverable (retrying will timeout again)
+            // - null (signal/OOM/killed): potentially recoverable (worth retrying)
+            // - non-zero: check output for deterministic failure patterns
+            const elapsed = Date.now() - new Date(instance.createdAt).getTime();
+            const isTimeout = code === null && elapsed >= instance.config.timeoutMs * 0.9;
+            const isDeterministicFailure = isTimeout || this.isDeterministicFailure(output, code);
+            const recoverable = !isDeterministicFailure;
+            if (isTimeout) {
+                logActivity({
+                    source: "agent",
+                    summary: `Agent timed out: ${instance.metadata.label}`,
+                    detail: `Elapsed ${Math.round(elapsed / 1000)}s of ${Math.round(instance.config.timeoutMs / 1000)}s limit — will not retry`,
+                });
+            }
+            const error = this.toAgentError(new Error(`Process exited with code ${code}`), ErrorCodes.DRIVER_ERROR, recoverable);
+            const event = transition(instance, "failed", `Exited with code ${code}`, error);
+            await this.registry.update(instanceId, {
+                state: "failed",
+                error,
+                updatedAt: instance.updatedAt,
+            });
+            // Release resources before retry check (maybeRetry re-allocates if needed)
+            this.resources.release(instanceId);
+            this.bus.emitLifecycle(event);
+            // Check if retry is possible BEFORE emitting terminal failure.
+            // This prevents external listeners (spawn.ts batch tracking) from
+            // prematurely processing the failure while retries are still pending,
+            // which was causing retry loop fan-out: continuation spawned new agents
+            // while the RuntimeManager was still retrying the original.
+            const retryDelay = shouldRetry(instance);
+            if (retryDelay !== null) {
+                logActivity({
+                    source: "agent",
+                    summary: `Agent failed, will retry: ${instance.metadata.label}`,
+                    detail: `Attempt ${instance.retryCount + 1}/${instance.config.maxRetries}, backoff ${retryDelay}ms, exit code ${code}, recoverable=${recoverable}, instance ${instanceId}`,
+                });
+                await this.maybeRetry(instance);
+                return; // Don't emit agent:failed or update task store — retry pending
+            }
+            // Terminal failure — no more retries, notify external listeners
+            logActivity({
+                source: "agent",
+                summary: `Agent failed (terminal, no retries left): ${instance.metadata.label}`,
+                detail: `Exit code ${code}, retryCount=${instance.retryCount}/${instance.config.maxRetries}, recoverable=${recoverable}, instance ${instanceId}`,
+            });
+            this.bus.emitFailed(instanceId, error);
+        }
+        // ── Finalize (reached only for terminal success or terminal failure) ──
+        this.bus.unsubscribe(instanceId);
+        const newState = success ? "completed" : "failed";
+        await updateTask(instance.taskId, {
+            status: newState,
+            exitCode: code ?? undefined,
+            finishedAt: new Date().toISOString(),
+            resultSummary,
+        }).catch(() => { });
+        // Persist to episodic memory
+        const task = await readTask(instance.taskId).catch(() => null);
+        if (task) {
+            rememberTaskOutcome({ ...task, status: newState, exitCode: code ?? undefined }, output).catch(() => { });
+        }
+        // Sync board task state (survives server restarts — inline spawn listeners don't)
+        if (task?.boardTaskId) {
+            updateBoardTaskState(task.boardTaskId, success
+                ? { state: "done" }
+                : { state: "todo", assignee: null }).catch(() => { });
+        }
+        logActivity({
+            source: "agent",
+            summary: `Agent ${newState}: ${instance.metadata.label}`,
+            detail: `Exit code ${code}, instance ${instanceId}`,
+        });
+    }
+    // -------------------------------------------------------------------------
+    // Internal: retry logic
+    // -------------------------------------------------------------------------
+    /**
+     * Emit terminal failure event and finalize task store + memory.
+     * Called when all retries are exhausted or a non-recoverable error occurs
+     * within the retry path. This ensures external listeners (spawn.ts batch
+     * tracking) are notified and the task store reflects the final state.
+     *
+     * Without this, tasks that fail during retries would leave the task store
+     * in a stale "running" state and batch continuation would never trigger.
+     */
+    async finalizeTerminalFailure(instance, error) {
+        const finalError = error ?? instance.error ?? {
+            code: ErrorCodes.MAX_RETRIES_EXCEEDED,
+            message: "Agent failed after all retry attempts",
+            timestamp: new Date().toISOString(),
+            recoverable: false,
+        };
+        this.bus.emitFailed(instance.id, finalError);
+        this.bus.unsubscribe(instance.id);
+        // Update linked AgentTask
+        const output = await readTaskOutput(instance.taskId).catch(() => "");
+        const resultSummary = output.trim().slice(0, 1000) || undefined;
+        await updateTask(instance.taskId, {
+            status: "failed",
+            finishedAt: new Date().toISOString(),
+            resultSummary,
+        }).catch(() => { });
+        // Persist to episodic memory
+        const task = await readTask(instance.taskId).catch(() => null);
+        if (task) {
+            rememberTaskOutcome({ ...task, status: "failed" }, output).catch(() => { });
+        }
+        logActivity({
+            source: "agent",
+            summary: `Agent failed (terminal): ${instance.metadata.label}`,
+            detail: `${instance.retryCount} retries exhausted, instance ${instance.id}`,
+        });
+    }
+    async maybeRetry(instance) {
+        // Guard: prevent duplicate retry scheduling for the same instance.
+        // This can happen when monitor poll and driver exit handler both trigger handleExit.
+        if (this.retryPending.has(instance.id)) {
+            logActivity({
+                source: "agent",
+                summary: `Retry already pending, skipping duplicate: ${instance.metadata.label}`,
+                detail: `Instance ${instance.id}, retryCount=${instance.retryCount}`,
+            });
+            return;
+        }
+        const delay = shouldRetry(instance);
+        if (delay === null) {
+            // Retries exhausted — emit terminal failure for external listeners.
+            // handleExit and spawn() skip emitting agent:failed when retry was
+            // possible, so this is the terminal notification path.
+            const terminalError = instance.error ?? {
+                code: ErrorCodes.MAX_RETRIES_EXCEEDED,
+                message: `Max retries (${instance.config.maxRetries}) exceeded after ${instance.retryCount} attempts`,
+                timestamp: new Date().toISOString(),
+                recoverable: false,
+            };
+            if (instance.retryCount > 0) {
+                logActivity({
+                    source: "agent",
+                    summary: `Agent max retries exceeded: ${instance.metadata.label}`,
+                    detail: `${instance.retryCount}/${instance.config.maxRetries} attempts, instance ${instance.id}`,
+                });
+            }
+            await this.finalizeTerminalFailure(instance, terminalError);
+            return;
+        }
+        // Check global retry budget to prevent retry storms when many agents fail at once
+        const now = Date.now();
+        this.globalRetryTimestamps = this.globalRetryTimestamps.filter((t) => now - t < RuntimeManager.GLOBAL_RETRY_WINDOW_MS);
+        if (this.globalRetryTimestamps.length >= this.globalRetryBudget) {
+            const budgetError = {
+                code: ErrorCodes.MAX_RETRIES_EXCEEDED,
+                message: `Global retry budget exhausted: ${this.globalRetryTimestamps.length} retries in ${RuntimeManager.GLOBAL_RETRY_WINDOW_MS / 60_000}min window`,
+                timestamp: new Date().toISOString(),
+                recoverable: false,
+            };
+            logActivity({
+                source: "agent",
+                summary: `Global retry budget exhausted, failing: ${instance.metadata.label}`,
+                detail: `${this.globalRetryTimestamps.length}/${this.globalRetryBudget} retries in ${RuntimeManager.GLOBAL_RETRY_WINDOW_MS / 60_000}min, instance ${instance.id}`,
+            });
+            await this.finalizeTerminalFailure(instance, budgetError);
+            return;
+        }
+        // Mark this instance as having a pending retry
+        this.retryPending.add(instance.id);
+        this.globalRetryTimestamps.push(now);
+        logActivity({
+            source: "agent",
+            summary: `Agent retrying in ${delay}ms: ${instance.metadata.label}`,
+            detail: `Attempt ${instance.retryCount + 1}/${instance.config.maxRetries}, instance ${instance.id}, global retries: ${this.globalRetryTimestamps.length}/${this.globalRetryBudget}`,
+        });
+        setTimeout(async () => {
+            this.retryPending.delete(instance.id);
+            let resourcesAllocated = false;
+            try {
+                const retryEvent = prepareRetry(instance);
+                this.bus.emitLifecycle(retryEvent);
+                await this.registry.update(instance.id, {
+                    state: "initializing",
+                    retryCount: instance.retryCount,
+                    error: undefined,
+                    pid: undefined,
+                    terminatedAt: undefined,
+                    updatedAt: instance.updatedAt,
+                });
+                // Re-read the original task for the prompt
+                const task = await readTask(instance.taskId).catch(() => null);
+                if (task) {
+                    instance._prompt = task.prompt;
+                }
+                // Re-allocate and spawn
+                if (!this.resources.canAllocate(instance.resources)) {
+                    // Resources unavailable — fail the retry instead of leaving a zombie
+                    const error = {
+                        code: ErrorCodes.RESOURCE_EXHAUSTED,
+                        message: `Resources unavailable for retry attempt ${instance.retryCount}`,
+                        timestamp: new Date().toISOString(),
+                        recoverable: false,
+                    };
+                    instance.state = "failed";
+                    instance.error = error;
+                    instance.updatedAt = new Date().toISOString();
+                    await this.registry.update(instance.id, {
+                        state: "failed",
+                        error,
+                        updatedAt: instance.updatedAt,
+                    });
+                    logActivity({
+                        source: "agent",
+                        summary: `Agent retry aborted (resources exhausted): ${instance.metadata.label}`,
+                        detail: `Attempt ${instance.retryCount}/${instance.config.maxRetries}, instance ${instance.id}`,
+                    });
+                    await this.finalizeTerminalFailure(instance, error);
+                    return;
+                }
+                this.resources.allocate(instance.id, instance.resources);
+                resourcesAllocated = true;
+                const pid = await this.driver.spawn(instance);
+                instance.pid = pid;
+                const runEvent = transition(instance, "running", "Retry spawned");
+                await this.registry.update(instance.id, {
+                    state: "running",
+                    pid,
+                    updatedAt: instance.updatedAt,
+                });
+                this.bus.emitLifecycle(runEvent);
+                this.bus.emitSpawned(instance.id, pid);
+                logActivity({
+                    source: "agent",
+                    summary: `Agent retry spawned: ${instance.metadata.label}`,
+                    detail: `Attempt ${instance.retryCount}/${instance.config.maxRetries}, PID ${pid}, instance ${instance.id}`,
+                });
+                if ("onExit" in this.driver && typeof this.driver.onExit === "function") {
+                    this.driver
+                        .onExit(instance.id, (code) => this.handleExit(instance.id, code));
+                }
+            }
+            catch (err) {
+                // Clean up on retry failure: release resources and fail the instance
+                // to prevent zombie agents stuck in "initializing" forever
+                if (resourcesAllocated) {
+                    this.resources.release(instance.id);
+                }
+                const error = {
+                    code: ErrorCodes.SPAWN_FAILED,
+                    message: `Retry spawn failed: ${err.message}`,
+                    timestamp: new Date().toISOString(),
+                    recoverable: false,
+                };
+                instance.state = "failed";
+                instance.error = error;
+                instance.updatedAt = new Date().toISOString();
+                await this.registry.update(instance.id, {
+                    state: "failed",
+                    error,
+                    updatedAt: instance.updatedAt,
+                }).catch(() => { });
+                logActivity({
+                    source: "agent",
+                    summary: `Agent retry spawn failed: ${instance.metadata.label}`,
+                    detail: `Attempt ${instance.retryCount}/${instance.config.maxRetries}: ${err.message}, instance ${instance.id}`,
+                });
+                await this.finalizeTerminalFailure(instance, error);
+            }
+        }, delay);
+    }
+    // -------------------------------------------------------------------------
+    // Internal: recovery on startup
+    // -------------------------------------------------------------------------
+    async recoverAgents() {
+        const instances = this.registry.listActive();
+        // Skip recovery entirely when there's nothing to recover (cold start)
+        if (instances.length === 0) {
+            log.info("No active instances to recover");
+            return;
+        }
+        let recovered = 0;
+        let marked = 0;
+        // Separate alive, dead, and initializing instances for parallel recovery
+        const alive = [];
+        const dead = [];
+        const initializing = [];
+        for (const instance of instances) {
+            if (instance.state === "running" || instance.state === "resuming") {
+                if (this.driver.isAlive(instance)) {
+                    alive.push(instance);
+                }
+                else {
+                    dead.push(instance);
+                }
+            }
+            else if (instance.state === "initializing") {
+                initializing.push(instance);
+            }
+        }
+        // Count alive agents (no I/O needed)
+        recovered = alive.length;
+        for (const instance of alive) {
+            logActivity({
+                source: "agent",
+                summary: `Recovered running agent: ${instance.metadata.label}`,
+                detail: `PID ${instance.pid} still alive`,
+            });
+        }
+        // Recover dead agents in parallel (each reads task output)
+        if (dead.length > 0) {
+            marked += dead.length;
+            await Promise.all(dead.map(async (instance) => {
+                const output = await readTaskOutput(instance.taskId).catch(() => "");
+                const hasOutput = output.trim().length > 0;
+                const finalState = hasOutput ? "completed" : "failed";
+                if (hasOutput) {
+                    transition(instance, "completed", "Recovered after restart");
+                }
+                else {
+                    const error = {
+                        code: ErrorCodes.DRIVER_ERROR,
+                        message: "Process died while runtime was down",
+                        timestamp: new Date().toISOString(),
+                        recoverable: false,
+                    };
+                    transition(instance, "failed", "Died during downtime", error);
+                }
+                await this.registry.update(instance.id, {
+                    state: instance.state,
+                    error: instance.error,
+                    terminatedAt: instance.terminatedAt,
+                    updatedAt: instance.updatedAt,
+                });
+                logActivity({
+                    source: "agent",
+                    summary: `Recovered ${finalState} agent: ${instance.metadata.label}`,
+                    detail: `PID ${instance.pid} was dead`,
+                });
+            }));
+        }
+        // Recover initializing agents in parallel
+        if (initializing.length > 0) {
+            marked += initializing.length;
+            await Promise.all(initializing.map(async (instance) => {
+                const error = {
+                    code: ErrorCodes.DRIVER_ERROR,
+                    message: "Runtime restarted during initialization",
+                    timestamp: new Date().toISOString(),
+                    recoverable: true,
+                };
+                transition(instance, "failed", "Runtime restarted", error);
+                await this.registry.update(instance.id, {
+                    state: "failed",
+                    error,
+                    updatedAt: instance.updatedAt,
+                });
+                await this.maybeRetry(instance);
+            }));
+        }
+        if (recovered > 0 || marked > 0) {
+            logActivity({
+                source: "agent",
+                summary: `Runtime recovery: ${recovered} alive, ${marked} resolved`,
+            });
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Internal: monitoring
+    // -------------------------------------------------------------------------
+    startMonitor() {
+        if (this.monitorTimer)
+            return;
+        this.monitorTimer = setInterval(() => {
+            this.monitorCycle().catch((err) => {
+                logActivity({
+                    source: "agent",
+                    summary: "Runtime monitor error",
+                    detail: err.message,
+                });
+            });
+        }, this.config.monitorIntervalMs);
+    }
+    stopMonitor() {
+        if (this.monitorTimer) {
+            clearInterval(this.monitorTimer);
+            this.monitorTimer = null;
+        }
+    }
+    async monitorCycle() {
+        const running = this.registry.list({ state: "running" });
+        for (const instance of running) {
+            // Skip instances with pending retries or active handleExit processing
+            // to prevent concurrent exit handling that causes duplicate retry scheduling
+            if (this.retryPending.has(instance.id) || this.handleExitGuard.has(instance.id)) {
+                continue;
+            }
+            if (!this.driver.isAlive(instance)) {
+                // Process died without triggering exit handler (recovered PID)
+                await this.handleExit(instance.id, null);
+            }
+        }
+        // Emit resource warning if nearing capacity
+        const snapshot = this.resources.snapshot();
+        const agentRatio = snapshot.activeAgents / snapshot.maxAgents;
+        const memoryRatio = snapshot.totalMemoryMB / snapshot.maxMemoryMB;
+        if (agentRatio > 0.8 || memoryRatio > 0.8) {
+            this.bus.emitResourceWarning(snapshot);
+        }
+    }
+    // -------------------------------------------------------------------------
+    // Helpers
+    // -------------------------------------------------------------------------
+    requireInstance(id) {
+        const instance = this.registry.get(id);
+        if (!instance) {
+            throw new RuntimeError(ErrorCodes.AGENT_NOT_FOUND, `Agent instance not found: ${id}`);
+        }
+        return instance;
+    }
+    assertState(instance, expected) {
+        if (instance.state !== expected) {
+            throw new RuntimeError(ErrorCodes.INVALID_TRANSITION, `Expected agent state '${expected}', got '${instance.state}'`, false, { agentId: instance.id, actual: instance.state, expected });
+        }
+    }
+    /**
+     * Detect deterministic failures that won't succeed on retry.
+     * Checks output for patterns like TypeScript errors, missing modules, etc.
+     * A broader pattern set prevents wasting retry budget on non-recoverable errors.
+     */
+    isDeterministicFailure(output, code) {
+        // null exit code = signal/OOM — worth retrying
+        if (code === null)
+            return false;
+        // Check output for patterns that indicate a deterministic failure
+        const tail = output.slice(-3000);
+        const deterministicPatterns = [
+            /error TS\d+:/i, // TypeScript compilation errors
+            /SyntaxError:/, // JS/TS syntax errors
+            /Cannot find module/, // Missing module (won't appear on retry)
+            /Module not found/, // Same
+            /ENOENT.*no such file/i, // Missing file
+            /Permission denied/i, // Permission issues (won't self-resolve)
+            /EACCES/, // Access denied (filesystem)
+            /ENOSPC/i, // Disk full
+            /Invalid API key/i, // Auth errors (won't self-resolve)
+            /authentication failed/i, // Auth errors
+            /unauthorized/i, // 401 errors
+            /forbidden/i, // 403 errors
+            / 402[:\s]/, // Payment required (insufficient credits)
+            /insufficient.credits/i, // OpenRouter credit exhaustion
+            /payment.required/i, // Billing errors
+            /out of.*credits/i, // Credit exhaustion variant
+            /can'?t afford/i, // OpenRouter "can't afford" message
+            /ERR_INVALID_ARG_TYPE/, // Programming error
+            /TypeError:.*is not a function/, // Programming error
+            /ReferenceError:/, // Undefined variable
+            /ENOMEM/i, // Out of memory (system-level)
+        ];
+        return deterministicPatterns.some((pattern) => pattern.test(tail));
+    }
+    toAgentError(err, code, recoverable) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            code,
+            message,
+            timestamp: new Date().toISOString(),
+            recoverable,
+        };
+    }
+}
+//# sourceMappingURL=manager.js.map