@rynx-ai/runtime 0.1.0 → 0.1.9

package/dist/runner/manager.js

@@ -28,17 +28,40 @@ import { StdioRunnerTransport } from "./transport.js";
 const CAP_KEY = "__cap__";
 /** Max stderr lines retained per handle for the crash exit-report tail. */
 const STDERR_TAIL_LINES = 40;
+/** Time allowed for a runner to exit after SIGTERM before SIGKILL escalation. */
+const DEFAULT_SHUTDOWN_GRACE_MS = 5_000;
+/** Time allowed for exit after SIGKILL before shutdown reports failure. */
+const DEFAULT_SHUTDOWN_KILL_GRACE_MS = 5_000;
+/** Keep per-Session spawn context small enough to remain an environment handoff,
+ * not an unbounded transport. */
+const MAX_SESSION_CONTEXT_ENV_ENTRIES = 32;
+const MAX_SESSION_CONTEXT_ENV_KEY_BYTES = 128;
+const MAX_SESSION_CONTEXT_ENV_VALUE_BYTES = 8_192;
+const MAX_SESSION_CONTEXT_ENV_TOTAL_BYTES = 32_768;
+/** A typed attach failure so transport adapters can distinguish an expected
+ * absent pane from an infrastructure failure without matching error strings. */
+export class TerminalOpenError extends Error {
+    code;
+    name = "TerminalOpenError";
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+    }
+}
 class ManagedTerminal {
     attachId;
     sendMsg;
+    onClose;
     ready;
     dataListener;
     exitListener;
     resolveReady;
     rejectReady;
-    constructor(attachId, sendMsg) {
+    state = "opening";
+    constructor(attachId, sendMsg, onClose) {
         this.attachId = attachId;
         this.sendMsg = sendMsg;
+        this.onClose = onClose;
         this.ready = new Promise((resolve, reject) => {
             this.resolveReady = resolve;
             this.rejectReady = reject;
@@ -59,21 +82,49 @@ class ManagedTerminal {
         this.sendMsg({ t: "term.resize", attachId: this.attachId, cols, rows });
     }
     close() {
+        if (this.state === "closed")
+            return;
+        const wasOpening = this.state === "opening";
+        this.state = "closed";
         this.sendMsg({ t: "term.close", attachId: this.attachId });
+        this.onClose();
+        if (wasOpening) {
+            this.rejectReady(new TerminalOpenError("terminal attachment was closed while opening", "terminal_open_failed"));
+        }
     }
     // ── internal (driven by onChildMessage) ──
     _opened(role) {
+        if (this.state !== "opening")
+            return;
+        this.state = "opened";
         this.resolveReady({ role });
     }
     _data(dataB64) {
         this.dataListener?.(Buffer.from(dataB64, "base64").toString("utf8"));
     }
     _exit(exitCode) {
-        this.exitListener?.({ exitCode });
+        if (this.state === "closed")
+            return;
+        const wasOpening = this.state === "opening";
+        this.state = "closed";
+        if (wasOpening) {
+            this.rejectReady(new TerminalOpenError(`terminal exited before opening (code=${exitCode})`, "terminal_open_failed"));
+        }
+        else {
+            this.exitListener?.({ exitCode });
+        }
     }
-    _fail(message) {
-        this.rejectReady(new Error(message));
-        this.exitListener?.({ exitCode: 1 });
+    _fail(message, code = "terminal_open_failed") {
+        if (this.state === "closed")
+            return;
+        const wasOpening = this.state === "opening";
+        this.state = "closed";
+        if (wasOpening) {
+            this.rejectReady(new TerminalOpenError(message, code));
+        }
+        else {
+            this.exitListener?.({ exitCode: 1 });
+        }
     }
 }
 export class RunnerManager {
@@ -83,16 +134,26 @@ export class RunnerManager {
     idleTtlMs;
     spawn;
     childEnv;
+    sessionContextProvider;
     now;
     defaultRuntime;
     handles = new Map();
+    /** Every spawned child that has not exited (or failed to spawn), including
+     *  handles already removed from routing by stopRunner/idle reap. */
+    childHandles = new Set();
     reapTimer;
+    shutdownGraceMs;
+    shutdownKillGraceMs;
+    stopping = false;
+    stopPromise;
     /** Sink for mirrored {@link SessionEvent}s from every session's forwarder. */
     mirrorListener = null;
     /** Sink for session rotations (claude `/clear`·`/fork`) — server records meta. */
     rotateListener = null;
     /** Session keys with a live codex forwarder — never reaped while present. */
     liveSessionKeys = new Set();
+    /** Last live-start error per local session, surfaced by the control API. */
+    liveErrors = new Map();
     constructor(opts) {
         this.config = opts.config;
         this.sessionStore = opts.sessionStore;
@@ -100,8 +161,11 @@ export class RunnerManager {
         this.idleTtlMs = opts.idleTtlMs ?? 300_000;
         this.spawn = opts.spawn ?? nodeSpawn;
         this.childEnv = opts.childEnv ?? {};
+        this.sessionContextProvider = opts.sessionContextProvider;
         this.now = opts.now ?? (() => Date.now());
         this.defaultRuntime = opts.config.AGENT_RUNTIME ?? "codex";
+        this.shutdownGraceMs = Math.max(0, opts.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS);
+        this.shutdownKillGraceMs = Math.max(0, opts.shutdownKillGraceMs ?? DEFAULT_SHUTDOWN_KILL_GRACE_MS);
         const reapIntervalMs = opts.reapIntervalMs ?? 60_000;
         if (reapIntervalMs > 0) {
             this.reapTimer = setInterval(() => this.reapIdle(), reapIntervalMs);
@@ -121,9 +185,24 @@ export class RunnerManager {
      */
     openTerminal(localThreadId, opts) {
         const handle = this.getOrSpawn(localThreadId);
+        return this.openTerminalOnHandle(handle, localThreadId, opts);
+    }
+    /**
+     * Attach to a Session runner that is already live. Unlike {@link openTerminal},
+     * this never creates a child, including when liveness changes between lookup
+     * and attach.
+     */
+    openLiveTerminal(localThreadId, opts) {
+        const handle = this.handles.get(localThreadId);
+        if (!handle || handle.dead || !this.liveSessionKeys.has(localThreadId)) {
+            throw new TerminalOpenError("terminal not live", "terminal_not_live");
+        }
+        return this.openTerminalOnHandle(handle, localThreadId, opts);
+    }
+    openTerminalOnHandle(handle, localThreadId, opts) {
         handle.lastUsedAt = this.now();
         const attachId = randomUUID();
-        const terminal = new ManagedTerminal(attachId, (msg) => handle.transport.send(msg));
+        const terminal = new ManagedTerminal(attachId, (msg) => handle.transport.send(msg), () => handle.terminals.delete(attachId));
         handle.terminals.set(attachId, terminal);
         handle.transport.send({
             t: "term.open",
@@ -140,6 +219,11 @@ export class RunnerManager {
         });
         return terminal;
     }
+    /** Whether this process already owns a live runner for the Session. Read-only; never spawns. */
+    hasLiveSession(localThreadId) {
+        const handle = this.handles.get(localThreadId);
+        return Boolean(handle && !handle.dead && this.liveSessionKeys.has(localThreadId));
+    }
     /**
      * Register the sink for mirrored {@link SessionEvent}s produced by every
      * session's persistent codex forwarder (web- AND TUI-initiated turns). The
@@ -165,7 +249,16 @@ export class RunnerManager {
         this.liveSessionKeys.add(localThreadId);
         const reqId = randomUUID();
         return new Promise((resolve) => {
-            handle.live.set(reqId, (res) => resolve(res.ok ?? false));
+            handle.live.set(reqId, (res) => {
+                const ok = res.ok ?? false;
+                if (ok) {
+                    this.liveErrors.delete(localThreadId);
+                }
+                else {
+                    this.liveErrors.set(localThreadId, res.error ?? "live session did not become ready");
+                }
+                resolve(ok);
+            });
             handle.transport.send({
                 t: "live.ensure",
                 reqId,
@@ -174,13 +267,17 @@ export class RunnerManager {
                 ...(opts?.cols ? { cols: opts.cols } : {}),
                 ...(opts?.rows ? { rows: opts.rows } : {}),
                 ...(opts?.runtime ? { runtime: opts.runtime } : {}),
+                ...(opts?.reasoningEffort ? { reasoningEffort: opts.reasoningEffort } : {}),
                 ...(opts?.agentName ? { agentName: opts.agentName } : {}),
                 ...(opts?.agentSpec ? { agentSpec: opts.agentSpec } : {}),
             });
         });
     }
+    lastLiveSessionError(localThreadId) {
+        return this.liveErrors.get(localThreadId);
+    }
     /**
-     * Inject a user turn into a session's live codex thread — omnigent's
+     * Inject a user turn into a session's live codex thread — reference implementation's
      * single-writer web send (`turn/start` / `turn/steer`); the forwarder mirrors
      * all output. Resolves true when the app-server accepted the turn, false when
      * the session has no live forwarder (caller falls back to the run path).
@@ -210,17 +307,29 @@ export class RunnerManager {
             handle.transport.send({ t: "live.interrupt", reqId, localThreadId });
         });
     }
-    /**
-     * Deliver a user's interactive-approval decision to the session's runner
-     * child (Phase D). Best-effort: no-op if the runner isn't live (the pending
-     * approval would have died with it). The child routes it to its codex client.
-     */
-    resolveApproval(localThreadId, approvalId, decision) {
+    /** Resolve a native question/approval without ever spawning a new runner. */
+    resolveInteraction(localThreadId, interactionId, resolution) {
         const handle = this.handles.get(localThreadId);
         if (!handle || handle.dead) {
-            return;
+            return Promise.resolve({ disposition: "not_found" });
         }
-        handle.transport.send({ t: "approval.resolve", localThreadId, approvalId, decision });
+        handle.lastUsedAt = this.now();
+        const reqId = randomUUID();
+        return new Promise((resolve) => {
+            handle.live.set(reqId, (reply) => {
+                resolve(reply.interactionResult ?? {
+                    disposition: "invalid",
+                    message: reply.error ?? "runner did not return an interaction result",
+                });
+            });
+            handle.transport.send({
+                t: "interaction.resolve",
+                reqId,
+                localThreadId,
+                interactionId,
+                resolution,
+            });
+        });
     }
     // ── AgentCapabilities ──────────────────────────────────────────────────────
     async listModels(runtime) {
@@ -254,20 +363,35 @@ export class RunnerManager {
         if (!handle) {
             return;
         }
-        handle.transport.send({ t: "shutdown" });
-        handle.child.kill("SIGTERM");
-        this.failHandle(handle, "runner stopped");
+        void this.terminateHandle(handle, "runner stopped").catch((error) => {
+            logTerminationFailure(handle, error);
+        });
     }
-    /** Kill every runner. Call on server shutdown. */
-    async stop() {
+    /** Stop every runner and join all child exits. Idempotent across concurrent calls. */
+    stop() {
+        if (this.stopPromise)
+            return this.stopPromise;
+        this.stopping = true;
         if (this.reapTimer) {
             clearInterval(this.reapTimer);
         }
-        for (const handle of this.handles.values()) {
-            handle.transport.send({ t: "shutdown" });
-            handle.child.kill("SIGTERM");
-        }
-        this.handles.clear();
+        const children = [...this.childHandles];
+        this.stopPromise = (async () => {
+            const results = await Promise.allSettled(children.map((handle) => this.terminateHandle(handle, "runner manager stopped")));
+            this.handles.clear();
+            this.liveSessionKeys.clear();
+            this.liveErrors.clear();
+            this.mirrorListener = null;
+            this.rotateListener = null;
+            const errors = results
+                .filter((result) => result.status === "rejected")
+                .map((result) => result.reason);
+            if (errors.length > 0) {
+                throw new AggregateError(errors, "runner manager shutdown failed");
+            }
+            this.childHandles.clear();
+        })();
+        return this.stopPromise;
     }
     // ── internals ──────────────────────────────────────────────────────────────
     /** Forward a capability to a runner child. Session-less caps (listModels/status)
@@ -283,6 +407,9 @@ export class RunnerManager {
         });
     }
     getOrSpawn(key) {
+        if (this.stopping) {
+            throw new AgentRuntimeError("runner manager is stopped", 503, "runner_stopped");
+        }
         this.reapIdle();
         const existing = this.handles.get(key);
         if (existing && !existing.dead) {
@@ -297,13 +424,29 @@ export class RunnerManager {
         const args = this.runnerEntry.endsWith(".ts")
             ? ["--import", "tsx", this.runnerEntry]
             : [this.runnerEntry];
-        const child = this.spawn(process.execPath, args, {
-            stdio: ["pipe", "pipe", "pipe"],
-            // `RYNX_RUNNER_SESSION` = the child's session key (localThreadId, or `__cap__`
-            // for the shared capability child) so its LocalAgentHost scopes the private
-            // CODEX_HOME to this session.
-            env: { ...process.env, ...this.childEnv, RYNX_RUNNER_SESSION: key },
-        });
+        const sessionContext = key === CAP_KEY
+            ? undefined
+            : this.openSessionContext(key);
+        let child;
+        try {
+            child = this.spawn(process.execPath, args, {
+                stdio: ["pipe", "pipe", "pipe"],
+                // `RYNX_RUNNER_SESSION` = the child's session key (localThreadId, or `__cap__`
+                // for the shared capability child) so its LocalAgentHost scopes the private
+                // CODEX_HOME to this session. It is deliberately applied last.
+                env: {
+                    ...process.env,
+                    ...this.childEnv,
+                    ...(sessionContext?.childEnv ?? {}),
+                    RYNX_RUNNER_SESSION: key,
+                },
+            });
+        }
+        catch (error) {
+            closeSessionContext(sessionContext);
+            throw error;
+        }
+        const completion = observeChildCompletion(child);
         const transport = new StdioRunnerTransport(child.stdout, child.stdin);
         const handle = {
             key,
@@ -312,10 +455,14 @@ export class RunnerManager {
             stderr: [],
             lastUsedAt: this.now(),
             dead: false,
+            completion,
+            ...(sessionContext ? { sessionContext } : {}),
             caps: new Map(),
             terminals: new Map(),
             live: new Map(),
         };
+        this.childHandles.add(handle);
+        void completion.then(() => this.childHandles.delete(handle));
         transport.onMessage((msg) => this.onChildMessage(handle, msg));
         if (child.stderr) {
             const rl = readline.createInterface({ input: child.stderr, crlfDelay: Infinity });
@@ -363,13 +510,28 @@ export class RunnerManager {
             case "term.error": {
                 const terminal = handle.terminals.get(msg.attachId);
                 handle.terminals.delete(msg.attachId);
-                terminal?._fail(msg.message);
+                terminal?._fail(msg.message, msg.code);
                 return;
             }
             case "mirror":
                 this.mirrorListener?.(msg.sessionId, msg.event);
                 return;
             case "rotate": {
+                if (handle.dead)
+                    return;
+                try {
+                    // Rotate the runtime-local context before publishing the new Session
+                    // alias to routing or observers. This prevents the new identity from
+                    // briefly inheriting stale context after `/clear` or `/fork`.
+                    handle.sessionContext?.rotate(msg.to);
+                }
+                catch (error) {
+                    const reason = `runner Session context rotation failed: ${errorMessage(error)}`;
+                    void this.terminateHandle(handle, reason).catch((terminationError) => {
+                        logTerminationFailure(handle, terminationError);
+                    });
+                    return;
+                }
                 // Terminal transfer: alias the new session to THIS runner so its injection
                 // (and live/approval) route to the same child that still owns the pane.
                 this.handles.set(msg.to, handle);
@@ -395,6 +557,12 @@ export class RunnerManager {
                     : { ok: msg.ok, error: msg.error });
                 return;
             }
+            case "interaction.resolved": {
+                const resolve = handle.live.get(msg.reqId);
+                handle.live.delete(msg.reqId);
+                resolve?.({ interactionResult: msg.result });
+                return;
+            }
         }
     }
     /** Mark a handle dead and reject every pending run/cap with the exit reason. */
@@ -403,11 +571,14 @@ export class RunnerManager {
             return;
         }
         handle.dead = true;
+        closeOwnedSessionContext(handle);
         // Drop every key mapping to this handle — its launch key AND any rotation
         // aliases (claude `/clear`·`/fork` terminal transfer).
         for (const [key, h] of this.handles) {
-            if (h === handle)
+            if (h === handle) {
                 this.handles.delete(key);
+                this.liveSessionKeys.delete(key);
+            }
         }
         const tail = handle.stderr.join("\n");
         const message = tail ? `${reason}\n--- runner log tail ---\n${tail}` : reason;
@@ -424,9 +595,32 @@ export class RunnerManager {
         handle.caps.clear();
         handle.terminals.clear();
         handle.live.clear();
-        this.liveSessionKeys.delete(handle.key);
         handle.transport.close();
     }
+    terminateHandle(handle, reason) {
+        if (handle.termination)
+            return handle.termination;
+        handle.termination = (async () => {
+            if (!handle.dead) {
+                try {
+                    handle.transport.send({ t: "shutdown" });
+                }
+                catch {
+                    // The process signal below remains the authoritative shutdown path.
+                }
+                this.failHandle(handle, reason);
+            }
+            signalChild(handle.child, "SIGTERM");
+            if (!(await waitForChildExit(handle.completion, this.shutdownGraceMs))) {
+                signalChild(handle.child, "SIGKILL");
+                if (!(await waitForChildExit(handle.completion, this.shutdownKillGraceMs))) {
+                    throw new Error(`runner child ${handle.child.pid ?? handle.key} did not exit after SIGKILL`);
+                }
+            }
+            this.childHandles.delete(handle);
+        })();
+        return handle.termination;
+    }
     reapIdle() {
         const now = this.now();
         for (const [key, handle] of this.handles) {
@@ -441,14 +635,106 @@ export class RunnerManager {
             if (now - handle.lastUsedAt < this.idleTtlMs) {
                 continue;
             }
-            handle.dead = true;
-            this.handles.delete(key);
-            handle.transport.send({ t: "shutdown" });
-            handle.child.kill("SIGTERM");
-            handle.transport.close();
+            void this.terminateHandle(handle, "idle runner reaped").catch((error) => {
+                logTerminationFailure(handle, error);
+            });
+        }
+    }
+    openSessionContext(sessionId) {
+        if (!this.sessionContextProvider)
+            return undefined;
+        const context = this.sessionContextProvider.open(sessionId);
+        try {
+            validateSessionContext(context);
+            return {
+                childEnv: Object.freeze({ ...context.childEnv }),
+                rotate: (newSessionId) => context.rotate(newSessionId),
+                close: () => context.close(),
+            };
+        }
+        catch (error) {
+            closeSessionContext(context);
+            throw error;
+        }
+    }
+}
+function validateSessionContext(context) {
+    if (!context || typeof context !== "object") {
+        throw new Error("Session context environment provider must return an object");
+    }
+    if (typeof context.rotate !== "function" || typeof context.close !== "function") {
+        throw new Error("Session context environment must provide rotate() and close()");
+    }
+    if (!context.childEnv || typeof context.childEnv !== "object" || Array.isArray(context.childEnv)) {
+        throw new Error("Session context environment must be a record");
+    }
+    const entries = Object.entries(context.childEnv);
+    if (entries.length > MAX_SESSION_CONTEXT_ENV_ENTRIES) {
+        throw new Error(`Session context environment exceeds ${MAX_SESSION_CONTEXT_ENV_ENTRIES} entries`);
+    }
+    let totalBytes = 0;
+    for (const [key, value] of entries) {
+        const normalizedKey = key.toUpperCase();
+        if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) {
+            throw new Error(`Session context environment key is invalid: ${JSON.stringify(key)}`);
+        }
+        if (isReservedSessionContextEnvKey(normalizedKey)) {
+            throw new Error(`Session context environment key is reserved: ${key}`);
+        }
+        if (typeof value !== "string" || value.includes("\0")) {
+            throw new Error(`Session context environment value is invalid for ${key}`);
+        }
+        const keyBytes = Buffer.byteLength(key);
+        const valueBytes = Buffer.byteLength(value);
+        if (keyBytes > MAX_SESSION_CONTEXT_ENV_KEY_BYTES) {
+            throw new Error(`Session context environment key is too large: ${key}`);
+        }
+        if (valueBytes > MAX_SESSION_CONTEXT_ENV_VALUE_BYTES) {
+            throw new Error(`Session context environment value is too large for ${key}`);
         }
+        totalBytes += keyBytes + valueBytes;
+        if (totalBytes > MAX_SESSION_CONTEXT_ENV_TOTAL_BYTES) {
+            throw new Error("Session context environment is too large");
+        }
+    }
+}
+function isReservedSessionContextEnvKey(key) {
+    return (key.startsWith("RYNX_RUNNER_") ||
+        key === "NODE_OPTIONS" ||
+        key === "NODE_PATH" ||
+        key === "NODE_UNIQUE_ID" ||
+        key.startsWith("NODE_CHANNEL_") ||
+        key === "ELECTRON_RUN_AS_NODE" ||
+        key === "LD_PRELOAD" ||
+        key === "LD_LIBRARY_PATH" ||
+        key === "DYLD_INSERT_LIBRARIES" ||
+        key === "DYLD_LIBRARY_PATH");
+}
+function closeOwnedSessionContext(handle) {
+    const context = handle.sessionContext;
+    if (!context)
+        return;
+    handle.sessionContext = undefined;
+    closeSessionContext(context);
+}
+function closeSessionContext(context) {
+    if (!context || typeof context.close !== "function")
+        return;
+    try {
+        context.close();
+    }
+    catch (error) {
+        console.error(JSON.stringify({
+            level: "error",
+            type: "runner",
+            event: "session_context_close_failed",
+            error: errorMessage(error),
+        }));
     }
 }
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
 function defaultRunnerEntry() {
     if (process.env.RYNX_RUNNER_ENTRY?.trim()) {
         return process.env.RYNX_RUNNER_ENTRY.trim();
@@ -456,3 +742,61 @@ function defaultRunnerEntry() {
     // dist/runner/manager.js → dist/runner-main.js
     return fileURLToPath(new URL("../runner-main.js", import.meta.url));
 }
+function observeChildCompletion(child) {
+    if (childHasExited(child))
+        return Promise.resolve();
+    return new Promise((resolve) => {
+        const onExit = () => {
+            child.off("error", onSpawnError);
+            resolve();
+        };
+        const onSpawnError = () => {
+            // A successfully spawned process may emit `error` when a later kill fails;
+            // that is not completion. A spawn failure has no pid and no future exit.
+            if (child.pid !== undefined)
+                return;
+            child.off("exit", onExit);
+            resolve();
+        };
+        child.once("error", onSpawnError);
+        child.once("exit", onExit);
+    });
+}
+function signalChild(child, signal) {
+    if (childHasExited(child))
+        return;
+    try {
+        child.kill(signal);
+    }
+    catch {
+        // The bounded completion wait distinguishes an exit race from a stuck child.
+    }
+}
+function childHasExited(child) {
+    return ((child.exitCode !== null && child.exitCode !== undefined) ||
+        (child.signalCode !== null && child.signalCode !== undefined));
+}
+function waitForChildExit(completion, timeoutMs) {
+    return new Promise((resolve) => {
+        let settled = false;
+        const finish = (exited) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve(exited);
+        };
+        const timer = setTimeout(() => finish(false), timeoutMs);
+        void completion.then(() => finish(true));
+    });
+}
+function logTerminationFailure(handle, error) {
+    console.error(JSON.stringify({
+        level: "error",
+        type: "runner",
+        event: "termination_failed",
+        key: handle.key,
+        pid: handle.child.pid,
+        error: error instanceof Error ? error.message : String(error),
+    }));
+}