@0xmaxma/claude-gateway 1.3.25 → 1.3.32

package/dist/agent/runner.js

@@ -49,6 +49,11 @@ const line_reply_manager_1 = require("./line-reply-manager");
 const markdown_1 = require("../telegram/markdown");
 const skills_1 = require("../skills");
 const builtin_commands_1 = require("./builtin-commands");
+const safe_mode_1 = require("./safe-mode");
+const recovery_executor_1 = require("./recovery-executor");
+const recovery_policy_1 = require("./recovery-policy");
+const incident_1 = require("./incident");
+const pty_stream_registry_1 = require("../shell/pty-stream-registry");
 const screen_1 = require("../shell/screen");
 const db_1 = require("../history/db");
 const media_store_1 = require("../history/media-store");
@@ -56,14 +61,16 @@ const cleanup_1 = require("../history/cleanup");
 const DEFAULT_IDLE_TIMEOUT_MINUTES = 30;
 const DEFAULT_MAX_CONCURRENT = 20;
 const ANTHROPIC_SOCKET_ERROR = 'socket connection was closed unexpectedly';
-// History re-injection ladder for request_too_large (32MB) recovery. Index =
-// number of consecutive 32MB recoveries on a session; the value is how many
-// history messages the NEXT spawn re-injects. Index 0 is the healthy default
-// (= MAX_HISTORY_MESSAGES, sourced from it so the two never drift); each retry
-// steps down a rung, shrinking the re-loaded context until it drops under
-// Anthropic's 32MB request ceiling. Past the last rung (0 history) the context
-// can't shrink further, so the runner stops escalating and asks the user to
-// /clear instead of looping forever.
+// History re-injection ladder for request_too_large (32MB) recovery: the
+// candidate history sizes a recovering session can step down to. The HEALTHY
+// spawn uses the configured cap (resolveMaxHistoryMessages), not an index into
+// this array; on each consecutive 32MB recovery the session drops to the next
+// ladder rung STRICTLY BELOW that cap (see spawnHistoryLimit), so every retry
+// actually shrinks the re-loaded context instead of re-trying the same size.
+// Once even the 0-history rung still trips 32MB the runner stops escalating and
+// asks the user to /clear instead of looping forever. The leading
+// MAX_HISTORY_MESSAGES only acts as a recovery rung when an operator configures
+// a cap higher than it.
 const TOO_LARGE_HISTORY_LADDER = [process_1.MAX_HISTORY_MESSAGES, 40, 30, 20, 10, 0];
 exports.MAX_IMAGE_SIZE_BYTES = 10 * 1024 * 1024;
 exports.DEFAULT_MODELS = [
@@ -82,6 +89,14 @@ const PROTECTED_WORKSPACE_FILES = [
     'IDENTITY.md', 'USER.md', 'HEARTBEAT.md',
 ];
 const MAX_API_IMAGES = 5;
+// Hard timeout for the one-shot local `claude -p` triage during recovery
+// (Epic #195, Phase 3b). A slow/hung triage collapses to a safe notify-only.
+const RECOVERY_TRIAGE_TIMEOUT_MS = 15000;
+// TTL after which a per-turn recovery budget entry is evicted. A turn's budget
+// only matters during its active stall window (a few interventions spaced by a
+// 30s cooldown), so anything older is a finished turn — pruned to keep the
+// budget map from growing unbounded over a long-lived runner (Epic #195, 3b).
+const RECOVERY_BUDGET_TTL_MS = 10 * 60000;
 // Trailing-edge window for coalescing channel messages that carry an image (or that
 // arrive while an image is already buffered) into a single turn. Reset on every new
 // related update, so an album burst or a client-split photo+caption is gathered
@@ -157,6 +172,27 @@ class AgentRunner extends events_1.EventEmitter {
         // restarting would just churn a context that cannot shrink. Cleared together
         // with the counter on a successful result and on /clear.
         this.tooLargeExhausted = new Set();
+        // Safe-mode manager (Epic #195, Phase 3): tracks repeated PTY-backend failures
+        // and, past a threshold, forces this agent to the headless backend so it keeps
+        // serving turns without a gateway restart. In-memory only — a restart clears it
+        // and re-reads the user's real config. spawnSession reads isActive() to set
+        // SessionProcess.forceHeadless.
+        this.safeMode = new safe_mode_1.SafeModeManager({
+            audit: (e) => this.logger.info('Safe-mode transition', {
+                agentId: e.agentId,
+                action: e.action,
+                reason: e.reason,
+                failures: e.failures,
+            }),
+        });
+        // Recovery executor state (Epic #195, Phase 3b). Per-turn intervention budget,
+        // keyed by turnKey so it resets each turn. In-memory only.
+        this.recoveryBudgets = new Map();
+        // Last injected turn per chat, for the C1 guarded resend: `delivered` flips
+        // true once assistant output reaches the channel, so a resend after recovery
+        // only fires when the stalled turn produced nothing (no double-submit). `resent`
+        // guards against resending more than once per turn.
+        this.lastTurn = new Map();
         // Tracks pending Telegram image paths per chatId (queue) for size accumulation after each turn.
         this.pendingImagePaths = new Map();
         // Coalescing buffer: collects channel messages that arrive close together so a
@@ -177,6 +213,46 @@ class AgentRunner extends events_1.EventEmitter {
         this.skillRegistry = { skills: new Map() };
         // Cancel function for the daily history cleanup timer
         this.cancelCleanup = null;
+        /**
+         * One-shot local `claude -p` triage runner (Phase 3b). Feeds the closed
+         * classification prompt on stdin and returns raw stdout. Hard timeout →
+         * timedOut, which the executor collapses to a safe notify-only verdict. Only
+         * invoked when autoRecover is enabled.
+         */
+        this.recoveryTriageSpawn = async (prompt) => {
+            const { spawn } = await Promise.resolve().then(() => __importStar(require('child_process')));
+            return new Promise((resolve) => {
+                let done = false;
+                let out = '';
+                const finish = (r) => {
+                    if (done)
+                        return;
+                    done = true;
+                    clearTimeout(timer);
+                    resolve(r);
+                };
+                const timer = setTimeout(() => {
+                    try {
+                        child.kill('SIGKILL');
+                    }
+                    catch { /* already gone */ }
+                    finish({ stdout: '', timedOut: true });
+                }, RECOVERY_TRIAGE_TIMEOUT_MS);
+                const child = spawn('claude', ['-p', '--output-format', 'text'], {
+                    stdio: ['pipe', 'pipe', 'ignore'],
+                });
+                child.stdout?.on('data', (d) => { out += String(d); });
+                child.on('error', () => finish({ stdout: '', timedOut: true }));
+                child.on('close', () => finish({ stdout: out }));
+                try {
+                    child.stdin?.write(prompt);
+                    child.stdin?.end();
+                }
+                catch {
+                    finish({ stdout: '', timedOut: true });
+                }
+            });
+        };
         this.agentConfig = agentConfig;
         this.gatewayConfig = gatewayConfig;
         this.logger = logger ?? (0, logger_1.createLogger)(agentConfig.id, gatewayConfig.gateway.logDir);
@@ -229,6 +305,13 @@ class AgentRunner extends events_1.EventEmitter {
                     this.handleCommandRequest(raw, res);
                     return;
                 }
+                if (url.pathname === '/recover') {
+                    // Cross-process recovery bridge (Epic #195, Phase 3b): the receiver
+                    // process detects a stall but recovery must run here, where the live
+                    // control surfaces (session stdin, restart, safe-mode) live.
+                    this.handleRecoverRequest(raw, res);
+                    return;
+                }
                 // Default: /channel — existing channel message handler
                 // Connection: close prevents keep-alive pool reuse: after the OS's
                 // keepAliveTimeout expires the server closes the TCP connection, and the
@@ -315,6 +398,160 @@ class AgentRunner extends events_1.EventEmitter {
         });
         this.logger.info('Channel callback server listening', { port: this.callbackPort });
     }
+    /**
+     * Handle POST /recover from the receiver process (Epic #195, Phase 3b).
+     * The watchdog detects a stall in the receiver, but recovery must run here in
+     * the runner, where the live control surfaces live. Runs the pure executor
+     * with effects bound to the affected session and returns the outcome so the
+     * receiver can persist it to the incident bundle.
+     */
+    async handleRecoverRequest(raw, res) {
+        const respond = (data, status = 200) => {
+            res.writeHead(status, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify(data));
+        };
+        let body;
+        try {
+            body = JSON.parse(raw);
+        }
+        catch {
+            respond({ ok: false, error: 'Invalid JSON' }, 400);
+            return;
+        }
+        const chatId = body.chatId ?? '';
+        const stage = body.stage ?? '';
+        if (!chatId || !stage) {
+            respond({ ok: false, error: 'missing chatId/stage' }, 400);
+            return;
+        }
+        const session = this.sessions.get(chatId);
+        const autoRecover = this.gatewayConfig.gateway.selfHealing?.autoRecover === true;
+        const req = {
+            incidentId: body.incidentId ?? '',
+            agentId: this.agentConfig.id,
+            chatId,
+            // Prefer the live session's id (survives a request that names a stale one).
+            sessionId: session?.sessionId ?? body.sessionId ?? '',
+            stage,
+            failureClass: body.failureClass ?? null,
+            turnKey: body.turnKey ?? chatId,
+        };
+        try {
+            const result = await (0, recovery_executor_1.runRecovery)(req, {
+                autoRecover,
+                effects: this.buildRecoveryEffects(chatId),
+                now: () => Date.now(),
+                budget: {
+                    get: (turnKey) => this.recoveryBudgets.get(turnKey) ?? (0, recovery_policy_1.initialBudget)(turnKey),
+                    set: (s) => {
+                        this.recoveryBudgets.set(s.turnKey, s);
+                        this.pruneRecoveryBudgets(s.lastAt);
+                    },
+                },
+                // Live triage is the most sensitive surface — only ever run when the
+                // operator has opted in. When off, the executor uses the deterministic
+                // per-stage default action (still whitelist-clamped).
+                triageSpawn: autoRecover ? this.recoveryTriageSpawn : undefined,
+                gatherEvidence: async () => {
+                    const s = this.sessions.get(chatId);
+                    if (!s)
+                        return null;
+                    try {
+                        const snap = await pty_stream_registry_1.ptyStreamRegistry.screenText(s.sessionId);
+                        if (!snap)
+                            return null;
+                        // Scrub the untrusted screen before it leaves the process / reaches
+                        // the triage model. Redact the chat id explicitly on top of the
+                        // pattern-based secret scrub.
+                        return { screenText: (0, incident_1.scrubText)(snap.text, [chatId]) };
+                    }
+                    catch {
+                        return null;
+                    }
+                },
+                resendAfterRecover: autoRecover,
+                log: (msg, meta) => this.logger.info(msg, meta),
+            });
+            respond({ ok: true, outcome: (0, recovery_executor_1.toRecoveryOutcome)(result), result });
+        }
+        catch (err) {
+            this.logger.error('Recovery attempt failed', { chatId, stage, error: err.message });
+            respond({ ok: false, error: err.message }, 500);
+        }
+    }
+    /**
+     * Route raw interactive-terminal input to the session with this actual
+     * sessionId (Issue #201). The dashboard's Terminal Viewer input mode
+     * streams keystrokes here via the pty-stream WebSocket. Sessions are keyed by
+     * chatId internally, so we match on the process's own sessionId. Returns true
+     * only when a live pty-shell session accepted the bytes (headless / missing /
+     * not-writable → false), so the caller can surface an accurate result.
+     */
+    sendInputToSession(sessionId, data) {
+        if (!sessionId || typeof data !== 'string' || data.length === 0)
+            return false;
+        for (const proc of this.sessions.values()) {
+            if (proc.sessionId === sessionId) {
+                return proc.sendInput(data);
+            }
+        }
+        return false;
+    }
+    /**
+     * Build the recovery effects bound to one chat's session (Phase 3b). Each
+     * effect resolves the session fresh so it survives a restart mid-recovery.
+     * Keystroke effects go through the wrapper control channel; restart/backend
+     * effects act on the session and safe-mode manager. Effects intentionally
+     * omitted (redeliver-forward, restart-receiver, bridge-menu) are transport /
+     * wrapper concerns not bridged to the runner — the executor reports them as
+     * unsupported rather than guessing.
+     */
+    buildRecoveryEffects(chatId) {
+        const control = (key, option) => {
+            this.sessions.get(chatId)?.sendControl(key, option);
+        };
+        return {
+            esc: () => control('esc'),
+            escEsc: () => control('esc-esc'),
+            enter: () => control('enter'),
+            selectOption: (option) => control('select-option', option),
+            restartSession: () => this.restartProcess(chatId),
+            fallbackHeadless: async () => {
+                // Flip to the headless backend, then restart so the new session respawns
+                // headless (spawnSession reads safeMode.isActive → forceHeadless).
+                this.safeMode.enter(this.agentConfig.id, 'recovery: fallback-headless');
+                await this.restartProcess(chatId);
+            },
+            resendLast: () => {
+                const lt = this.lastTurn.get(chatId);
+                // Guard: only resend a turn that produced no output and was not already
+                // resent — never double-submit.
+                if (!lt || lt.delivered || lt.resent)
+                    return false;
+                const s = this.sessions.get(chatId);
+                if (!s)
+                    return false;
+                lt.resent = true;
+                s.setProcessing(true);
+                s.sendMessage(lt.text);
+                s.touch();
+                return true;
+            },
+        };
+    }
+    /**
+     * Evict finished-turn budget entries (Epic #195, Phase 3b). Called on each
+     * budget write so the per-turn map cannot grow without bound on a long-lived
+     * runner. `nowMs` is the timestamp of the write just made; any entry whose last
+     * attempt predates the TTL belongs to a turn that is no longer being recovered.
+     */
+    pruneRecoveryBudgets(nowMs) {
+        const cutoff = nowMs - RECOVERY_BUDGET_TTL_MS;
+        for (const [key, state] of this.recoveryBudgets) {
+            if (state.lastAt < cutoff)
+                this.recoveryBudgets.delete(key);
+        }
+    }
     /**
      * Handle POST /command requests from the receiver process.
      * Supports: get_model, set_model, restart.
@@ -599,7 +836,12 @@ class AgentRunner extends events_1.EventEmitter {
                 }
             }
             session.setProcessing(true);
-            session.sendMessage(blocks.join('\n'));
+            const turnText = blocks.join('\n');
+            session.sendMessage(turnText);
+            // Remember this turn for the C1 guarded resend (Phase 3b): reset the
+            // delivered/resent flags so a resend can only fire if this turn stalls
+            // before producing output.
+            this.lastTurn.set(chatId, { text: turnText, delivered: false, resent: false });
             session.touch();
             this.logger.debug('Injected channel turn into session', {
                 chatId,
@@ -716,19 +958,32 @@ class AgentRunner extends events_1.EventEmitter {
         // Apply per-session model override (caller already normalized to undefined when == agent default)
         if (modelOverride)
             proc.modelOverride = modelOverride;
+        // Per-agent → global → MAX_HISTORY_MESSAGES: the configured cap on how many
+        // history messages a healthy spawn re-injects. Lets an operator lower the
+        // context loaded at session start (e.g. 50 → 30) without touching code.
+        const configuredMax = (0, process_1.resolveMaxHistoryMessages)(this.agentConfig.history?.maxHistoryMessages, this.gatewayConfig.gateway.history?.maxHistoryMessages);
         // request_too_large (32MB) recovery: shrink the re-injected history on each
         // consecutive retry so a pathological context eventually fits. recoveryCount
-        // is 0 for healthy sessions → ladder rung 0 = MAX_HISTORY_MESSAGES (same as
-        // the SessionProcess default), so applying it unconditionally is a no-op for
-        // healthy sessions and the only path that sets historyLimit for recovering ones.
+        // is 0 for healthy sessions → use the configured cap directly; a recovering
+        // session steps down to the ladder rungs STRICTLY BELOW that cap, so each
+        // retry genuinely shrinks instead of re-trying the same (already-too-large)
+        // size when the cap has been lowered.
         const recoveryCount = this.tooLargeRecoveries.get(mapKey) ?? 0;
-        const ladderIdx = Math.min(recoveryCount, TOO_LARGE_HISTORY_LADDER.length - 1);
-        proc.historyLimit = TOO_LARGE_HISTORY_LADDER[ladderIdx];
+        proc.historyLimit = this.spawnHistoryLimit(configuredMax, recoveryCount);
         if (recoveryCount > 0) {
             this.logger.info('Spawning with reduced history after request_too_large', {
                 mapKey, recoveryCount, historyLimit: proc.historyLimit,
             });
         }
+        // Safe mode (Epic #195, Phase 3): if the PTY backend has repeatedly failed
+        // for this agent, force the headless backend for this spawn. Cleared
+        // automatically once safe mode exits (a later healthy turn / user restore).
+        if (this.safeMode.isActive(this.agentConfig.id)) {
+            proc.forceHeadless = true;
+            this.logger.info('Spawning in safe mode (headless backend forced)', {
+                mapKey, agentId: this.agentConfig.id,
+            });
+        }
         await proc.start();
         // Forward all session output lines so listeners on AgentRunner (GatewayRouter,
         // CronScheduler, tests) receive them without needing individual session references.
@@ -736,7 +991,16 @@ class AgentRunner extends events_1.EventEmitter {
         // Notify typing indicator when session permanently fails (max restarts exceeded)
         if (source !== 'api') {
             proc.once('failed', () => {
-                this.writeTypingError(mapKey, 'PROCESS_FAILED');
+                // Safe mode (Epic #195, Phase 3): a hard failure of the PTY (interactive
+                // TUI) backend counts toward the safe-mode threshold. Once crossed, the
+                // agent auto-flips to headless on the next spawn so the user's next
+                // message is served instead of re-wedging. Headless failures don't count
+                // (there is no PTY wrapper to blame / fall back from). When we just
+                // entered safe mode, tell the user that specifically instead of the
+                // generic "stopped" notice (one .error file, so pick the better code).
+                const enteredSafeMode = proc.backend === 'pty-shell' &&
+                    this.safeMode.recordPtyFailure(this.agentConfig.id);
+                this.writeTypingError(mapKey, enteredSafeMode ? 'SAFE_MODE_ENABLED' : 'PROCESS_FAILED');
                 this.sessions.delete(mapKey);
                 // LINE: surface an error for any outstanding postback button so a tap
                 // returns the interrupted notice instead of a stale "still thinking".
@@ -899,6 +1163,13 @@ class AgentRunner extends events_1.EventEmitter {
                             // the next 32MB (if any) starts fresh at the top of the ladder.
                             this.tooLargeRecoveries.delete(mapKey);
                             this.tooLargeExhausted.delete(mapKey);
+                            // Safe mode (Epic #195, Phase 3): a healthy PTY turn resets the
+                            // consecutive-failure counter and lifts safe mode if it was active
+                            // (the interactive backend recovered). Headless successes don't
+                            // touch it — that's the fallback doing its job, not the PTY healing.
+                            if (proc.backend === 'pty-shell') {
+                                this.safeMode.recordSuccess(this.agentConfig.id);
+                            }
                         }
                         // When a menu was rendered to buttons this turn, the wrapper appends
                         // the same menu text to the turn's result — strip that suffix so it
@@ -941,6 +1212,11 @@ class AgentRunner extends events_1.EventEmitter {
                             else {
                                 this.writeAutoForward(mapKey, text);
                             }
+                            // Assistant output reached the channel — mark the turn delivered so
+                            // a later recovery does not resend a message that was answered (C1).
+                            const lt = this.lastTurn.get(mapKey);
+                            if (lt)
+                                lt.delivered = true;
                         }
                         replyCalled = false; // reset for next turn
                         replyToolUseId = null;
@@ -1261,6 +1537,30 @@ class AgentRunner extends events_1.EventEmitter {
         }
         // Process will be re-spawned on next incoming message
     }
+    /**
+     * The 32MB-recovery rungs for a given healthy cap: the ladder sizes STRICTLY
+     * below the cap, in descending order. Filtering by `< cap` (not `<=`) drops any
+     * rung equal to or above the cap so a lowered cap never yields a recovery step
+     * that re-injects the same (or more) history — every step actually shrinks.
+     */
+    recoveryRungs(configuredMax) {
+        return TOO_LARGE_HISTORY_LADDER.filter(r => r < configuredMax);
+    }
+    /**
+     * History re-injection cap for a spawn, given the configured healthy cap and how
+     * many consecutive 32MB recoveries have happened on the session. recoveryCount 0
+     * = healthy → the full configured cap. Each later recovery drops to the next
+     * rung strictly below the cap; once those are exhausted it stays at 0 (no
+     * history). Kept in sync with the exhaustion threshold in handleRequestTooLarge.
+     */
+    spawnHistoryLimit(configuredMax, recoveryCount) {
+        if (recoveryCount <= 0)
+            return configuredMax;
+        const rungs = this.recoveryRungs(configuredMax);
+        if (rungs.length === 0)
+            return 0;
+        return rungs[Math.min(recoveryCount - 1, rungs.length - 1)];
+    }
     /**
      * Unified recovery for the recoverable "Request too large (max 32MB)" error.
      * Reached from two backends that surface the SAME error differently:
@@ -1270,21 +1570,26 @@ class AgentRunner extends events_1.EventEmitter {
      *    (is_error + "Request too large (max"); the long-lived process otherwise
      *    stays alive and rejects every subsequent turn forever (Bug B).
      *
-     * Each consecutive recovery shrinks the history re-injected on the next spawn
-     * (TOO_LARGE_HISTORY_LADDER: 50→40→30→20→10→0) so a pathological context drops
-     * under the 32MB ceiling. The respawn happens on the user's NEXT message (no
-     * auto-loop), and the counter resets on the next successful result. Once even a
-     * zero-history spawn still trips 32MB, stop escalating and ask the user to
-     * /clear rather than climb the ladder again.
+     * Each consecutive recovery shrinks the history re-injected on the next spawn,
+     * stepping down the TOO_LARGE_HISTORY_LADDER rungs strictly below the configured
+     * cap (default 50 → 40→30→20→10→0), so a pathological context drops under the
+     * 32MB ceiling. The respawn happens on the user's NEXT message (no auto-loop),
+     * and the counter resets on the next successful result. Once even a zero-history
+     * spawn still trips 32MB, stop escalating and ask the user to /clear rather than
+     * climb the ladder again.
      */
     handleRequestTooLarge(mapKey, proc) {
         proc.setProcessing(false);
+        // Recovery steps through the ladder rungs strictly below the configured cap;
+        // the number of those rungs is how many shrink attempts exist before even the
+        // smallest (0-history) spawn has been tried and still trips 32MB.
+        const configuredMax = (0, process_1.resolveMaxHistoryMessages)(this.agentConfig.history?.maxHistoryMessages, this.gatewayConfig.gateway.history?.maxHistoryMessages);
+        const stepCount = this.recoveryRungs(configuredMax).length; // shrink attempts available
         const count = (this.tooLargeRecoveries.get(mapKey) ?? 0) + 1;
-        const lastRung = TOO_LARGE_HISTORY_LADDER.length - 1; // index of the 0-history rung
-        if (count > lastRung) {
-            // Even the zero-history spawn tripped 32MB — context can't shrink further.
-            // Pin the counter at the last rung and always surface the /clear next step.
-            this.tooLargeRecoveries.set(mapKey, lastRung);
+        if (count > stepCount) {
+            // Even the smallest rung tripped 32MB — context can't shrink further.
+            // Pin the counter at the last step and always surface the /clear next step.
+            this.tooLargeRecoveries.set(mapKey, stepCount);
             this.logger.error('Request too large persists with zero re-injected history', { mapKey, count });
             this.writeAutoForward(mapKey, '⚠️ ยังเกิน 32MB แม้จะล้าง context จนว่างแล้ว — พิมพ์ /clear เพื่อเริ่มเซสชันใหม่ หรือ /restart ค่ะ');
             // Restart ONCE to clear the wedged process the first time the ladder is
@@ -1297,7 +1602,7 @@ class AgentRunner extends events_1.EventEmitter {
         }
         this.tooLargeRecoveries.set(mapKey, count);
         this.logger.warn('Request too large (32MB) — restarting with reduced history', {
-            mapKey, attempt: count, nextHistoryLimit: TOO_LARGE_HISTORY_LADDER[count],
+            mapKey, attempt: count, nextHistoryLimit: this.spawnHistoryLimit(configuredMax, count),
         });
         // Ordering matters and makes the notice delivery race-free: writeAutoForward
         // persists the `.forward` file synchronously HERE, before restartProcess()