mixdog 0.7.1

package/src/agent/bridge-stall-watchdog.mjs

@@ -0,0 +1,337 @@
+/**
+ * Bridge stall watchdog — per-session ticker that fires a `notifyFn` alert
+ * when a bridge worker's SSE stream goes silent for too long.
+ *
+ * Motivation (v0.6.233):
+ *   The global stream-watchdog already aborts on a hard 600s stall, but the
+ *   bridge worker's `notifyFn` only fires on the completion path. A stall
+ *   that never reaches the hard-abort boundary (e.g. provider goes quiet
+ *   mid-iteration and the lead gives up waiting) left the lead waiting
+ *   indefinitely with no "worker finished" notification.
+ *
+ *   This watchdog sits inside the bridge worker lifecycle, not the
+ *   orchestrator. It uses the same staleness signal (lastStreamDeltaAt
+ *   falling back to askStartedAt) and emits a user-facing notification
+ *   via the existing notifyFn path once the per-session threshold is
+ *   crossed — then aborts the session so the outer try/catch renders
+ *   the normal error footer.
+ *
+ * Non-goals:
+ *   - Does not replace the global stream-watchdog (that still runs at
+ *     300s/600s for provider-level stalls that never dispatched via bridge).
+ *   - Treats normal tool work as expected silence, but still aborts tools
+ *     that exceed the role's toolRunningSeconds cap.
+ */
+
+import { flushSessionMetrics, hideSessionFromList, closeSession } from './orchestrator/session/manager.mjs';
+import {
+    DEFAULT_STALL_ABORT_S,
+    DEFAULT_STALL_WARN_S,
+    STALL_TICK_MS,
+    envThresholdSeconds,
+    resolveBridgeStallThresholds,
+    resolveBridgeToolThresholdSeconds,
+} from './orchestrator/stall-policy.mjs';
+
+// How long a terminal-stage session must sit idle before the watchdog
+// flushes metrics, closes/tombstones it, and hides it from bridge type=list
+// output. Aligned to 1h (3600s) to match index.mjs's _scheduleBridgeReap so
+// the bridge worker lifecycle has ONE consistent terminal window — the 1h
+// reap keeps a finished worker resumable for same-task reuse; without this
+// alignment the watchdog would hide a session long before the reap timer.
+const TERMINAL_REAP_MS = 3_600_000;
+
+const TICK_MS = STALL_TICK_MS;
+// DEFAULT_WARN_S / DEFAULT_ABORT_S — two-stage stall envelope for roles not
+// declared in defaults/hidden-roles.json (public / custom user-workflow roles
+// such as "worker", "reviewer", or any user-defined name). These roles carry
+// no stallCap entry so the fallback keeps watchdog coverage without requiring
+// every custom role to be registered.
+// warn=abort=600s → mid-stream "slow" notify is disabled; the watchdog emits a
+//   single notification at the abort deadline ("stalled — aborting") and aborts
+//   the session. (resolveBridgeStallThresholds sets warn === abort.)
+const DEFAULT_WARN_S = DEFAULT_STALL_WARN_S;
+const DEFAULT_ABORT_S = DEFAULT_STALL_ABORT_S;
+
+// Read stallCap from the declarative hidden-role config (defaults/hidden-roles.json).
+// User-workflow roles (worker, reviewer, custom names) carry no stallCap and fall
+// through to DEFAULT_WARN_S / DEFAULT_ABORT_S — no per-role code branches here.
+// STALL_TIMEOUT_S env override sets the abort boundary; warn is clamped to
+// min(300, abort/2) so it always fires before abort.
+function resolveStallThresholds(role) {
+    return resolveBridgeStallThresholds(role);
+}
+
+function resolveToolThresholdSeconds(role, thresholdSeconds) {
+    return resolveBridgeToolThresholdSeconds(role, thresholdSeconds);
+}
+
+/**
+ * Decide whether an entry is stalled right now.
+ * Pure function — exposed for tests so we can feed synthetic runtime shapes.
+ *
+ * Returns one of:
+ *   'skip'  — entry missing, closed, or in tool_running / terminal stage.
+ *   'ok'    — entry live but below threshold.
+ *   'stall' — stale beyond threshold; caller should notify + abort.
+ *
+ * `tool_running` is treated as a stall ONLY when the per-tool runtime exceeds
+ * the role-specific threshold from `resolveToolThresholdSeconds` (defaults are
+ * generous so normal long-running tools — build, test, archive — do not trip).
+ * Below that threshold the verdict is `skip`, because tool work is client-side
+ * and the server is not silent.
+ * Terminal stages (idle/done/error/cancelling) are skipped too since
+ * askSession has already returned or is unwinding.
+ */
+export function inspectBridgeEntry(entry, abortSeconds = DEFAULT_ABORT_S, now = Date.now(), role = null, firstByteAbortSeconds = abortSeconds) {
+    if (!entry) return { verdict: 'skip' };
+    // Success-path empty synthesis — the loop finished without emitting
+    // any content/reasoning, but it is NOT a stall: no abort, no notify.
+    // Classified explicitly AHEAD of closed/stage so the stall verdict can
+    // never mask it, and so an entry that is both closed and empty-final
+    // still reports its empty-synthesis classification rather than 'skip'.
+    if (entry.emptyFinal) {
+        return { verdict: 'empty-synthesis', stage: entry.stage || null };
+    }
+    if (entry.closed) return { verdict: 'skip' };
+    const stage = entry.stage || null;
+    if (stage === 'tool_running') {
+        const toolStart = entry.toolStartedAt;
+        const toolThreshold = resolveToolThresholdSeconds(role, abortSeconds);
+        if (toolStart) {
+            const toolRuntimeS = Math.round((now - toolStart) / 1000);
+            if (toolRuntimeS >= toolThreshold) {
+                return { verdict: 'stall', staleSeconds: toolRuntimeS, stage, reason: 'tool-runtime-exceeded', toolName: entry.lastToolCall || null };
+            }
+            return { verdict: 'skip' };
+        }
+        // toolStartedAt missing — fall back to last stream delta / ask start so a
+        // session pinned in tool_running with no per-tool clock can still recover.
+        const ref = entry.lastStreamDeltaAt || entry.lastProgressAt || entry.updatedAt || entry.askStartedAt;
+        if (!ref) return { verdict: 'skip' };
+        const staleSeconds = Math.round((now - ref) / 1000);
+        if (staleSeconds < toolThreshold) return { verdict: 'ok', staleSeconds, stage };
+        return { verdict: 'stall', staleSeconds, stage, reason: 'tool-runtime-fallback', toolName: entry.lastToolCall || null };
+    }
+    if (stage === 'idle' || stage === 'done' || stage === 'error' || stage === 'cancelling') {
+        // Terminal stages never abort, but may need flush+hide after 120s (fix B).
+        const progressRef = entry.lastProgressAt || entry.doneAt || entry.updatedAt;
+        if (progressRef && (now - progressRef) >= TERMINAL_REAP_MS) {
+            return { verdict: 'terminal-reap', staleSeconds: Math.round((now - progressRef) / 1000), stage };
+        }
+        return { verdict: 'skip' };
+    }
+    // No first byte yet: still 'requesting' and never produced a single SSE
+    // delta — the stream never started, so it is hung, not slow reasoning.
+    // Abort on the shorter first-byte deadline so the lead gets a
+    // 'no first byte — aborting' notice in minutes, not the full 10-min window.
+    //
+    // Per-provider-call scoping: lastStreamDeltaAt persists across turns on
+    // the same session, so after a tool call the next 'requesting' phase
+    // still observes the previous turn's delta timestamp. Treat any
+    // lastStreamDeltaAt that predates the current 'requesting' phase
+    // (lastProgressAt is updated on the stage transition) as belonging to a
+    // prior call — the first-byte deadline applies to this fresh
+    // post-tool request, measured from the phase start rather than the
+    // original askStartedAt (which would be a stale pre-tool timestamp).
+    if (stage === 'requesting' && entry.askStartedAt) {
+        const phaseStart = entry.lastProgressAt || entry.askStartedAt;
+        const sawFirstByteThisCall = entry.lastStreamDeltaAt && entry.lastStreamDeltaAt >= phaseStart;
+        if (!sawFirstByteThisCall) {
+            const waitS = Math.round((now - phaseStart) / 1000);
+            if (waitS >= firstByteAbortSeconds) return { verdict: 'stall', staleSeconds: waitS, stage, reason: 'no-first-byte' };
+            return { verdict: 'ok', staleSeconds: waitS, stage };
+        }
+    }
+    const ref = entry.lastStreamDeltaAt || entry.lastProgressAt || entry.updatedAt || entry.askStartedAt;
+    if (!ref) return { verdict: 'skip' };
+    const staleSeconds = Math.round((now - ref) / 1000);
+    if (staleSeconds < abortSeconds) return { verdict: 'ok', staleSeconds, stage };
+    return { verdict: 'stall', staleSeconds, stage };
+}
+
+/**
+ * Start a per-session stall watchdog.
+ *
+ * @param {object} params
+ * @param {string} params.sessionId
+ * @param {() => object|null} params.getRuntime      returns manager.getSessionRuntime(sessionId)
+ * @param {() => number} params.getIteration        returns latest known iteration count
+ * @param {(reason: Error) => void} params.abort    aborts the session controller
+ * @param {(msg: string) => void} params.notify     notifyFn-style emitter
+ * @param {string} [params.modelTag]                `[model] ` prefix to match other bridge emits
+ * @param {string} [params.role]
+ * @param {number} [params.warnSeconds]             override for tests; falls back to resolveStallThresholds
+ * @param {number} [params.abortSeconds]            override for tests; falls back to resolveStallThresholds
+ * @param {number} [params.tickMs]                  override for tests
+ * @returns {{ stop: () => void, fired: () => boolean }}
+ */
+export function startBridgeStallWatchdog(params) {
+    const {
+        sessionId,
+        // Caller may supply getSessionId() so the watchdog reads the *current*
+        // session id at flush/hide time. Detached bridge dispatch replaces
+        // activeSession on each retry attempt; without this the watchdog would
+        // flush/hide the original (already-closed) id and miss the replacement.
+        getSessionId,
+        getRuntime,
+        getIteration,
+        abort,
+        notify,
+        modelTag = '',
+        role = 'worker',
+        tickMs = TICK_MS,
+    } = params;
+    const currentSessionId = () => {
+        if (typeof getSessionId === 'function') {
+            try { return getSessionId() || sessionId; } catch { return sessionId; }
+        }
+        return sessionId;
+    };
+    const _defaults = resolveStallThresholds(role);
+    const warnSeconds = params.warnSeconds ?? _defaults.warn;
+    const abortSeconds = params.abortSeconds ?? _defaults.abort;
+    const firstByteAbortSeconds = params.firstByteAbortSeconds ?? _defaults.firstByteAbort ?? abortSeconds;
+
+    let fired = false;
+    let warned = false;
+    let handle = null;
+    // Tracks whether the stall-abort notify(msg) actually delivered. The outer
+    // bridge catch in src/agent/index.mjs used to set _delivered=true solely on
+    // fired(), which dropped the pending record even when notify rejected and
+    // the fallback addPending was the only surviving trace. Expose this so the
+    // caller only marks delivered when the user-facing notice really landed.
+    let _stallNotifyDelivered = false;
+
+    // Track which sessions have already been reaped to avoid repeat flushes.
+    let _reaped = false;
+    // Whether getRuntime() has ever returned a non-null entry. Once true, a
+    // subsequent null entry means the session was closed/reaped externally
+    // (e.g. closeSession() cleared _runtimeState before the watchdog's
+    // terminal-reap path could observe a terminal stage). Fix (3) uses this
+    // to stop the interval instead of polling a vanished entry forever.
+    let _seenEntry = false;
+
+    const tick = async () => {
+        let entry = null;
+        try { entry = getRuntime(); } catch { entry = null; }
+        if (entry) {
+            _seenEntry = true;
+        } else if (_seenEntry) {
+            // Entry vanished after at least one live observation — the session
+            // was closed externally. Clear the interval so the timer can't
+            // outlive the session and keep polling a null runtime forever.
+            if (handle) { clearInterval(handle); handle = null; }
+            return;
+        }
+        const res = inspectBridgeEntry(entry, abortSeconds, Date.now(), role, firstByteAbortSeconds);
+        const now = Date.now();
+        const _stage = entry?.stage ?? 'unknown';
+        const _askAgeS = entry?.askStartedAt ? ((now - entry.askStartedAt) / 1000).toFixed(1) : 'unknown';
+        const _lastDeltaAgeS = entry?.lastStreamDeltaAt ? ((now - entry.lastStreamDeltaAt) / 1000).toFixed(1) : 'never';
+        const _lastToolCall = entry?.lastToolCall || 'none';
+
+        // Shared message construction for the warn ("slow") and abort
+        // ("stalled") stages — both pull iter/iterPart/isToolStall/toolPart/
+        // causePart from the same `res` / `getIteration()` and only differ
+        // in the verb used for a tool stall ("tool slow" vs "tool stalled")
+        // and a trailing suffix. Closed over the per-tick `_stage`/`_askAgeS`/
+        // `_lastDeltaAgeS`/`_lastToolCall` snapshot above.
+        const buildStallMessage = ({ verb, toolVerb, suffix }) => {
+            const iter = (() => { try { return getIteration(); } catch { return null; } })();
+            const iterPart = typeof iter === 'number' && iter > 0 ? ` at iter ${iter}` : '';
+            const isToolStall = res.reason === 'tool-runtime-exceeded' || res.reason === 'tool-runtime-fallback';
+            const toolPart = isToolStall && res.toolName ? ` in tool ${res.toolName}` : '';
+            const causePart = isToolStall
+                ? `${toolVerb}${toolPart}`
+                : (res.reason === 'no-first-byte' ? 'no first byte (request never streamed)' : 'no SSE delta');
+            return `${modelTag}${role} ${verb} — ${causePart} for ${res.staleSeconds}s${iterPart} stage=${_stage} askAgeS=${_askAgeS} lastDeltaAgeS=${_lastDeltaAgeS} lastToolCall=${_lastToolCall}${suffix}`;
+        };
+
+        // Fix B / #3: terminal-reap path — flush metrics, hide, AND
+        // close/tombstone the persisted session, no abort. hideSessionFromList
+        // alone only flips in-memory listHidden; the statusline aggregator
+        // reads the on-disk JSON and keeps rendering a non-closed bridge worker
+        // as idle until the store sweep. closeSession plants closed===true so
+        // the aggregator drops it immediately — one consistent 300s lifecycle.
+        if (res.verdict === 'terminal-reap' && !_reaped) {
+            _reaped = true;
+            const reapId = currentSessionId();
+            try { await flushSessionMetrics(reapId); } catch (e) { process.stderr.write(`[stall-watchdog] flushSessionMetrics failed: ${e?.message ?? e}\n`); }
+            try { hideSessionFromList(reapId); } catch (e) { process.stderr.write(`[stall-watchdog] hideSessionFromList failed: ${e?.message ?? e}\n`); }
+            try { closeSession(reapId, 'terminal-reap'); } catch (e) { process.stderr.write(`[stall-watchdog] closeSession failed: ${e?.message ?? e}\n`); }
+            if (handle) { clearInterval(handle); handle = null; }
+            return;
+        }
+
+        // Warn stage: stale >= warnSeconds but < abortSeconds — notify once, keep running.
+        if (!warned && !fired && res.verdict === 'ok' && res.staleSeconds >= warnSeconds) {
+            warned = true;
+            const warnMsg = buildStallMessage({ verb: 'slow', toolVerb: 'tool slow', suffix: '' });
+            try { notify(warnMsg); } catch (e) { process.stderr.write(`[stall-watchdog] notify(warn) failed: ${e?.message ?? e}\n`); }
+        }
+
+        if (fired) return;
+        // Empty-synthesis is a success-path terminal state. Skip stall
+        // handling entirely so the watchdog never aborts/notifies on a
+        // legitimately finished (but content-empty) bridge worker.
+        if (res.verdict === 'empty-synthesis') {
+            if (handle) { clearInterval(handle); handle = null; }
+            return;
+        }
+        if (res.verdict !== 'stall') return;
+        fired = true;
+        // stale >= abortSeconds: notify abort + abort session.
+        const msg = buildStallMessage({ verb: 'stalled', toolVerb: 'tool stalled', suffix: ' — aborting' });
+        // Await the notify so an async rejection (channel down, transport
+        // teardown) routes to the fallback addPending instead of being
+        // swallowed by an unobserved promise. Only flag _stallNotifyDelivered
+        // on actual success — the outer catch reads this to decide whether to
+        // drop the pending record or keep it replayable on next boot.
+        let _notifyOk = false;
+        try {
+            await Promise.resolve(notify(msg));
+            _notifyOk = true;
+        } catch (e) {
+            // notify rejected (sync throw or async rejection) — persist so
+            // recoverPending surfaces it on next boot.
+            const _dataDir = process.env.CLAUDE_PLUGIN_DATA;
+            const _sid = currentSessionId();
+            if (_dataDir && _sid) {
+                const _p = import('./orchestrator/dispatch-persist.mjs').then(({ addPending }) => {
+                    addPending(_dataDir, `stall_${_sid}_${Date.now()}`, 'bridge', [msg]);
+                });
+                _p.catch((err) => {
+                    process.stderr.write(`[stall-watchdog] addPending failed: ${err?.message ?? err}\n`);
+                    process.emit('unhandledRejection', err, _p);
+                });
+            }
+            process.stderr.write(`[stall-watchdog] notify(abort) failed: ${e?.message ?? e}\n`);
+        }
+        if (_notifyOk) _stallNotifyDelivered = true;
+        try {
+            const reason = new Error(`bridge stall watchdog: ${res.staleSeconds}s`);
+            reason.name = 'BridgeStallAbortError';
+            abort(reason);
+        } catch { /* controller already gone / non-Error rejection — let outer flow finish */ }
+        // Don't keep ticking once we've fired; outer finally will stop() us
+        // but clear eagerly so a slow unwind can't double-notify.
+        if (handle) { clearInterval(handle); handle = null; }
+    };
+
+    handle = setInterval(tick, tickMs);
+    if (typeof handle.unref === 'function') handle.unref();
+
+    return {
+        stop() {
+            if (handle) { clearInterval(handle); handle = null; }
+        },
+        fired() { return fired; },
+        // True only when the stall-abort notify(msg) actually delivered. The
+        // outer bridge catch consults this before clearing the pending record
+        // so a notify rejection keeps the loss replayable on next boot.
+        delivered() { return _stallNotifyDelivered; },
+    };
+}
+
+export const _internals = { TICK_MS, DEFAULT_WARN_S, DEFAULT_ABORT_S, resolveStallThresholds, envThresholdSeconds };