mixdog 0.7.18 → 0.8.0

package/src/agent/bridge-stall-watchdog.mjs

@@ -1,337 +0,0 @@
-/**
- * Bridge stall watchdog — per-session ticker that fires a `notifyFn` alert
- * when a bridge worker's SSE stream goes silent for too long.
- *
- * Motivation (v0.6.233):
- *   The global stream-watchdog already aborts on a hard 600s stall, but the
- *   bridge worker's `notifyFn` only fires on the completion path. A stall
- *   that never reaches the hard-abort boundary (e.g. provider goes quiet
- *   mid-iteration and the lead gives up waiting) left the lead waiting
- *   indefinitely with no "worker finished" notification.
- *
- *   This watchdog sits inside the bridge worker lifecycle, not the
- *   orchestrator. It uses the same staleness signal (lastStreamDeltaAt
- *   falling back to askStartedAt) and emits a user-facing notification
- *   via the existing notifyFn path once the per-session threshold is
- *   crossed — then aborts the session so the outer try/catch renders
- *   the normal error footer.
- *
- * Non-goals:
- *   - Does not replace the global stream-watchdog (that still runs at
- *     300s/600s for provider-level stalls that never dispatched via bridge).
- *   - Treats normal tool work as expected silence, but still aborts tools
- *     that exceed the role's toolRunningSeconds cap.
- */
-
-import { flushSessionMetrics, hideSessionFromList, closeSession } from './orchestrator/session/manager.mjs';
-import {
-    DEFAULT_STALL_ABORT_S,
-    DEFAULT_STALL_WARN_S,
-    STALL_TICK_MS,
-    envThresholdSeconds,
-    resolveBridgeStallThresholds,
-    resolveBridgeToolThresholdSeconds,
-} from './orchestrator/stall-policy.mjs';
-
-// How long a terminal-stage session must sit idle before the watchdog
-// flushes metrics, closes/tombstones it, and hides it from bridge type=list
-// output. Aligned to 1h (3600s) to match index.mjs's _scheduleBridgeReap so
-// the bridge worker lifecycle has ONE consistent terminal window — the 1h
-// reap keeps a finished worker resumable for same-task reuse; without this
-// alignment the watchdog would hide a session long before the reap timer.
-const TERMINAL_REAP_MS = 3_600_000;
-
-const TICK_MS = STALL_TICK_MS;
-// DEFAULT_WARN_S / DEFAULT_ABORT_S — two-stage stall envelope for roles not
-// declared in defaults/hidden-roles.json (public / custom user-workflow roles
-// such as "worker", "reviewer", or any user-defined name). These roles carry
-// no stallCap entry so the fallback keeps watchdog coverage without requiring
-// every custom role to be registered.
-// warn=abort=600s → mid-stream "slow" notify is disabled; the watchdog emits a
-//   single notification at the abort deadline ("stalled — aborting") and aborts
-//   the session. (resolveBridgeStallThresholds sets warn === abort.)
-const DEFAULT_WARN_S = DEFAULT_STALL_WARN_S;
-const DEFAULT_ABORT_S = DEFAULT_STALL_ABORT_S;
-
-// Read stallCap from the declarative hidden-role config (defaults/hidden-roles.json).
-// User-workflow roles (worker, reviewer, custom names) carry no stallCap and fall
-// through to DEFAULT_WARN_S / DEFAULT_ABORT_S — no per-role code branches here.
-// STALL_TIMEOUT_S env override sets the abort boundary; warn is clamped to
-// min(300, abort/2) so it always fires before abort.
-function resolveStallThresholds(role) {
-    return resolveBridgeStallThresholds(role);
-}
-
-function resolveToolThresholdSeconds(role, thresholdSeconds) {
-    return resolveBridgeToolThresholdSeconds(role, thresholdSeconds);
-}
-
-/**
- * Decide whether an entry is stalled right now.
- * Pure function — exposed for tests so we can feed synthetic runtime shapes.
- *
- * Returns one of:
- *   'skip'  — entry missing, closed, or in tool_running / terminal stage.
- *   'ok'    — entry live but below threshold.
- *   'stall' — stale beyond threshold; caller should notify + abort.
- *
- * `tool_running` is treated as a stall ONLY when the per-tool runtime exceeds
- * the role-specific threshold from `resolveToolThresholdSeconds` (defaults are
- * generous so normal long-running tools — build, test, archive — do not trip).
- * Below that threshold the verdict is `skip`, because tool work is client-side
- * and the server is not silent.
- * Terminal stages (idle/done/error/cancelling) are skipped too since
- * askSession has already returned or is unwinding.
- */
-export function inspectBridgeEntry(entry, abortSeconds = DEFAULT_ABORT_S, now = Date.now(), role = null, firstByteAbortSeconds = abortSeconds) {
-    if (!entry) return { verdict: 'skip' };
-    // Success-path empty synthesis — the loop finished without emitting
-    // any content/reasoning, but it is NOT a stall: no abort, no notify.
-    // Classified explicitly AHEAD of closed/stage so the stall verdict can
-    // never mask it, and so an entry that is both closed and empty-final
-    // still reports its empty-synthesis classification rather than 'skip'.
-    if (entry.emptyFinal) {
-        return { verdict: 'empty-synthesis', stage: entry.stage || null };
-    }
-    if (entry.closed) return { verdict: 'skip' };
-    const stage = entry.stage || null;
-    if (stage === 'tool_running') {
-        const toolStart = entry.toolStartedAt;
-        const toolThreshold = resolveToolThresholdSeconds(role, abortSeconds);
-        if (toolStart) {
-            const toolRuntimeS = Math.round((now - toolStart) / 1000);
-            if (toolRuntimeS >= toolThreshold) {
-                return { verdict: 'stall', staleSeconds: toolRuntimeS, stage, reason: 'tool-runtime-exceeded', toolName: entry.lastToolCall || null };
-            }
-            return { verdict: 'skip' };
-        }
-        // toolStartedAt missing — fall back to last stream delta / ask start so a
-        // session pinned in tool_running with no per-tool clock can still recover.
-        const ref = entry.lastStreamDeltaAt || entry.lastProgressAt || entry.updatedAt || entry.askStartedAt;
-        if (!ref) return { verdict: 'skip' };
-        const staleSeconds = Math.round((now - ref) / 1000);
-        if (staleSeconds < toolThreshold) return { verdict: 'ok', staleSeconds, stage };
-        return { verdict: 'stall', staleSeconds, stage, reason: 'tool-runtime-fallback', toolName: entry.lastToolCall || null };
-    }
-    if (stage === 'idle' || stage === 'done' || stage === 'error' || stage === 'cancelling') {
-        // Terminal stages never abort, but may need flush+hide after the terminal-reap window (TERMINAL_REAP_MS, 1h) (fix B).
-        const progressRef = entry.lastProgressAt || entry.doneAt || entry.updatedAt;
-        if (progressRef && (now - progressRef) >= TERMINAL_REAP_MS) {
-            return { verdict: 'terminal-reap', staleSeconds: Math.round((now - progressRef) / 1000), stage };
-        }
-        return { verdict: 'skip' };
-    }
-    // No first byte yet: still 'requesting' and never produced a single SSE
-    // delta — the stream never started, so it is hung, not slow reasoning.
-    // Abort on the shorter first-byte deadline so the lead gets a
-    // 'no first byte — aborting' notice in minutes, not the full 10-min window.
-    //
-    // Per-provider-call scoping: lastStreamDeltaAt persists across turns on
-    // the same session, so after a tool call the next 'requesting' phase
-    // still observes the previous turn's delta timestamp. Treat any
-    // lastStreamDeltaAt that predates the current 'requesting' phase
-    // (lastProgressAt is updated on the stage transition) as belonging to a
-    // prior call — the first-byte deadline applies to this fresh
-    // post-tool request, measured from the phase start rather than the
-    // original askStartedAt (which would be a stale pre-tool timestamp).
-    if (stage === 'requesting' && entry.askStartedAt) {
-        const phaseStart = entry.lastProgressAt || entry.askStartedAt;
-        const sawFirstByteThisCall = entry.lastStreamDeltaAt && entry.lastStreamDeltaAt >= phaseStart;
-        if (!sawFirstByteThisCall) {
-            const waitS = Math.round((now - phaseStart) / 1000);
-            if (waitS >= firstByteAbortSeconds) return { verdict: 'stall', staleSeconds: waitS, stage, reason: 'no-first-byte' };
-            return { verdict: 'ok', staleSeconds: waitS, stage };
-        }
-    }
-    const ref = entry.lastStreamDeltaAt || entry.lastProgressAt || entry.updatedAt || entry.askStartedAt;
-    if (!ref) return { verdict: 'skip' };
-    const staleSeconds = Math.round((now - ref) / 1000);
-    if (staleSeconds < abortSeconds) return { verdict: 'ok', staleSeconds, stage };
-    return { verdict: 'stall', staleSeconds, stage };
-}
-
-/**
- * Start a per-session stall watchdog.
- *
- * @param {object} params
- * @param {string} params.sessionId
- * @param {() => object|null} params.getRuntime      returns manager.getSessionRuntime(sessionId)
- * @param {() => number} params.getIteration        returns latest known iteration count
- * @param {(reason: Error) => void} params.abort    aborts the session controller
- * @param {(msg: string) => void} params.notify     notifyFn-style emitter
- * @param {string} [params.modelTag]                `[model] ` prefix to match other bridge emits
- * @param {string} [params.role]
- * @param {number} [params.warnSeconds]             override for tests; falls back to resolveStallThresholds
- * @param {number} [params.abortSeconds]            override for tests; falls back to resolveStallThresholds
- * @param {number} [params.tickMs]                  override for tests
- * @returns {{ stop: () => void, fired: () => boolean }}
- */
-export function startBridgeStallWatchdog(params) {
-    const {
-        sessionId,
-        // Caller may supply getSessionId() so the watchdog reads the *current*
-        // session id at flush/hide time. Detached bridge dispatch replaces
-        // activeSession on each retry attempt; without this the watchdog would
-        // flush/hide the original (already-closed) id and miss the replacement.
-        getSessionId,
-        getRuntime,
-        getIteration,
-        abort,
-        notify,
-        modelTag = '',
-        role = 'worker',
-        tickMs = TICK_MS,
-    } = params;
-    const currentSessionId = () => {
-        if (typeof getSessionId === 'function') {
-            try { return getSessionId() || sessionId; } catch { return sessionId; }
-        }
-        return sessionId;
-    };
-    const _defaults = resolveStallThresholds(role);
-    const warnSeconds = params.warnSeconds ?? _defaults.warn;
-    const abortSeconds = params.abortSeconds ?? _defaults.abort;
-    const firstByteAbortSeconds = params.firstByteAbortSeconds ?? _defaults.firstByteAbort ?? abortSeconds;
-
-    let fired = false;
-    let warned = false;
-    let handle = null;
-    // Tracks whether the stall-abort notify(msg) actually delivered. The outer
-    // bridge catch in src/agent/index.mjs used to set _delivered=true solely on
-    // fired(), which dropped the pending record even when notify rejected and
-    // the fallback addPending was the only surviving trace. Expose this so the
-    // caller only marks delivered when the user-facing notice really landed.
-    let _stallNotifyDelivered = false;
-
-    // Track which sessions have already been reaped to avoid repeat flushes.
-    let _reaped = false;
-    // Whether getRuntime() has ever returned a non-null entry. Once true, a
-    // subsequent null entry means the session was closed/reaped externally
-    // (e.g. closeSession() cleared _runtimeState before the watchdog's
-    // terminal-reap path could observe a terminal stage). Fix (3) uses this
-    // to stop the interval instead of polling a vanished entry forever.
-    let _seenEntry = false;
-
-    const tick = async () => {
-        let entry = null;
-        try { entry = getRuntime(); } catch { entry = null; }
-        if (entry) {
-            _seenEntry = true;
-        } else if (_seenEntry) {
-            // Entry vanished after at least one live observation — the session
-            // was closed externally. Clear the interval so the timer can't
-            // outlive the session and keep polling a null runtime forever.
-            if (handle) { clearInterval(handle); handle = null; }
-            return;
-        }
-        const res = inspectBridgeEntry(entry, abortSeconds, Date.now(), role, firstByteAbortSeconds);
-        const now = Date.now();
-        const _stage = entry?.stage ?? 'unknown';
-        const _askAgeS = entry?.askStartedAt ? ((now - entry.askStartedAt) / 1000).toFixed(1) : 'unknown';
-        const _lastDeltaAgeS = entry?.lastStreamDeltaAt ? ((now - entry.lastStreamDeltaAt) / 1000).toFixed(1) : 'never';
-        const _lastToolCall = entry?.lastToolCall || 'none';
-
-        // Shared message construction for the warn ("slow") and abort
-        // ("stalled") stages — both pull iter/iterPart/isToolStall/toolPart/
-        // causePart from the same `res` / `getIteration()` and only differ
-        // in the verb used for a tool stall ("tool slow" vs "tool stalled")
-        // and a trailing suffix. Closed over the per-tick `_stage`/`_askAgeS`/
-        // `_lastDeltaAgeS`/`_lastToolCall` snapshot above.
-        const buildStallMessage = ({ verb, toolVerb, suffix }) => {
-            const iter = (() => { try { return getIteration(); } catch { return null; } })();
-            const iterPart = typeof iter === 'number' && iter > 0 ? ` at iter ${iter}` : '';
-            const isToolStall = res.reason === 'tool-runtime-exceeded' || res.reason === 'tool-runtime-fallback';
-            const toolPart = isToolStall && res.toolName ? ` in tool ${res.toolName}` : '';
-            const causePart = isToolStall
-                ? `${toolVerb}${toolPart}`
-                : (res.reason === 'no-first-byte' ? 'no first byte (request never streamed)' : 'no SSE delta');
-            return `${modelTag}${role} ${verb} — ${causePart} for ${res.staleSeconds}s${iterPart} stage=${_stage} askAgeS=${_askAgeS} lastDeltaAgeS=${_lastDeltaAgeS} lastToolCall=${_lastToolCall}${suffix}`;
-        };
-
-        // Fix B / #3: terminal-reap path — flush metrics, hide, AND
-        // close/tombstone the persisted session, no abort. hideSessionFromList
-        // alone only flips in-memory listHidden; the statusline aggregator
-        // reads the on-disk JSON and keeps rendering a non-closed bridge worker
-        // as idle until the store sweep. closeSession plants closed===true so
-        // the aggregator drops it immediately — one consistent 1h lifecycle (TERMINAL_REAP_MS).
-        if (res.verdict === 'terminal-reap' && !_reaped) {
-            _reaped = true;
-            const reapId = currentSessionId();
-            try { await flushSessionMetrics(reapId); } catch (e) { process.stderr.write(`[stall-watchdog] flushSessionMetrics failed: ${e?.message ?? e}\n`); }
-            try { hideSessionFromList(reapId); } catch (e) { process.stderr.write(`[stall-watchdog] hideSessionFromList failed: ${e?.message ?? e}\n`); }
-            try { closeSession(reapId, 'terminal-reap'); } catch (e) { process.stderr.write(`[stall-watchdog] closeSession failed: ${e?.message ?? e}\n`); }
-            if (handle) { clearInterval(handle); handle = null; }
-            return;
-        }
-
-        // Warn stage: stale >= warnSeconds but < abortSeconds — notify once, keep running.
-        if (!warned && !fired && res.verdict === 'ok' && res.staleSeconds >= warnSeconds) {
-            warned = true;
-            const warnMsg = buildStallMessage({ verb: 'slow', toolVerb: 'tool slow', suffix: '' });
-            try { notify(warnMsg); } catch (e) { process.stderr.write(`[stall-watchdog] notify(warn) failed: ${e?.message ?? e}\n`); }
-        }
-
-        if (fired) return;
-        // Empty-synthesis is a success-path terminal state. Skip stall
-        // handling entirely so the watchdog never aborts/notifies on a
-        // legitimately finished (but content-empty) bridge worker.
-        if (res.verdict === 'empty-synthesis') {
-            if (handle) { clearInterval(handle); handle = null; }
-            return;
-        }
-        if (res.verdict !== 'stall') return;
-        fired = true;
-        // stale >= abortSeconds: notify abort + abort session.
-        const msg = buildStallMessage({ verb: 'stalled', toolVerb: 'tool stalled', suffix: ' — aborting' });
-        // Await the notify so an async rejection (channel down, transport
-        // teardown) routes to the fallback addPending instead of being
-        // swallowed by an unobserved promise. Only flag _stallNotifyDelivered
-        // on actual success — the outer catch reads this to decide whether to
-        // drop the pending record or keep it replayable on next boot.
-        let _notifyOk = false;
-        try {
-            await Promise.resolve(notify(msg));
-            _notifyOk = true;
-        } catch (e) {
-            // notify rejected (sync throw or async rejection) — persist so
-            // recoverPending surfaces it on next boot.
-            const _dataDir = process.env.CLAUDE_PLUGIN_DATA;
-            const _sid = currentSessionId();
-            if (_dataDir && _sid) {
-                const _p = import('./orchestrator/dispatch-persist.mjs').then(({ addPending }) => {
-                    addPending(_dataDir, `stall_${_sid}_${Date.now()}`, 'bridge', [msg]);
-                });
-                _p.catch((err) => {
-                    process.stderr.write(`[stall-watchdog] addPending failed: ${err?.message ?? err}\n`);
-                    process.emit('unhandledRejection', err, _p);
-                });
-            }
-            process.stderr.write(`[stall-watchdog] notify(abort) failed: ${e?.message ?? e}\n`);
-        }
-        if (_notifyOk) _stallNotifyDelivered = true;
-        try {
-            const reason = new Error(`bridge stall watchdog: ${res.staleSeconds}s`);
-            reason.name = 'BridgeStallAbortError';
-            abort(reason);
-        } catch { /* controller already gone / non-Error rejection — let outer flow finish */ }
-        // Don't keep ticking once we've fired; outer finally will stop() us
-        // but clear eagerly so a slow unwind can't double-notify.
-        if (handle) { clearInterval(handle); handle = null; }
-    };
-
-    handle = setInterval(tick, tickMs);
-    if (typeof handle.unref === 'function') handle.unref();
-
-    return {
-        stop() {
-            if (handle) { clearInterval(handle); handle = null; }
-        },
-        fired() { return fired; },
-        // True only when the stall-abort notify(msg) actually delivered. The
-        // outer bridge catch consults this before clearing the pending record
-        // so a notify rejection keeps the loss replayable on next boot.
-        delivered() { return _stallNotifyDelivered; },
-    };
-}
-
-export const _internals = { TICK_MS, DEFAULT_WARN_S, DEFAULT_ABORT_S, resolveStallThresholds, envThresholdSeconds };