@edspencer/paddock 0.0.1 → 0.56.0

package/packages/server/dist/recovery.d.ts

@@ -0,0 +1,262 @@
+import { type RecoveryConfig } from "./recovery-config.js";
+import type { Project } from "./projects.js";
+/**
+ * The kinds of transcript entry the hang detector cares about. Everything that
+ * isn't a keeper `assistant` turn or a terminated `<task-notification>` (system
+ * lines, attachments, queue-operations, tool_result user turns, clean/running
+ * notifications, …) is `other` and doesn't move the pending state.
+ */
+export type TailEvent = "assistant" | "terminated-notification" | "other";
+/** A minimal structural view of one parsed transcript JSONL entry. */
+interface TranscriptEntry {
+    type?: string;
+    message?: {
+        role?: string;
+        content?: unknown;
+    };
+    /** Present on a `type:"queue-operation"` entry (`enqueue`/`dequeue`). */
+    operation?: string;
+    /** The payload of a `queue-operation` enqueue — for a killed background task
+     *  this IS the `<task-notification>` (string, or an array of text blocks). #347 */
+    content?: unknown;
+}
+/**
+ * Classify one already-parsed transcript entry as the hang detector sees it.
+ *
+ * A terminated `<task-notification>` reaches the transcript in TWO shapes, and we
+ * must catch both (issue #347):
+ *   - As a materialised `type:"user"` entry — but that only appears once a LATER
+ *     turn flushes the SDK's input queue, often tens of seconds after the kill.
+ *   - At the moment of the kill, as a `type:"queue-operation"` `enqueue` whose
+ *     `content` IS the notification. This is the one that lands inside the watch
+ *     window; missing it (the original bug) let every real hang go undetected.
+ *
+ * A keeper reply is `assistant`; a killed/stopped notification in either shape is
+ * `terminated-notification`; everything else (system lines, attachments, a
+ * `dequeue`, a completed/running notification, plain user text) is `other`.
+ */
+export declare function classifyEntry(entry: TranscriptEntry): TailEvent;
+/**
+ * Parse one raw JSONL line into its {@link TailEvent} plus, for a terminated
+ * notification, its human-readable `<summary>` (used by the live surface frame).
+ * Unparseable/blank → `other`.
+ */
+export declare function parseTailLine(line: string): {
+    event: TailEvent;
+    summary?: string;
+};
+/** Classify one raw JSONL line (unparseable/blank → `other`). */
+export declare function classifyLine(line: string): TailEvent;
+/**
+ * Reduce an ordered stream of tail events to whether the transcript currently ends
+ * (logically) with an un-answered terminated task-notification — the hung
+ * signature. An `assistant` entry clears any pending notification (the keeper
+ * woke); a terminated notification sets it; `other` entries are inert. True iff a
+ * terminated notification is still pending after the last event.
+ */
+export declare function detectHang(events: Iterable<TailEvent>): boolean;
+/** A scheduled-timer handle — opaque; produced by `setTimer`, passed to `clearTimer`. */
+export type TimerHandle = ReturnType<typeof setTimeout>;
+/** Appended-transcript reader result: the new text and the file's current size. */
+export interface TranscriptTail {
+    /** UTF-8 text appended since the requested offset (may be empty). */
+    text: string;
+    /** The file's current size in bytes (the next offset to read from). */
+    size: number;
+}
+/** Injectable primitives so the timing-sensitive watch is deterministically testable. */
+export interface RecoveryEngineDeps {
+    /** Resolve the recovery config for a dispatch — instance default here; the engine
+     *  layers the per-project override on top via {@link resolveRecoveryConfig}. */
+    cfg: {
+        recovery: RecoveryConfig;
+    };
+    /** Look up the target project (for its `.chats/` dir + `recovery` override). Throws
+     *  for an unknown/deleted project — the engine treats that as "nothing to recover". */
+    getProject: (slug: string) => Promise<Project>;
+    /** Inject the recovery nudge into the still-alive session (Layer 2's exact path:
+     *  `startAgentTurn({ resume, prompt: RECOVERY_NUDGE, sender: { kind: "recovery" } })`). */
+    reDrive: (project: Project, sessionId: string) => Promise<void>;
+    /** Surface the detected kill to any attached client LIVE (Layer 2 `surfaceKilledTask`)
+     *  so the "keeper is idle / Continue" affordance appears WITHOUT a refresh — the
+     *  notification is otherwise trapped in the SDK input queue until the next turn
+     *  flushes it (issue #347). `summary` is the notification's `<summary>`, if any.
+     *  Default no-op (surfacing off). */
+    surface?: (project: Project, sessionId: string, summary: string | undefined) => void;
+    /**
+     * Is a live turn ALREADY driving this session? (issue #352 — double-dispatch
+     * guard.) When the watch decides a keeper is hung, something else may have begun
+     * driving the same session in the meantime — a human message, a queued-message
+     * drain, or a prior recovery nudge. Under session-mode `chatSession(resume)` a
+     * SECOND resume interrupts the first, so an auto re-drive fired into a live turn
+     * would be swallowed (the "first message swallowed" symptom, #350/#347). If this
+     * reports the session busy, the engine DEFERS (see {@link sdkSessionLive}) rather
+     * than firing — the keeper is not idle. Default: never busy (so the pure
+     * unit-tested engine behaviour is unchanged).
+     */
+    isBusy?: (sessionId: string) => boolean;
+    /**
+     * Is a `claude` SUBPROCESS still alive on this session at the herdctl/SDK layer?
+     * (issue #397.) {@link isBusy} only reflects turns PADDOCK's hub started, so it
+     * is BLIND to a session herdctl's `SessionReaper` is keeping alive for a killed
+     * background task (keepAlive) or holding through its ~15s re-invocation grace —
+     * throughout which `reaper.isSessionLive` is TRUE. Auto re-drive resumes via a
+     * FRESH `openChatSession` = a NEW subprocess on the same session id; firing while
+     * a prior subprocess is still live spawns a COMPETING concurrent resume that the
+     * SDK resolves by interrupting the in-flight turn (`[Request interrupted by
+     * user]`) — the recovery turn then produces nothing and the user is still stuck.
+     * Wired in prod to `getSessionLifecycle()?.reaper.isSessionLive(sessionId)`;
+     * default `() => false` keeps the pure engine behaviour unchanged (batch mode /
+     * no reaper). Treated as a busy signal ORed with {@link isBusy}; when EITHER is
+     * set the engine DEFERS the fire (not a permanent stand-down — see {@link
+     * DEFAULT_SETTLE_WINDOW_MS}) so the nudge lands exactly once, after the session
+     * is genuinely idle (reaper reaped, hub not running).
+     */
+    sdkSessionLive?: (sessionId: string) => boolean;
+    /** Wall clock (ms). Default `Date.now`. */
+    now?: () => number;
+    /** Schedule a one-shot timer. Default `setTimeout`. */
+    setTimer?: (fn: () => void, ms: number) => TimerHandle;
+    /** Cancel a scheduled timer. Default `clearTimeout`. */
+    clearTimer?: (h: TimerHandle) => void;
+    /** Read the transcript bytes appended since `offset`. Default reads `<dir>/.chats/<id>.jsonl`. */
+    readTail?: (file: string, offset: number) => Promise<TranscriptTail>;
+    /**
+     * Cheap current size (bytes) of the transcript, for the arm-time baseline — a
+     * `stat`, NOT a full read. Called once per armed watch on EVERY session-mode
+     * keeper turn, so it must not slurp the (multi-MB) transcript. A missing file
+     * (turn wrote none yet) → 0. Default `fsp.stat(file).size`.
+     */
+    fileSize?: (file: string) => Promise<number>;
+    /** Poll cadence (ms) while watching. Default {@link DEFAULT_POLL_MS}. */
+    pollMs?: number;
+    /** Extra window (ms) beyond `debounceMs` to allow the kill to land. Default {@link DEFAULT_KILL_GRACE_MS}. */
+    killGraceMs?: number;
+    /**
+     * Cadence (ms) at which a DEFERRED fire re-checks liveness while the session is
+     * still held live by the hub or the reaper (issue #397). Default {@link
+     * DEFAULT_SETTLE_POLL_MS}.
+     */
+    settlePollMs?: number;
+    /**
+     * Total budget (ms) to keep deferring a fire while the session stays live before
+     * giving up, so a session the reaper never releases can't defer forever (issue
+     * #397). Default {@link DEFAULT_SETTLE_WINDOW_MS}.
+     */
+    settleWindowMs?: number;
+    /** Optional structured-log sink for observability; default no-op. */
+    log?: (msg: string, meta?: Record<string, unknown>) => void;
+}
+/** Default poll cadence — brisk enough to catch the kill+debounce, cheap enough to ignore. */
+export declare const DEFAULT_POLL_MS = 750;
+/**
+ * How long past `debounceMs` we keep watching for the kill to land. herdctl#374's
+ * teardown fires ~2–8s after the turn boundary, so the notification can appear late;
+ * this grace ensures its full `debounceMs` quiet window still fits inside the watch.
+ */
+export declare const DEFAULT_KILL_GRACE_MS = 12000;
+/**
+ * Cadence (ms) at which a deferred fire re-checks whether the session has gone
+ * idle (issue #397). Brisk enough to fire promptly once the reaper releases,
+ * cheap enough to ignore.
+ */
+export declare const DEFAULT_SETTLE_POLL_MS = 1000;
+/**
+ * How long (ms) the engine keeps DEFERRING an otherwise-ready fire while the
+ * session is still live (hub or reaper) before standing down (issue #397). Must
+ * comfortably exceed herdctl's `DEFAULT_REINVOCATION_GRACE_MS` (15s) — the reaper
+ * holds a just-killed-task session live for that grace — plus margin, so the
+ * deferred nudge lands once the reaper reaps rather than giving up early. Bounds
+ * the defer so a session the reaper never releases can't retry forever.
+ */
+export declare const DEFAULT_SETTLE_WINDOW_MS = 30000;
+/**
+ * Layer 3 automatic-recovery engine (issue #301). Arm a post-turn transcript watch
+ * after each session-mode keeper turn; if a killed-at-turn-boundary notification
+ * appears with no keeper response, auto-inject a recovery nudge — guarded by
+ * debounce + a per-session retry cap, and cleared when a human next messages.
+ */
+export declare class RecoveryEngine {
+    private readonly deps;
+    private readonly now;
+    private readonly setTimer;
+    private readonly clearTimer;
+    private readonly readTail;
+    private readonly fileSize;
+    private readonly pollMs;
+    private readonly killGraceMs;
+    private readonly settlePollMs;
+    private readonly settleWindowMs;
+    private readonly log;
+    private readonly surface;
+    private readonly isBusy;
+    private readonly sdkSessionLive;
+    /** Per-session retry/debounce bookkeeping — cleared on a human message. */
+    private readonly guards;
+    /** In-flight watches keyed by session id — re-arming or a human message cancels one. */
+    private readonly watches;
+    constructor(deps: RecoveryEngineDeps);
+    /**
+     * Arm (or re-arm) a post-turn watch for a just-completed SESSION-mode keeper turn.
+     * No-op unless SOMETHING wants it for this project: Layer 2 `surfaceKilledTask`
+     * (surface the kill live) or Layer 3 `autoReDrive` (auto re-drive, while the
+     * session has retries left). Re-arming cancels any prior watch for the same
+     * session. Safe to call fire-and-forget; never throws (a lookup/IO failure just
+     * declines to watch).
+     */
+    armWatch(input: {
+        slug: string;
+        sessionId: string;
+    }): void;
+    private armWatchAsync;
+    /**
+     * A detected turn-boundary kill is ready to recover — kick off the fire, giving
+     * it a bounded {@link settleWindowMs} budget to wait out any still-live session.
+     * See {@link fireWhenIdle} for the surface/re-drive actions and the defer guard.
+     */
+    private fire;
+    /**
+     * Handle a detected turn-boundary kill. Two independent, resolved-config-gated
+     * actions, in order:
+     *   1. `surfaceKilledTask` — push the kill to attached clients LIVE so the
+     *      "keeper is idle / Continue" affordance appears without a refresh. Never
+     *      throws; a bad sink can't wedge the engine.
+     *   2. `autoReDrive` (under the retry cap) — bump the session's retry bookkeeping
+     *      FIRST (so a re-drive that itself hangs can't exceed the cap), then inject
+     *      the recovery nudge via the shared Layer 2 path. A failed inject is logged.
+     *
+     * Liveness guard + defer (issues #352 / #397). Two busy signals mean the session
+     * is NOT idle and firing would collide with a live `claude` subprocess:
+     *   - {@link isBusy} (hub) — a live turn PADDOCK itself started (a human message,
+     *     a queued-message drain, or a prior recovery nudge).
+     *   - {@link sdkSessionLive} (reaper) — a subprocess herdctl is keeping alive for
+     *     the just-killed background task, or holding through its ~15s re-invocation
+     *     grace. `isBusy` is BLIND to this, which is what let the auto re-drive fire a
+     *     COMPETING resume and get interrupted (`[Request interrupted by user]`, #397).
+     *
+     * When EITHER is set we must NOT fire — but standing down *permanently* leaves
+     * recovery incomplete: the reaper reaps SILENTLY (no turn completes, so nothing
+     * re-arms a watch), and the user stays stuck. So instead we DEFER: re-check after
+     * {@link settlePollMs} and fire exactly once the session is genuinely idle
+     * (reaper reaped, hub not running). The deferral is bounded by
+     * {@link settleWindowMs} so a session that never releases can't retry forever, and
+     * is tracked in {@link watches} so a completing turn ({@link armWatch}) or a human
+     * message ({@link onHumanMessage}) supersedes/cancels it — no double fire.
+     */
+    private fireWhenIdle;
+    /**
+     * A human just messaged this session — clear its retry/debounce bookkeeping and
+     * cancel any in-flight watch, so a genuinely-new later hang is recovered fresh
+     * (the retry cap counts auto re-drives BETWEEN human messages).
+     */
+    onHumanMessage(sessionId: string): void;
+    /** Cancel the in-flight watch for a session, if any (idempotent). */
+    private cancelWatch;
+    /** Cancel every in-flight watch (e.g. on shutdown). */
+    stopAll(): void;
+    /** Test/inspection helper: the current retry count for a session (0 if untracked). */
+    retryCountFor(sessionId: string): number;
+    /** Test/inspection helper: whether a watch is currently armed for a session. */
+    isWatching(sessionId: string): boolean;
+}
+export {};