@integrity-labs/agt-cli 0.28.1068 → 0.28.1070

package/dist/pane-fatal-error-EL336LRO.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/lib/pane-fatal-error.ts"],"sourcesContent":["/**\n * ENG-11278 — a session that answers every turn with a fatal error is not healthy.\n *\n * ## The incident\n *\n * Pedders host v8, 2026-10-02. marty sat for ~2h20m, and grace for hours, replying\n * to every inbound message with\n *\n *     ⎿  Prompt is too long · automatic compaction failed: ENOENT: no such file or\n *        directory, mkdir '/root/.claude/projects/-root--augmented-grace-project' ·\n *        /clear to start fresh\n *\n * while manager.log said `decision=healthy … session_healthy_after=true` on every\n * poll. Nothing was going to change that: `isSessionHealthy` checks tmux + the\n * claude process (both fine), and the ENG-6153 wedge detector keys on UNDRAINED\n * inbound — but this session drains every message, it just answers each one with\n * the error. Its sibling condition, `⚠ Transcript writes are failing (ENOENT)`,\n * is drawn only in Claude Code's status bar (ENG-10957 found it by accident on\n * ada): no log line, no alert, health green.\n *\n * ## What this module decides\n *\n * Pure, no I/O, so the rules are unit-testable (same split as\n * `wedge-detection.ts`). The poll loop owns the pane capture, the per-agent\n * streak and the restart history, and performs the respawn.\n *\n *   - {@link classifyPaneFatalError}: is the pane CURRENTLY showing a fatal,\n *     self-perpetuating error? Returns a kind, never pane text.\n *   - {@link decidePaneFatalAction}: given that verdict, the streak and the\n *     recent restarts, what should the manager do this cycle?\n *\n * ## Reading the pane: blocks, not lines (Obi Wan 1436d0c3a9e5, d2dc0c460de5, 50a5cb896fad)\n *\n * The first version scanned every line for the fatal phrases and only excluded\n * `⏺` heads and `←` heads. That read the phrase anywhere else: on a continuation\n * line of the model's own reply, in a `⎿` tool result (a `Bash` that greps\n * manager.log), in a later line of an inbound message. Any of those could\n * force-respawn a working agent and page, and an untrusted sender could put\n * them there on purpose. So the pane is parsed the way Claude Code draws it.\n *\n * The capture is `tmux capture-pane -p -J` (wrapped lines joined), and splits\n * into three regions around the prompt box, whose rules (`────`) are always the\n * bottom of the screen:\n *\n *     …transcript…           <- turns, model output, status lines\n *     ──────────────         <- prompt box top\n *     ❯ <what is being typed>\n *     ──────────────         <- prompt box bottom = the LAST rule on screen\n *       ⏵⏵ bypass … · ⚠ Transcript writes are failing (ENOENT)   <- status bar\n *\n * The transcript is a sequence of BLOCKS. A block head sits at column 0:\n * `⏺` model output (text or a tool call), `←` an inbound channel message, `❯`\n * a submitted prompt, a spinner glyph (`✻ Cogitated for 1m 29s · done 5:26 PM`)\n * for status. Everything indented under a head belongs to it, including the\n * `⎿` lines: a tool's output under its `⏺`, a tip under a spinner, and a turn's\n * RESULT under its `←`/`❯` head. Claude Code indents every continuation, and\n * `-J` stops a long line wrapping into a fake column-0 head, so a reply line\n * that happens to begin with `←` or `⎿` stays inside its `⏺` block.\n *\n * A genuine turn failure has exactly one shape — the turn's FIRST continuation\n * line is a `⎿` carrying Claude Code's error, attached directly to the inbound\n * or prompt that caused it (grace on v8, verbatim):\n *\n *     ← telegram · 8837161157: Yes I think it’s best to start on standard…\n *       ⎿  Prompt is too long · automatic compaction failed: ENOENT: … · /clear to start fresh\n *\n * Only that counts, and only when it is the state of the session NOW: at least\n * {@link MIN_FATAL_TURNS} consecutive turns ending that way with no model output\n * after the first of them (a wedged session fails EVERY turn; a recovered one\n * shows `⏺` after the old error). A single failed turn, a quoted phrase, or a\n * tool printing the log is never enough.\n *\n * The `Transcript writes are failing` banner is matched only in the status bar,\n * and on its own is NOT fatal: a session can show it and keep answering. It\n * yields `transcript-writes-failing`, which the decision maps to a warning\n * alert, never a respawn. Its root cause (an orphaned transcript bind) is\n * repaired by ENG-11277's mount probe.\n *\n * Hysteresis ({@link PaneFatalConfig.minCycles}) covers the remaining window, a\n * capture taken between an error and the model's next output.\n */\n\nexport type PaneFatalKind =\n  /** `automatic compaction failed: …` or a manual `/compact` that errored. */\n  | 'compaction-failed'\n  /** `Prompt is too long · … · /clear to start fresh` without a compaction clause. */\n  | 'prompt-too-long'\n  /**\n   * Claude Code's status-bar warning that the session transcript cannot be\n   * written, with no failing turns. Warn-only: never a respawn on its own.\n   */\n  | 'transcript-writes-failing';\n\n/** Kinds that mean the session cannot answer at all, so a fresh respawn is the remedy. */\nexport function isRespawnKind(kind: PaneFatalKind | null): kind is 'compaction-failed' | 'prompt-too-long' {\n  return kind === 'compaction-failed' || kind === 'prompt-too-long';\n}\n\nexport interface PaneFatalVerdict {\n  /** Respawn kind when the session is failing every turn; else the banner kind; else null. */\n  kind: PaneFatalKind | null;\n  /** Consecutive most-recent turns whose result was a fatal error. */\n  fatalTurns: number;\n  /** `Transcript writes are failing` is in the status bar. */\n  transcriptBanner: boolean;\n}\n\nexport type PaneFatalMode = 'off' | 'shadow' | 'enforce';\n\nexport interface PaneFatalConfig {\n  mode: PaneFatalMode;\n  /** Consecutive poll cycles a kind must be seen before acting. */\n  minCycles: number;\n  /** Max fatal-error respawns per agent inside {@link restartWindowMs}. */\n  maxRestartsInWindow: number;\n  /** Sliding window for {@link maxRestartsInWindow}. */\n  restartWindowMs: number;\n}\n\nconst DEFAULTS = {\n  minCycles: 2,\n  maxRestartsInWindow: 3,\n  restartWindowMs: 6 * 60 * 60 * 1000,\n} as const;\n\n/**\n * Consecutive failed turns required on one screen. Grace's pane shows the error\n * once per inbound; one failed turn is a bad turn, two in a row with nothing\n * from the model in between is a session that cannot answer.\n */\nexport const MIN_FATAL_TURNS = 2;\n\n/** A prompt-box rule. Claude Code draws it full width; 8 is a floor, not a guess at width. */\nconst RULE = /^\\s*[─━]{8,}/;\n/** `⏺` anywhere a line starts (indent tolerated): any model output resets the streak. */\nconst MODEL_HEAD = /^\\s*⏺/;\n/** Spinner / timing heads: `✻ Cogitated for 1m 29s · done 5:26 PM`, `* Churning…`. */\nconst STATUS_HEAD = /^[✻✢✳✶✽✺✹·*]/;\n/** A turn's result line: indented, then `⎿`. Captures the text after the glyph. */\nconst RESULT_LINE = /^\\s+⎿\\s*(.*)$/;\n/**\n * The error text Claude Code renders as a turn result, anchored at the START of\n * the `⎿` text so a result that merely contains the phrase does not match.\n */\nconst COMPACTION_FAILED = /^(?:Prompt is too long · automatic compaction failed\\b|Error during compaction\\b)/;\nconst PROMPT_TOO_LONG = /^Prompt is too long\\b.*\\/clear to start fresh/;\nconst TRANSCRIPT_WRITES_FAILING = /Transcript writes are failing/;\n\ntype Block =\n  | { type: 'model' }\n  | { type: 'turn'; result: string | null }\n  | { type: 'neutral' };\n\nfunction headType(line: string): 'model' | 'turn' | 'empty-prompt' | 'status' | 'other' | null {\n  if (MODEL_HEAD.test(line)) return 'model';\n  if (line.length === 0 || /^\\s/.test(line)) return null; // continuation\n  if (line.startsWith('←')) return 'turn';\n  if (line.startsWith('❯')) return line.slice(1).trim().length > 0 ? 'turn' : 'empty-prompt';\n  if (STATUS_HEAD.test(line)) return 'status';\n  return 'other';\n}\n\nfunction parseBlocks(lines: readonly string[]): Block[] {\n  const blocks: Block[] = [];\n  let turn: { type: 'turn'; result: string | null; sawContinuation: boolean } | null = null;\n  for (const line of lines) {\n    const head = headType(line);\n    if (head === null) {\n      // Continuation of the current block. For a turn, only the FIRST non-blank\n      // continuation line can be its result; anything after it is the message's\n      // own text or later output, never Claude Code's verdict on the turn.\n      if (turn && !turn.sawContinuation && line.trim().length > 0) {\n        turn.sawContinuation = true;\n        const m = RESULT_LINE.exec(line);\n        turn.result = m ? m[1]! : null;\n      }\n      continue;\n    }\n    turn = null;\n    if (head === 'model') blocks.push({ type: 'model' });\n    else if (head === 'turn') {\n      turn = { type: 'turn', result: null, sawContinuation: false };\n      blocks.push(turn);\n    } else blocks.push({ type: 'neutral' });\n  }\n  return blocks;\n}\n\nfunction fatalKindOf(result: string): 'compaction-failed' | 'prompt-too-long' | null {\n  if (COMPACTION_FAILED.test(result)) return 'compaction-failed';\n  if (PROMPT_TOO_LONG.test(result)) return 'prompt-too-long';\n  return null;\n}\n\n/**\n * Parse a `tmux capture-pane -p -J` capture. Pure; never returns pane text.\n * An unreadable (null/empty) capture is \"can't tell\", which the caller treats\n * as healthy, never as a reason to respawn.\n */\nexport function classifyPane(pane: string | null | undefined): PaneFatalVerdict {\n  const clean: PaneFatalVerdict = { kind: null, fatalTurns: 0, transcriptBanner: false };\n  if (!pane) return clean;\n  const lines = pane.split('\\n').map((l) => l.replace(/\\s+$/, ''));\n\n  const rules: number[] = [];\n  lines.forEach((l, i) => {\n    if (RULE.test(l)) rules.push(i);\n  });\n  // No prompt box on screen: no status bar to read, so the banner fails closed.\n  const lastRule = rules.length > 0 ? rules[rules.length - 1]! : -1;\n  const statusBar = lastRule >= 0 ? lines.slice(lastRule + 1) : [];\n  const transcriptEnd = rules.length >= 2 ? rules[rules.length - 2]! : lastRule >= 0 ? lastRule : lines.length;\n  const transcriptBanner = statusBar.some((l) => TRANSCRIPT_WRITES_FAILING.test(l));\n\n  let fatalTurns = 0;\n  let lastFatal: 'compaction-failed' | 'prompt-too-long' | null = null;\n  for (const block of parseBlocks(lines.slice(0, transcriptEnd))) {\n    if (block.type === 'model') {\n      fatalTurns = 0; // the model answered: whatever failed before is history\n      lastFatal = null;\n    } else if (block.type === 'turn' && block.result !== null) {\n      const kind = fatalKindOf(block.result);\n      if (kind) {\n        fatalTurns += 1;\n        lastFatal = kind;\n      } else {\n        fatalTurns = 0; // the turn produced a normal result (`/clear`, `/compact` ok)\n        lastFatal = null;\n      }\n    }\n    // A turn with no result yet is pending; neutral blocks say nothing.\n  }\n\n  const kind: PaneFatalKind | null =\n    fatalTurns >= MIN_FATAL_TURNS && lastFatal\n      ? lastFatal\n      : transcriptBanner\n        ? 'transcript-writes-failing'\n        : null;\n  return { kind, fatalTurns, transcriptBanner };\n}\n\n/** {@link classifyPane}'s kind alone. */\nexport function classifyPaneFatalError(pane: string | null | undefined): PaneFatalKind | null {\n  return classifyPane(pane).kind;\n}\n\nfunction parseMode(raw: string | undefined): PaneFatalMode {\n  const v = (raw ?? '').trim().toLowerCase();\n  if (v === 'off' || v === 'shadow' || v === 'enforce') return v;\n  // Default ENFORCE, unlike the ENG-6153 wedge detector's shadow default. That\n  // detector infers a wedge from timing heuristics that can misread a long\n  // turn; this one matches Claude Code's own fatal-error text, which no\n  // working session displays as its latest result. The cost of the wrong\n  // default is asymmetric too: shadow here is what v8 already had, an agent\n  // answering every message with an error for hours while health read green.\n  // The restart budget below bounds the downside of enforce.\n  return 'enforce';\n}\n\nfunction parseIntAtLeast(raw: string | undefined, fallback: number, floor: number): number {\n  const n = raw ? Number.parseInt(raw, 10) : NaN;\n  return Number.isInteger(n) && n >= floor ? n : fallback;\n}\n\n/**\n * Config from the environment:\n *   AGT_PANE_FATAL_RESTART_MODE        off | shadow | enforce (default enforce)\n *   AGT_PANE_FATAL_MIN_CYCLES          default 2, floor 2 (one sighting never acts)\n *   AGT_PANE_FATAL_MAX_RESTARTS        default 3 per window, floor 1\n *   AGT_PANE_FATAL_WINDOW_MINUTES      default 360, floor 30\n */\nexport function resolvePaneFatalConfig(env: NodeJS.ProcessEnv = process.env): PaneFatalConfig {\n  return {\n    mode: parseMode(env.AGT_PANE_FATAL_RESTART_MODE),\n    minCycles: parseIntAtLeast(env.AGT_PANE_FATAL_MIN_CYCLES, DEFAULTS.minCycles, 2),\n    maxRestartsInWindow: parseIntAtLeast(env.AGT_PANE_FATAL_MAX_RESTARTS, DEFAULTS.maxRestartsInWindow, 1),\n    restartWindowMs:\n      parseIntAtLeast(env.AGT_PANE_FATAL_WINDOW_MINUTES, DEFAULTS.restartWindowMs / 60_000, 30) * 60_000,\n  };\n}\n\nexport type PaneFatalAction =\n  /** Nothing to do: no fatal kind, streak too short, or detection off. */\n  | 'none'\n  /**\n   * The transcript-writes banner alone: the session is still answering, so do\n   * NOT respawn it. Log and raise a warning recommending a restart (shadow and\n   * enforce alike: a warning touches nothing).\n   */\n  | 'warn'\n  /** Shadow mode: would restart. Log once per episode, do nothing. */\n  | 'would-restart'\n  /** Enforce: force a fresh respawn now. */\n  | 'restart'\n  /**\n   * The restart budget is spent and the condition is back. A respawn has\n   * demonstrably not cleared it, so another would be a restart loop: stop\n   * restarting and escalate to a human instead.\n   */\n  | 'give-up';\n\n/**\n * What to do this cycle. `consecutiveCycles` INCLUDES this one; `recentRestartsMs`\n * are the timestamps of earlier fatal-error respawns of this agent.\n */\nexport function decidePaneFatalAction(\n  input: {\n    kind: PaneFatalKind | null;\n    consecutiveCycles: number;\n    recentRestartsMs: readonly number[];\n    nowMs: number;\n  },\n  config: PaneFatalConfig,\n): PaneFatalAction {\n  if (config.mode === 'off' || input.kind === null) return 'none';\n  if (input.consecutiveCycles < config.minCycles) return 'none';\n  if (!isRespawnKind(input.kind)) return 'warn';\n  if (config.mode === 'shadow') return 'would-restart';\n  const inWindow = input.recentRestartsMs.filter((t) => input.nowMs - t < config.restartWindowMs).length;\n  return inWindow >= config.maxRestartsInWindow ? 'give-up' : 'restart';\n}\n\n/** Drop restart timestamps that have aged out of the window (bounded memory). */\nexport function pruneRestartHistory(\n  history: readonly number[],\n  nowMs: number,\n  config: PaneFatalConfig,\n): number[] {\n  return history.filter((t) => nowMs - t < config.restartWindowMs);\n}\n"],"mappings":";;;AA8FO,SAAS,cAAc,MAA6E;AACzG,SAAO,SAAS,uBAAuB,SAAS;AAClD;AAuBA,IAAM,WAAW;AAAA,EACf,WAAW;AAAA,EACX,qBAAqB;AAAA,EACrB,iBAAiB,IAAI,KAAK,KAAK;AACjC;AAOO,IAAM,kBAAkB;AAG/B,IAAM,OAAO;AAEb,IAAM,aAAa;AAEnB,IAAM,cAAc;AAEpB,IAAM,cAAc;AAKpB,IAAM,oBAAoB;AAC1B,IAAM,kBAAkB;AACxB,IAAM,4BAA4B;AAOlC,SAAS,SAAS,MAA6E;AAC7F,MAAI,WAAW,KAAK,IAAI,EAAG,QAAO;AAClC,MAAI,KAAK,WAAW,KAAK,MAAM,KAAK,IAAI,EAAG,QAAO;AAClD,MAAI,KAAK,WAAW,QAAG,EAAG,QAAO;AACjC,MAAI,KAAK,WAAW,QAAG,EAAG,QAAO,KAAK,MAAM,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,SAAS;AAC5E,MAAI,YAAY,KAAK,IAAI,EAAG,QAAO;AACnC,SAAO;AACT;AAEA,SAAS,YAAY,OAAmC;AACtD,QAAM,SAAkB,CAAC;AACzB,MAAI,OAAiF;AACrF,aAAW,QAAQ,OAAO;AACxB,UAAM,OAAO,SAAS,IAAI;AAC1B,QAAI,SAAS,MAAM;AAIjB,UAAI,QAAQ,CAAC,KAAK,mBAAmB,KAAK,KAAK,EAAE,SAAS,GAAG;AAC3D,aAAK,kBAAkB;AACvB,cAAM,IAAI,YAAY,KAAK,IAAI;AAC/B,aAAK,SAAS,IAAI,EAAE,CAAC,IAAK;AAAA,MAC5B;AACA;AAAA,IACF;AACA,WAAO;AACP,QAAI,SAAS,QAAS,QAAO,KAAK,EAAE,MAAM,QAAQ,CAAC;AAAA,aAC1C,SAAS,QAAQ;AACxB,aAAO,EAAE,MAAM,QAAQ,QAAQ,MAAM,iBAAiB,MAAM;AAC5D,aAAO,KAAK,IAAI;AAAA,IAClB,MAAO,QAAO,KAAK,EAAE,MAAM,UAAU,CAAC;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,YAAY,QAAgE;AACnF,MAAI,kBAAkB,KAAK,MAAM,EAAG,QAAO;AAC3C,MAAI,gBAAgB,KAAK,MAAM,EAAG,QAAO;AACzC,SAAO;AACT;AAOO,SAAS,aAAa,MAAmD;AAC9E,QAAM,QAA0B,EAAE,MAAM,MAAM,YAAY,GAAG,kBAAkB,MAAM;AACrF,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,QAAQ,KAAK,MAAM,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,EAAE,CAAC;AAE/D,QAAM,QAAkB,CAAC;AACzB,QAAM,QAAQ,CAAC,GAAG,MAAM;AACtB,QAAI,KAAK,KAAK,CAAC,EAAG,OAAM,KAAK,CAAC;AAAA,EAChC,CAAC;AAED,QAAM,WAAW,MAAM,SAAS,IAAI,MAAM,MAAM,SAAS,CAAC,IAAK;AAC/D,QAAM,YAAY,YAAY,IAAI,MAAM,MAAM,WAAW,CAAC,IAAI,CAAC;AAC/D,QAAM,gBAAgB,MAAM,UAAU,IAAI,MAAM,MAAM,SAAS,CAAC,IAAK,YAAY,IAAI,WAAW,MAAM;AACtG,QAAM,mBAAmB,UAAU,KAAK,CAAC,MAAM,0BAA0B,KAAK,CAAC,CAAC;AAEhF,MAAI,aAAa;AACjB,MAAI,YAA4D;AAChE,aAAW,SAAS,YAAY,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG;AAC9D,QAAI,MAAM,SAAS,SAAS;AAC1B,mBAAa;AACb,kBAAY;AAAA,IACd,WAAW,MAAM,SAAS,UAAU,MAAM,WAAW,MAAM;AACzD,YAAMA,QAAO,YAAY,MAAM,MAAM;AACrC,UAAIA,OAAM;AACR,sBAAc;AACd,oBAAYA;AAAA,MACd,OAAO;AACL,qBAAa;AACb,oBAAY;AAAA,MACd;AAAA,IACF;AAAA,EAEF;AAEA,QAAM,OACJ,cAAc,mBAAmB,YAC7B,YACA,mBACE,8BACA;AACR,SAAO,EAAE,MAAM,YAAY,iBAAiB;AAC9C;AAGO,SAAS,uBAAuB,MAAuD;AAC5F,SAAO,aAAa,IAAI,EAAE;AAC5B;AAEA,SAAS,UAAU,KAAwC;AACzD,QAAM,KAAK,OAAO,IAAI,KAAK,EAAE,YAAY;AACzC,MAAI,MAAM,SAAS,MAAM,YAAY,MAAM,UAAW,QAAO;AAQ7D,SAAO;AACT;AAEA,SAAS,gBAAgB,KAAyB,UAAkB,OAAuB;AACzF,QAAM,IAAI,MAAM,OAAO,SAAS,KAAK,EAAE,IAAI;AAC3C,SAAO,OAAO,UAAU,CAAC,KAAK,KAAK,QAAQ,IAAI;AACjD;AASO,SAAS,uBAAuB,MAAyB,QAAQ,KAAsB;AAC5F,SAAO;AAAA,IACL,MAAM,UAAU,IAAI,2BAA2B;AAAA,IAC/C,WAAW,gBAAgB,IAAI,2BAA2B,SAAS,WAAW,CAAC;AAAA,IAC/E,qBAAqB,gBAAgB,IAAI,6BAA6B,SAAS,qBAAqB,CAAC;AAAA,IACrG,iBACE,gBAAgB,IAAI,+BAA+B,SAAS,kBAAkB,KAAQ,EAAE,IAAI;AAAA,EAChG;AACF;AA0BO,SAAS,sBACd,OAMA,QACiB;AACjB,MAAI,OAAO,SAAS,SAAS,MAAM,SAAS,KAAM,QAAO;AACzD,MAAI,MAAM,oBAAoB,OAAO,UAAW,QAAO;AACvD,MAAI,CAAC,cAAc,MAAM,IAAI,EAAG,QAAO;AACvC,MAAI,OAAO,SAAS,SAAU,QAAO;AACrC,QAAM,WAAW,MAAM,iBAAiB,OAAO,CAAC,MAAM,MAAM,QAAQ,IAAI,OAAO,eAAe,EAAE;AAChG,SAAO,YAAY,OAAO,sBAAsB,YAAY;AAC9D;AAGO,SAAS,oBACd,SACA,OACA,QACU;AACV,SAAO,QAAQ,OAAO,CAAC,MAAM,QAAQ,IAAI,OAAO,eAAe;AACjE;","names":["kind"]}