claude-slack-channel-bots 0.7.2 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-slack-channel-bots",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Multi-session Slack-to-Claude bridge — run multiple Claude Code bots across Slack channels via Socket Mode",
   "type": "module",
   "bin": {
@@ -31,7 +31,7 @@
     "@modelcontextprotocol/sdk": "^1.0.0",
     "@slack/socket-mode": "^2.0.0",
     "@slack/web-api": "^7.0.0",
-    "agent-director": "^0.7.0",
+    "agent-director": "^0.7.8",
     "semver": "^7.6.0"
   },
   "devDependencies": {

package/src/agent-director-errors.ts

@@ -11,6 +11,10 @@
  *   - ErrSystemInstallNotFound  (Client.create / resolveSystemBinary — no `agent-director` on PATH or at ~/.agent-director)
  *   - ErrSystemInstallTooOld    (Client.create / resolveSystemBinary — system binary older than required minimum)
  *   - ErrSystemInstallUnreachable (Client.create / resolveSystemBinary — system binary present but not executable or fails --version)
+ *   - ErrSystemInstallDisappeared (any verb / binary gone after valid construction — b.xht)
+ *   - ErrTmuxNotAvailable       (spawn / tmux binary not found or not executable)
+ *   - ErrCwdNotFound            (spawn / route cwd does not exist on disk)
+ *   - ErrCwdNotADirectory       (spawn / route cwd path exists but is not a directory)
  *   - ErrInstanceIdCollision    (spawn / SR-1.4 idempotency)
  *   - ErrSpawnNotFound          (get / status / decide on missing row)
  *   - ErrNoSessionId            (resume / SR-1.3 fallthrough)
@@ -41,6 +45,10 @@ export {
   ErrSystemInstallNotFound,
   ErrSystemInstallTooOld,
   ErrSystemInstallUnreachable,
+  ErrSystemInstallDisappeared,
+  ErrTmuxNotAvailable,
+  ErrCwdNotFound,
+  ErrCwdNotADirectory,
   ErrCallTimeout,
   ErrInstanceIdCollision,
   ErrSpawnNotFound,

package/src/health-check.ts

@@ -8,6 +8,8 @@
  * SPDX-License-Identifier: MIT
  */
 
+import { setOutageFlag, clearOutageFlag } from './outage-state.ts'
+
 // ---------------------------------------------------------------------------
 // Types
 // ---------------------------------------------------------------------------
@@ -15,7 +17,7 @@
 export interface HealthCheckDeps {
   isSessionAlive(channelId: string): Promise<boolean>
   isRestartPendingOrActive(channelId: string): boolean
-  hasReachedMaxFailures(channelId: string): boolean
+  statRoute(cwd: string): Promise<boolean>
   scheduleRestart(channelId: string, cwd: string): void
   isShuttingDown(): boolean
   getRoutes(): Record<string, string>
@@ -27,6 +29,8 @@ export interface HealthCheckDeps {
 
 let deps: HealthCheckDeps | null = null
 let intervalId: ReturnType<typeof setInterval> | null = null
+let tickInFlight = false
+let skippedTicks = 0
 
 // ---------------------------------------------------------------------------
 // initHealthCheck
@@ -46,21 +50,45 @@ export function startHealthCheck(intervalSeconds: number): void {
   intervalId = setInterval(async () => {
     if (!deps) return
     if (deps.isShuttingDown()) return
-
-    const routes = deps.getRoutes()
-
-    for (const [channelId, cwd] of Object.entries(routes)) {
-      try {
-        if (deps.isRestartPendingOrActive(channelId)) continue
-        if (deps.hasReachedMaxFailures(channelId)) continue
-
-        const alive = await deps.isSessionAlive(channelId)
-        if (!alive) {
-          deps.scheduleRestart(channelId, cwd)
+    if (tickInFlight) {
+      skippedTicks++
+      // Fire exactly once when the streak crosses 5 (4 → 5 transition). At the
+      // 120 s interval, five consecutive skips represents ~10 minutes of
+      // tick-budget exhaustion — long enough to indicate genuine health-check
+      // wedging (e.g., a persistently hung statRoute or isSessionAliveAdapter)
+      // rather than transient slowness. Further skips within the same streak
+      // are silent; the next successfully-started tick body resets the counter
+      // and re-arms the warning for a future streak.
+      if (skippedTicks === 5) {
+        console.error('[slack] health-check: tick body in flight; skipped 5 consecutive ticks — investigate budget exhaustion')
+      }
+      return
+    }
+    tickInFlight = true
+    skippedTicks = 0
+    try {
+      const routes = deps.getRoutes()
+
+      for (const [channelId, cwd] of Object.entries(routes)) {
+        try {
+          if (deps.isRestartPendingOrActive(channelId)) continue
+
+          if (await deps.statRoute(cwd)) {
+            clearOutageFlag(channelId, 'cwd-unreachable')
+          } else {
+            setOutageFlag(channelId, 'cwd-unreachable', cwd)
+          }
+
+          const alive = await deps.isSessionAlive(channelId)
+          if (!alive) {
+            deps.scheduleRestart(channelId, cwd)
+          }
+        } catch (err) {
+          console.error(`[slack] health-check: error checking channel=${channelId}:`, err)
         }
-      } catch (err) {
-        console.error(`[slack] health-check: error checking channel=${channelId}:`, err)
       }
+    } finally {
+      tickInFlight = false
     }
   }, intervalSeconds * 1000)
 }
@@ -86,4 +114,6 @@ export function _resetHealthCheckState(): void {
     intervalId = null
   }
   deps = null
+  tickInFlight = false
+  skippedTicks = 0
 }

package/src/outage-state.ts

@@ -0,0 +1,287 @@
+/**
+ * outage-state.ts — Channel-scoped outage-flag state machine + Slack emit surface.
+ *
+ * Tracks three orthogonal outage classes per routed Slack channel and emits
+ * onset / all-clear messages via the injected `postToChannel` hook. The module
+ * is intentionally free of Date / timestamp logic — operators scroll back to
+ * the onset message for timing context.
+ *
+ * Public API surface (all exported):
+ *   - initOutageState(deps)                — install production dependencies
+ *   - getOutageFlags(channelId)            — read live flag set
+ *   - setOutageFlag(channelId, cls, detail?)  — raise flag + emit onset message
+ *   - clearOutageFlag(channelId, cls)      — lower flag; emits all-clear when set empties
+ *   - resetAllToHealthy(channelIds)        — silent bulk wipe (boot-time reset)
+ *   - withOutageDetection(ch, cwd, fn)     — AD verb wrapper; raises/clears flags on error/success
+ *   - withSpawnDetection(ch, cwd, fn)      — like withOutageDetection + clears cwd-unreachable on success
+ *   - _resetOutageState()                  — test-only state reset
+ *
+ * Template exports (used by tests):
+ *   - ONSET_TEMPLATES
+ *   - ALL_CLEAR_TEMPLATE
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+import type { Client } from 'agent-director'
+import {
+  ErrSystemInstallDisappeared,
+  ErrTmuxNotAvailable,
+  ErrCwdNotFound,
+  ErrCwdNotADirectory,
+} from './agent-director-errors.ts'
+
+// ---------------------------------------------------------------------------
+// Data types
+// ---------------------------------------------------------------------------
+
+/** Union of all outage classifications. No 'healthy' member — absence == healthy. */
+export type OutageClass = 'ad-unreachable' | 'cwd-unreachable' | 'tmux-unavailable'
+
+/**
+ * Detail record for a single outage class in a bad stretch.
+ * No `enteredAtIso` — timestamps are intentionally absent from the all-clear template.
+ */
+export interface ClassRecord {
+  detail?: string
+}
+
+/** Per-channel state entry. */
+interface ChannelEntry {
+  /** Currently active outage flags. */
+  flags: Set<OutageClass>
+  /** Class → detail record for the current bad stretch. Reset to empty Map on all-clear. */
+  badStretchClasses: Map<OutageClass, ClassRecord>
+}
+
+/** Dependencies injected via `initOutageState` — wires the module to production Slack + AD. */
+export interface OutageStateDeps {
+  /** Fire-and-forget Slack post; errors MUST be handled internally by the caller. */
+  postToChannel(channelId: string, text: string): void
+  /** Return the singleton AD Client. Same semantics as getClient() in agent-director-client.ts. */
+  getClient(): Client
+}
+
+// ---------------------------------------------------------------------------
+// Module-scoped state
+// ---------------------------------------------------------------------------
+
+let deps: OutageStateDeps | undefined
+const entries = new Map<string, ChannelEntry>()
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Lazy-create a ChannelEntry for `channelId` on first access. */
+function entryFor(channelId: string): ChannelEntry {
+  let entry = entries.get(channelId)
+  if (!entry) {
+    entry = { flags: new Set(), badStretchClasses: new Map() }
+    entries.set(channelId, entry)
+  }
+  return entry
+}
+
+// ---------------------------------------------------------------------------
+// Slack message templates
+// ---------------------------------------------------------------------------
+
+/** Stable iteration order for the all-clear template. */
+const STABLE_CLASS_ORDER: OutageClass[] = [
+  'ad-unreachable',
+  'cwd-unreachable',
+  'tmux-unavailable',
+]
+
+/**
+ * ONSET_TEMPLATES — one template function per outage class.
+ * The optional `detail` parameter carries class-specific context
+ * (binary path for ad-unreachable; cwd path for cwd-unreachable).
+ */
+export const ONSET_TEMPLATES: Record<OutageClass, (detail?: string) => string> = {
+  'ad-unreachable': (binaryPath?: string) =>
+    `:rotating_light: *agent-director unreachable* — affects every routed channel.\nBinary: \`${binaryPath ?? '<unknown>'}\`\nRemediation: reinstall agent-director.`,
+
+  'tmux-unavailable': (_detail?: string) =>
+    `:rotating_light: *tmux unavailable* — affects every routed channel.\nRemediation: install or repair tmux.`,
+
+  'cwd-unreachable': (cwd?: string) =>
+    `:rotating_light: *Route cwd unreachable* — \`${cwd ?? '<unknown>'}\`\nRemediation: restore the directory or remove this route from \`config.json\`.`,
+}
+
+/**
+ * ALL_CLEAR_TEMPLATE — renders a Slack all-clear message from the bad-stretch
+ * history snapshot. Entries are emitted in the stable class order regardless
+ * of the order flags were raised. No timestamps.
+ */
+export function ALL_CLEAR_TEMPLATE(resolved: Map<OutageClass, ClassRecord>): string {
+  const parts: string[] = []
+  for (const cls of STABLE_CLASS_ORDER) {
+    const rec = resolved.get(cls)
+    if (rec === undefined) continue
+    const detailSuffix = rec.detail !== undefined ? ` (\`${rec.detail}\`)` : ''
+    parts.push(`\`${cls}\`${detailSuffix}`)
+  }
+  return `:white_check_mark: *All clear.* Resolved: ${parts.join(', ')}.`
+}
+
+// ---------------------------------------------------------------------------
+// Public API — init + accessors
+// ---------------------------------------------------------------------------
+
+/**
+ * initOutageState — installs production dependencies. Called once from
+ * `src/server.ts:main()` after `routingConfig` is loaded, before the
+ * Socket Mode connect block.
+ */
+export function initOutageState(d: OutageStateDeps): void {
+  deps = d
+}
+
+/**
+ * getOutageFlags — returns the live read-only flag set for `channelId`.
+ * Returns an empty `ReadonlySet` sentinel when no entry exists yet.
+ */
+export function getOutageFlags(channelId: string): ReadonlySet<OutageClass> {
+  return entries.get(channelId)?.flags ?? (new Set<OutageClass>() as ReadonlySet<OutageClass>)
+}
+
+// ---------------------------------------------------------------------------
+// Public API — mutators
+// ---------------------------------------------------------------------------
+
+/**
+ * setOutageFlag — raises `cls` for `channelId` and emits an onset Slack
+ * message. Same-flag re-raise is a silent no-op (dedupe).
+ *
+ * State mutates BEFORE the emit so a synchronous throw in `postToChannel`
+ * cannot cause double-emission on the next observation.
+ */
+export function setOutageFlag(channelId: string, cls: OutageClass, detail?: string): void {
+  if (!deps) return
+  const entry = entryFor(channelId)
+  if (entry.flags.has(cls)) return // same-flag dedupe
+  // Mutate state BEFORE emit (SR-V-2.x state-before-emit contract).
+  entry.flags.add(cls)
+  entry.badStretchClasses.set(cls, { detail })
+  deps.postToChannel(channelId, ONSET_TEMPLATES[cls](detail))
+}
+
+/**
+ * clearOutageFlag — lowers `cls` for `channelId`. Emits the all-clear Slack
+ * message ONLY when the clear leaves the flag set empty and there is a
+ * non-empty bad-stretch history (i.e., at least one onset was recorded).
+ * Intermediate clears (flag set still non-empty after removal) are silent.
+ *
+ * State mutates BEFORE the emit (same contract as setOutageFlag).
+ */
+export function clearOutageFlag(channelId: string, cls: OutageClass): void {
+  if (!deps) return
+  const entry = entries.get(channelId)
+  if (!entry) return
+  if (!entry.flags.has(cls)) return // same-state dedupe
+  // Mutate state BEFORE emit.
+  entry.flags.delete(cls)
+  if (entry.flags.size === 0 && entry.badStretchClasses.size > 0) {
+    // Snapshot history and reset BEFORE the postToChannel call.
+    const snapshot = new Map(entry.badStretchClasses)
+    entry.badStretchClasses = new Map()
+    deps.postToChannel(channelId, ALL_CLEAR_TEMPLATE(snapshot))
+  }
+}
+
+/**
+ * resetAllToHealthy — silently wipes every channel's flag set and bad-stretch
+ * history to a clean slate. No `postToChannel` calls. Used at boot time
+ * after `socket.start()` as a defensive boundary for pre-auth observations
+ * (boot-time call is added by Epic 2).
+ */
+export function resetAllToHealthy(channelIds: string[]): void {
+  for (const channelId of channelIds) {
+    entries.set(channelId, { flags: new Set(), badStretchClasses: new Map() })
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API — AD verb wrappers
+// ---------------------------------------------------------------------------
+
+/**
+ * withOutageDetection — centralized wrapper for AD verb calls that should
+ * participate in outage detection.
+ *
+ * On error:
+ *   - ErrSystemInstallDisappeared → raises 'ad-unreachable' (detail = binaryPath)
+ *   - ErrTmuxNotAvailable         → raises 'tmux-unavailable'
+ *   - ErrCwdNotFound / ErrCwdNotADirectory → raises 'cwd-unreachable' (detail = routeCwd)
+ *     UNLESS routeCwd is undefined, in which case logs loudly and rethrows
+ *     WITHOUT raising the flag (defensive carve-out for verb-class drift).
+ *   - Other errors → no flag change; rethrow unchanged.
+ *
+ * On success: clears 'ad-unreachable' and 'tmux-unavailable', returns result.
+ *
+ * The original error is always rethrown so callers can handle it normally.
+ */
+export async function withOutageDetection<T>(
+  channelId: string,
+  routeCwd: string | undefined,
+  fn: (client: Client) => Promise<T>,
+): Promise<T> {
+  if (!deps) {
+    throw new Error(
+      'outage-state: withOutageDetection called before initOutageState — caller-site bug',
+    )
+  }
+  try {
+    const result = await fn(deps.getClient())
+    clearOutageFlag(channelId, 'ad-unreachable')
+    clearOutageFlag(channelId, 'tmux-unavailable')
+    return result
+  } catch (err) {
+    if (err instanceof ErrSystemInstallDisappeared) {
+      setOutageFlag(channelId, 'ad-unreachable', err.binaryPath)
+    } else if (err instanceof ErrTmuxNotAvailable) {
+      setOutageFlag(channelId, 'tmux-unavailable')
+    } else if (err instanceof ErrCwdNotFound || err instanceof ErrCwdNotADirectory) {
+      if (routeCwd !== undefined) {
+        setOutageFlag(channelId, 'cwd-unreachable', routeCwd)
+      } else {
+        console.error(
+          `[slack] outage-state: withOutageDetection: cwd error on channel=${channelId} but routeCwd is undefined — verb-class drift; rethrowing without raising flag`,
+          err,
+        )
+      }
+    }
+    throw err
+  }
+}
+
+/**
+ * withSpawnDetection — like `withOutageDetection` but also clears
+ * 'cwd-unreachable' on success. Spawn and resume verbs are the only calls
+ * that actually exercise the route's cwd, so cwd health is only confirmed
+ * by a successful spawn/resume.
+ */
+export async function withSpawnDetection<T>(
+  channelId: string,
+  routeCwd: string | undefined,
+  fn: (client: Client) => Promise<T>,
+): Promise<T> {
+  const result = await withOutageDetection(channelId, routeCwd, fn)
+  clearOutageFlag(channelId, 'cwd-unreachable')
+  return result
+}
+
+// ---------------------------------------------------------------------------
+// Test-only
+// ---------------------------------------------------------------------------
+
+/**
+ * _resetOutageState — clears all module-scoped state. For tests only.
+ * Production code must not call this.
+ */
+export function _resetOutageState(): void {
+  deps = undefined
+  entries.clear()
+}

package/src/permission-click-handler.ts

@@ -16,11 +16,11 @@
  * SPDX-License-Identifier: MIT
  */
 
-import { AgentDirectorError, ErrAlreadyDecided } from 'agent-director'
-import type { Client } from 'agent-director'
+import { AgentDirectorError, ErrAlreadyDecided, ErrSystemInstallDisappeared, ErrTmuxNotAvailable } from 'agent-director'
 import type { WebClient } from '@slack/web-api'
 
 import { decideWithToken } from './agent-director-client.ts'
+import { withOutageDetection } from './outage-state.ts'
 import { parsePermissionActionId, type PermissionDecision } from './permission-action-id.ts'
 import { getLivePermission, markHandled } from './permission-poller.ts'
 import { emitTrail as defaultEmitTrail } from './permission-trail.ts'
@@ -32,8 +32,6 @@ import type {
 } from './permission-trail.ts'
 
 export interface ClickDeps {
-  /** Returns an AD Client whose `decide` method this handler will invoke. */
-  getClient: () => Pick<Client, 'decide'>
   web: Pick<WebClient, 'chat'>
   log?: (...args: unknown[]) => void
   /**
@@ -141,6 +139,21 @@ export async function handlePermissionClick(
     live_pending: earlyEntry !== undefined,
   })
 
+  const ctxChannel = context.channel ?? earlyEntry?.channelId
+  if (!ctxChannel) {
+    logDeps(
+      deps,
+      `[slack] permission-click-handler: cannot resolve channelId ` +
+        `(context.channel=${context.channel ?? 'undefined'}, ` +
+        `earlyEntry.channelId=${earlyEntry?.channelId ?? 'undefined'}, ` +
+        `claude_instance_id=${claudeInstanceId}, ` +
+        `action_id=${actionId}, ` +
+        `request_token=${requestToken}) ` +
+        `— logging and bypassing this click. This is a CSCB bug — investigate.`,
+    )
+    return true
+  }
+
   // SR-4.1, SR-4.2, SR-7.2: AD is the source of truth. Always call decide
   // with the decoded request_token — including for stale clicks whose
   // composite key is no longer in the pending map. This is the click's ONLY
@@ -152,14 +165,29 @@ export async function handlePermissionClick(
     decision,
   }
   try {
-    await decideWithToken(deps.getClient(), {
-      claude_instance_id: claudeInstanceId,
-      decision,
-      request_token: requestToken,
-    })
+    await withOutageDetection(ctxChannel, undefined, (client) =>
+      decideWithToken(client, {
+        claude_instance_id: claudeInstanceId,
+        decision,
+        request_token: requestToken,
+      }),
+    )
     // SR-V-2.7 call-side success emission.
     emit({ ...decideEnvelope, result_class: 'ok' satisfies AdDecideResponseClass })
   } catch (err) {
+    if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+      // SR-V-2.7 ad/tmux carve-out: the wrapper already raised the outage
+      // flag (one Slack onset alert via the state machine). Forensic
+      // requirement: the trail JSONL must still carry the typed error name
+      // and message so post-incident debugging can correlate the click
+      // attempt with the AD-down cause — the loud Slack alert tells the
+      // operator something is broken; the trail entry tells the engineer
+      // what was actually thrown.
+      const result_class: AdDecideResponseClass = err.errName as AdDecideResponseClass
+      const raw_error_message = err.message
+      emit({ ...decideEnvelope, result_class, raw_error_message })
+      return true
+    }
     // SR-V-2.7 call-side failure emission. Classify against the same AD
     // error names the existing branches discriminate on so the trail's
     // result_class stays consistent with src/agent-director-errors.ts. The

package/src/permission-poller.ts

@@ -35,6 +35,8 @@
 import {
   AgentDirectorError,
   ErrSpawnNotFound,
+  ErrSystemInstallDisappeared,
+  ErrTmuxNotAvailable,
 } from 'agent-director'
 import type {
   GetResult as ADGetResult,
@@ -50,6 +52,7 @@ import type {
   GetPermissionParams,
   GetPermissionResult,
 } from './agent-director-client.ts'
+import { withOutageDetection } from './outage-state.ts'
 import { encodePermissionActionId } from './permission-action-id.ts'
 import { emitTrail as defaultEmitTrail } from './permission-trail.ts'
 import type {
@@ -303,7 +306,12 @@ function classifySlackError(err: unknown): string {
 async function runTick(deps: PollerDeps): Promise<void> {
   if (tickInFlight) {
     skippedTicks++
-    if (skippedTicks >= 5) {
+    // Fire exactly once when the streak crosses 5 (4 → 5 transition). Further
+    // skips within the same streak are silent; the next successfully-started
+    // tick body resets skippedTicks to 0 and re-arms the warning for a future
+    // streak. Mirrors src/health-check.ts:55-62 — see that block for the
+    // budget-exhaustion rationale.
+    if (skippedTicks === 5) {
       logViaDeps(deps, `[slack] permission-poller: skipped ${skippedTicks} consecutive ticks — tick budget exceeded`)
     }
     return
@@ -324,13 +332,23 @@ async function runTick(deps: PollerDeps): Promise<void> {
     const seenComposite = new Set<string>()
     const nonConformingInstanceIds = new Set<string>()
     for (const row of rows) {
+      const rowChannelId = row.labels['channel']
+      if (!rowChannelId) {
+        logViaDeps(deps, `[slack] permission-poller: spawn ${row.claude_instance_id} has no channel label — skipping`)
+        continue
+      }
+
       let got: GetResultWithPermissionRequests
       try {
-        got = (await client.get({
-          claude_instance_id: row.claude_instance_id,
-        })) as unknown as GetResultWithPermissionRequests
+        got = (await withOutageDetection(rowChannelId, undefined, () =>
+          client.get({ claude_instance_id: row.claude_instance_id })
+        )) as unknown as GetResultWithPermissionRequests
       } catch (err) {
         if (err instanceof ErrSpawnNotFound) continue
+        if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+          // Outage flag raised; skip per-event log.
+          continue
+        }
         const e = err instanceof AgentDirectorError ? err : null
         logViaDeps(deps, `[slack] permission-poller: get failed for ${row.claude_instance_id}: ${e?.errName ?? String(err)}`)
         continue
@@ -368,8 +386,14 @@ async function runTick(deps: PollerDeps): Promise<void> {
     for (const entry of closedEntries) {
       let info: GetPermissionResult
       try {
-        info = await getPermission(client, { request_token: entry.requestToken })
+        info = await withOutageDetection(entry.channelId, undefined, () =>
+          getPermission(client, { request_token: entry.requestToken })
+        )
       } catch (err) {
+        if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+          // Outage flag raised; skip per-event log.
+          continue
+        }
         if (isErrPermissionRequestNotFound(err)) {
           logViaDeps(deps, `[slack] permission-poller: get-permission not-found for ${entry.claudeInstanceId} token=${entry.requestToken} — generic deny + drop`)
           emitRowDecision(deps, 'not_found_generic_deny', entry.claudeInstanceId, entry.requestToken)

package/src/restart.ts

@@ -22,17 +22,10 @@ export interface RestartDeps {
   isShuttingDown(): boolean
 }
 
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-
-export const MAX_CONSECUTIVE_FAILURES = 3
-
 // ---------------------------------------------------------------------------
 // Module-scoped state
 // ---------------------------------------------------------------------------
 
-const failureCounters = new Map<string, number>()
 const pendingRestartTimers = new Map<string, ReturnType<typeof setTimeout>>()
 const activeLaunches = new Set<string>()
 let deps: RestartDeps | null = null
@@ -61,14 +54,6 @@ export function scheduleRestart(channelId: string, cwd: string, sessionId?: stri
     return
   }
 
-  const failures = failureCounters.get(channelId) ?? 0
-  if (failures >= MAX_CONSECUTIVE_FAILURES) {
-    console.error(
-      `[slack] Max consecutive failures (${MAX_CONSECUTIVE_FAILURES}) reached — giving up on channel=${channelId} cwd="${cwd}"`,
-    )
-    return
-  }
-
   // Cancel any existing timer for this channel
   const existing = pendingRestartTimers.get(channelId)
   if (existing !== undefined) {
@@ -130,11 +115,7 @@ export function scheduleRestart(channelId: string, cwd: string, sessionId?: stri
       }
 
       if (!ok) {
-        const count = (failureCounters.get(channelId) ?? 0) + 1
-        failureCounters.set(channelId, count)
-        console.error(
-          `[slack] Session relaunch failed for channel=${channelId} (failure ${count}/${MAX_CONSECUTIVE_FAILURES})`,
-        )
+        console.error(`[slack] Session relaunch failed for channel=${channelId}`)
       }
     } finally {
       activeLaunches.delete(channelId)
@@ -144,14 +125,6 @@ export function scheduleRestart(channelId: string, cwd: string, sessionId?: stri
   pendingRestartTimers.set(channelId, timer)
 }
 
-// ---------------------------------------------------------------------------
-// resetFailureCounter
-// ---------------------------------------------------------------------------
-
-export function resetFailureCounter(channelId: string): void {
-  failureCounters.set(channelId, 0)
-}
-
 // ---------------------------------------------------------------------------
 // cancelAllRestartTimers
 // ---------------------------------------------------------------------------
@@ -165,17 +138,13 @@ export function cancelAllRestartTimers(): void {
 }
 
 // ---------------------------------------------------------------------------
-// isRestartPendingOrActive / hasReachedMaxFailures — query functions
+// isRestartPendingOrActive — query function
 // ---------------------------------------------------------------------------
 
 export function isRestartPendingOrActive(channelId: string): boolean {
   return pendingRestartTimers.has(channelId) || activeLaunches.has(channelId)
 }
 
-export function hasReachedMaxFailures(channelId: string): boolean {
-  return (failureCounters.get(channelId) ?? 0) >= MAX_CONSECUTIVE_FAILURES
-}
-
 // ---------------------------------------------------------------------------
 // _resetRestartState — exported for test cleanup
 // ---------------------------------------------------------------------------
@@ -184,7 +153,6 @@ export function _resetRestartState(): void {
   for (const timer of pendingRestartTimers.values()) {
     clearTimeout(timer)
   }
-  failureCounters.clear()
   pendingRestartTimers.clear()
   activeLaunches.clear()
   deps = null

package/src/server.ts

@@ -27,6 +27,7 @@ import {
   chmodSync,
   existsSync,
   renameSync,
+  promises as fsPromises,
 } from 'fs'
 
 import {
@@ -54,6 +55,7 @@ import {
 } from './session-manager.ts'
 import { cleanSession, getCozempicAvailable } from './cozempic.ts'
 import { ErrSpawnNotFound } from 'agent-director'
+import { ErrSystemInstallDisappeared, ErrTmuxNotAvailable } from './agent-director-errors.ts'
 import { getClient, closeClient } from './agent-director-client.ts'
 import {
   emitBlockActionReceived,
@@ -64,10 +66,8 @@ import { startPermissionPoller, stopPermissionPoller } from './permission-poller
 import {
   initRestart,
   scheduleRestart,
-  resetFailureCounter,
   cancelAllRestartTimers,
   isRestartPendingOrActive,
-  hasReachedMaxFailures,
 } from './restart.ts'
 import { initHealthCheck, startHealthCheck, stopHealthCheck } from './health-check.ts'
 import { loadTokens, isDryRun } from './tokens.ts'
@@ -99,6 +99,7 @@ import {
 } from './registry.ts'
 import { runAgentDirectorStartupGate } from './agent-director-startup.ts'
 import { installSlackChannelBotTemplate } from './agent-director-template.ts'
+import { initOutageState, setOutageFlag, clearOutageFlag, resetAllToHealthy, withOutageDetection } from './outage-state.ts'
 
 // Re-export constants so they stay in one place (lib.ts)
 export { MAX_PENDING, MAX_PAIRING_REPLIES, PAIRING_EXPIRY_MS } from './lib.ts'
@@ -521,9 +522,6 @@ async function handleInitialized(
     console.error(`[slack] Session replaced existing connection for CWD "${normalizedCwd}"`)
   }
   console.error(`[slack] Session connected: channel=${matchedChannelId} cwd="${normalizedCwd}"`)
-
-  // Reset failure counter — session reconnected successfully
-  resetFailureCounter(matchedChannelId)
 }
 
 // ---------------------------------------------------------------------------
@@ -756,7 +754,7 @@ socket.on('interactive', async (evt) => {
 
     const handled = await handlePermissionClick(
       actionId,
-      { getClient, web },
+      { web },
       { channel: channelId, messageTs, user: userId },
     )
     if (handled) {
@@ -838,6 +836,97 @@ async function shutdown(signal: string): Promise<void> {
 process.on('SIGTERM', () => { shutdown('SIGTERM').catch(() => process.exit(1)) })
 process.on('SIGINT',  () => { shutdown('SIGINT').catch(() => process.exit(1)) })
 
+// ---------------------------------------------------------------------------
+// _buildIsSessionAliveAdapter
+// ---------------------------------------------------------------------------
+
+/**
+ * _buildIsSessionAliveAdapter — test-only factory for the tick-path liveness
+ * probe. Production code wires this via main() as `isSessionAliveAdapter`;
+ * tests call it directly to exercise the four SRD § Liveness probe branches
+ * without importing the private closure inside main().
+ *
+ * @internal
+ */
+export function _buildIsSessionAliveAdapter(
+  getRoutingConfig: () => RoutingConfig | null | undefined,
+): (channelId: string) => Promise<boolean> {
+  return async (channelId: string) => {
+    const routingConfig = getRoutingConfig()
+    if (!routingConfig?.routes[channelId]) return false
+    const claude_instance_id = instanceIdFor(channelId, routingConfig.routes[channelId]?.normalizedName)
+    try {
+      const r = await getClient().status({ claude_instance_id })
+      clearOutageFlag(channelId, 'ad-unreachable')
+      clearOutageFlag(channelId, 'tmux-unavailable')
+      return AGENT_DIRECTOR_LIVE_STATES.has(r.state)
+    } catch (err) {
+      if (err instanceof ErrSpawnNotFound) {
+        clearOutageFlag(channelId, 'ad-unreachable')
+        clearOutageFlag(channelId, 'tmux-unavailable')
+        return false
+      }
+      if (err instanceof ErrSystemInstallDisappeared) {
+        setOutageFlag(channelId, 'ad-unreachable', err.binaryPath)
+        return false
+      }
+      if (err instanceof ErrTmuxNotAvailable) {
+        setOutageFlag(channelId, 'tmux-unavailable')
+        return false
+      }
+      console.error(`[slack] isSessionAlive: status error for channel=${channelId}:`, err)
+      return false
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// _buildStatRouteImpl
+// ---------------------------------------------------------------------------
+
+/**
+ * _buildStatRouteImpl — test-only factory for the health-check tick's
+ * statRoute dependency. Production code wires this via main()'s
+ * initHealthCheck call with no deps; tests inject `stat` / `setTimeout` /
+ * `clearTimeout` to exercise the 5-second timeout budget and the
+ * fsPromises.stat error swallowing against the REAL factory (not a replica).
+ *
+ * @internal
+ */
+export function _buildStatRouteImpl(deps?: {
+  stat?: (path: string) => Promise<{ isDirectory(): boolean }>
+  setTimeout?: (fn: () => void, ms: number) => ReturnType<typeof setTimeout>
+  clearTimeout?: (handle: ReturnType<typeof setTimeout> | undefined) => void
+}): (cwd: string) => Promise<boolean> {
+  const stat = deps?.stat ?? ((p: string) => fsPromises.stat(p) as Promise<{ isDirectory(): boolean }>)
+  const setT = deps?.setTimeout ?? ((fn: () => void, ms: number) => setTimeout(fn, ms))
+  const clearT = deps?.clearTimeout ?? ((h: ReturnType<typeof setTimeout> | undefined) => clearTimeout(h))
+  const STAT_TIMEOUT_MS = 5_000
+  return async (cwd: string) => {
+    const statPromise = (async () => {
+      try {
+        const st = await stat(cwd)
+        return st.isDirectory()
+      } catch (err) {
+        console.error(`[slack] health-check: statRoute(${cwd}) failed:`, err)
+        return false
+      }
+    })()
+    let timeoutHandle: ReturnType<typeof setTimeout> | undefined
+    const timeoutPromise = new Promise<boolean>((resolve) => {
+      timeoutHandle = setT(() => {
+        console.error(`[slack] health-check: statRoute(${cwd}) timed out after ${STAT_TIMEOUT_MS}ms — treating as unreachable`)
+        resolve(false)
+      }, STAT_TIMEOUT_MS)
+    })
+    try {
+      return await Promise.race([statPromise, timeoutPromise])
+    } finally {
+      clearT(timeoutHandle)
+    }
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Main
 //
@@ -910,6 +999,15 @@ export async function main(): Promise<void> {
     }
   }
 
+  initOutageState({
+    postToChannel: (channelId, text) => {
+      web.chat.postMessage({ channel: channelId, text }).catch((err) => {
+        console.error(`[slack] outage-state: postMessage failed for channel=${channelId}:`, err)
+      })
+    },
+    getClient,
+  })
+
   if (isDryRun()) {
     console.error('[slack] Running in dry-run mode — Slack disabled')
     botUserId = 'U000DRY'
@@ -926,6 +1024,10 @@ export async function main(): Promise<void> {
     await socket.start()
     console.error('[slack] Socket Mode connected')
 
+    if (routingConfig) {
+      resetAllToHealthy(Object.keys(routingConfig.routes))
+    }
+
     // SR-2.1 permission poller — single-threaded interval loop monitors AD
     // state for spawns in check_permission and posts Block Kit prompts.
     if (routingConfig) {
@@ -1127,18 +1229,7 @@ export async function main(): Promise<void> {
   // SR-11 Event 6a. Any AGENT_DIRECTOR_LIVE_STATES value → alive; terminal
   // states (ended, missing) and ErrSpawnNotFound → dead. Other errors fall
   // back to "dead" defensively — health-check will retry.
-  const isSessionAliveAdapter = async (channelId: string): Promise<boolean> => {
-    if (!routingConfig?.routes[channelId]) return false
-    const claude_instance_id = instanceIdFor(channelId, routingConfig.routes[channelId]?.normalizedName)
-    try {
-      const r = await getClient().status({ claude_instance_id })
-      return AGENT_DIRECTOR_LIVE_STATES.has(r.state)
-    } catch (err) {
-      if (err instanceof ErrSpawnNotFound) return false
-      console.error(`[slack] isSessionAlive: status error for channel=${channelId}:`, err)
-      return false
-    }
-  }
+  const isSessionAliveAdapter = _buildIsSessionAliveAdapter(() => routingConfig)
 
   // Initialize restart module with library-backed adapters
   initRestart({
@@ -1153,9 +1244,12 @@ export async function main(): Promise<void> {
     killSession: async (channelId) => {
       try {
         const normalizedName = routingConfig?.routes[channelId]?.normalizedName
-        await getClient().kill({ claude_instance_id: instanceIdFor(channelId, normalizedName) })
+        await withOutageDetection(channelId, undefined, (client) =>
+          client.kill({ claude_instance_id: instanceIdFor(channelId, normalizedName) })
+        )
       } catch (err) {
         if (err instanceof ErrSpawnNotFound) return
+        if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) return
         console.error(`[slack] killSession (restart adapter): error for channel=${channelId}:`, err)
       }
     },
@@ -1226,7 +1320,7 @@ export async function main(): Promise<void> {
   initHealthCheck({
     isSessionAlive: isSessionAliveAdapter,
     isRestartPendingOrActive,
-    hasReachedMaxFailures,
+    statRoute: _buildStatRouteImpl(),
     scheduleRestart,
     isShuttingDown: () => shuttingDown,
     getRoutes: () => {

package/src/session-manager.ts

@@ -38,6 +38,13 @@ import type { WebClient } from '@slack/web-api'
 import { checkCozempicAvailable } from './cozempic.ts'
 import { type RoutingConfig, MCP_SERVER_NAME, normalizeChannelName } from './config.ts'
 import { getClient } from './agent-director-client.ts'
+import { withOutageDetection, withSpawnDetection } from './outage-state.ts'
+import {
+  ErrSystemInstallDisappeared,
+  ErrTmuxNotAvailable,
+  ErrCwdNotFound,
+  ErrCwdNotADirectory,
+} from './agent-director-errors.ts'
 import { recordStartupError } from './startup-errors.ts'
 import { isDryRun } from './tokens.ts'
 
@@ -184,12 +191,13 @@ export async function reconnectMcp(
   const claude_instance_id = instanceIdFor(channelId, routingConfig?.routes[channelId]?.normalizedName)
   console.error(`[slack] reconnecting MCP server "${MCP_SERVER_NAME}": channel=${channelId}`)
   try {
-    await getClient().sendKeys({
+    await withOutageDetection(channelId, undefined, (client) => client.sendKeys({
       claude_instance_id,
       text: `/mcp reconnect ${MCP_SERVER_NAME}`,
-    })
+    }))
     return true
   } catch (err) {
+    if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) return false
     const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('send-keys', 'UnknownError', String(err))
     console.error(`[slack] reconnectMcp: send-keys failed for channel=${channelId}: ${e.errName}`)
     postSpawnFailureToChannel(channelId, e, web)
@@ -263,20 +271,23 @@ export async function approveDevChannelsDialog(
 ): Promise<void> {
   void web
   const claude_instance_id = instanceIdFor(channelId, normalizedName)
-  const client = getClient()
   const deadline = Date.now() + _dialogPollTimeoutMs
 
   let approved = false
   while (Date.now() < deadline) {
     try {
-      const { pane } = await client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true })
+      const { pane } = await withOutageDetection(channelId, undefined, (client) => client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true }))
       if (pane.includes(DEV_CHANNELS_DIALOG_NEEDLE)) {
-        await client.sendKeys({ claude_instance_id, text: '', allow_pending: true })
+        await withOutageDetection(channelId, undefined, (client) => client.sendKeys({ claude_instance_id, text: '', allow_pending: true }))
         approved = true
         break
       }
     } catch (err) {
-      console.error(`[slack] approveDevChannelsDialog: readPane error channel=${channelId}: ${String(err)}`)
+      if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+        // Outage flag raised; continue polling
+      } else {
+        console.error(`[slack] approveDevChannelsDialog: readPane error channel=${channelId}: ${String(err)}`)
+      }
     }
     await new Promise((r) => setTimeout(r, _dialogPollIntervalMs))
   }
@@ -292,7 +303,7 @@ export async function approveDevChannelsDialog(
   while (Date.now() < deadline && misses < DIALOG_GONE_CONFIRMS_REQUIRED) {
     await new Promise((r) => setTimeout(r, _dialogPollIntervalMs))
     try {
-      const { pane } = await client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true })
+      const { pane } = await withOutageDetection(channelId, undefined, (client) => client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true }))
       misses = pane.includes(DEV_CHANNELS_DIALOG_NEEDLE) ? 0 : misses + 1
     } catch {
       /* tolerate transient readPane failure */
@@ -362,20 +373,23 @@ export async function approveTrustFolderDialog(
 ): Promise<void> {
   void web
   const claude_instance_id = instanceIdFor(channelId, normalizedName)
-  const client = getClient()
   const deadline = Date.now() + _trustDialogPollTimeoutMs
 
   let approved = false
   while (Date.now() < deadline) {
     try {
-      const { pane } = await client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true })
+      const { pane } = await withOutageDetection(channelId, undefined, (client) => client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true }))
       if (pane.includes(TRUST_DIALOG_NEEDLE)) {
-        await client.sendKeys({ claude_instance_id, text: '', allow_pending: true })
+        await withOutageDetection(channelId, undefined, (client) => client.sendKeys({ claude_instance_id, text: '', allow_pending: true }))
         approved = true
         break
       }
     } catch (err) {
-      console.error(`[slack] approveTrustFolderDialog: readPane error channel=${channelId}: ${String(err)}`)
+      if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+        // Outage flag raised; continue polling
+      } else {
+        console.error(`[slack] approveTrustFolderDialog: readPane error channel=${channelId}: ${String(err)}`)
+      }
     }
     await new Promise((r) => setTimeout(r, _trustDialogPollIntervalMs))
   }
@@ -389,7 +403,7 @@ export async function approveTrustFolderDialog(
   while (Date.now() < deadline && misses < DIALOG_GONE_CONFIRMS_REQUIRED) {
     await new Promise((r) => setTimeout(r, _trustDialogPollIntervalMs))
     try {
-      const { pane } = await client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true })
+      const { pane } = await withOutageDetection(channelId, undefined, (client) => client.readPane({ claude_instance_id, n_lines: 40, allow_pending: true }))
       misses = pane.includes(TRUST_DIALOG_NEEDLE) ? 0 : misses + 1
     } catch {
       /* tolerate transient readPane failure */
@@ -439,13 +453,16 @@ export async function waitForWaitingAndReconnect(
   while (Date.now() < deadline) {
     let state: string
     try {
-      const r = await getClient().status({ claude_instance_id })
+      const r = await withOutageDetection(channelId, undefined, (client) => client.status({ claude_instance_id }))
       state = r.state
     } catch (err) {
       if (err instanceof ErrSpawnNotFound) {
         console.error(`[slack] waitForWaitingAndReconnect: spawn not found for channel=${channelId} — aborting poll`)
         return true
       }
+      if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+        return false
+      }
       const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('status', 'UnknownError', String(err))
       console.error(`[slack] waitForWaitingAndReconnect: status error for channel=${channelId}: ${e.errName}`)
       postSpawnFailureToChannel(channelId, e, web)
@@ -507,7 +524,7 @@ function buildSpawnParams(
 /** Best-effort kill — never throws. */
 async function tryKill(channelId: string, normalizedName: string | undefined): Promise<void> {
   try {
-    await getClient().kill({ claude_instance_id: instanceIdFor(channelId, normalizedName) })
+    await withOutageDetection(channelId, undefined, (client) => client.kill({ claude_instance_id: instanceIdFor(channelId, normalizedName) }))
   } catch {
     /* ignore */
   }
@@ -521,9 +538,12 @@ async function tryDelete(
   isStartup: boolean,
 ): Promise<boolean> {
   try {
-    await getClient().delete({ claude_instance_id: [instanceIdFor(channelId, normalizedName)] })
+    await withOutageDetection(channelId, undefined, (client) => client.delete({ claude_instance_id: [instanceIdFor(channelId, normalizedName)] }))
     return true
   } catch (err) {
+    if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+      return false
+    }
     const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('delete', 'UnknownError', String(err))
     console.error(`[slack] tryDelete: failed for channel=${channelId}: ${e.errName}`)
     if (isStartup) recordStartupError('spawn-failed', `delete failed for channel=${channelId}: ${e.errName}`, e)
@@ -560,43 +580,58 @@ export async function spawnForRoute(
 
   const params = buildSpawnParams(channelId, route, routingConfig)
   const normalizedName = routingConfig.routes[channelId]?.normalizedName
-  const client = getClient()
 
   // Attempt fresh spawn ---
   try {
-    const r = await client.spawn(params)
+    const r = await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
     console.error(`[slack] spawnForRoute: spawned channel=${channelId} instanceId=${r.claude_instance_id}`)
     await approveTrustFolderDialog(channelId, web, isStartup, normalizedName)
     await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
     return { channelId, action: 'spawned' }
   } catch (err) {
-    if (!(err instanceof ErrInstanceIdCollision)) {
+    if (err instanceof ErrInstanceIdCollision) {
+      // Collision → fall through to get-then-act
+      console.error(`[slack] spawnForRoute: ErrInstanceIdCollision for channel=${channelId} — fetching current state`)
+    } else if (
+      err instanceof ErrSystemInstallDisappeared ||
+      err instanceof ErrTmuxNotAvailable ||
+      err instanceof ErrCwdNotFound ||
+      err instanceof ErrCwdNotADirectory
+    ) {
+      return { channelId, action: 'failed' }
+    } else {
       const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('spawn', 'UnknownError', String(err))
       console.error(`[slack] spawnForRoute: spawn failed for channel=${channelId}: ${e.errName}`)
       if (isStartup) recordStartupError('spawn-failed', `spawn failed for channel=${channelId}: ${e.errName}`, e)
       postSpawnFailureToChannel(channelId, e, web, isStartup)
       return { channelId, action: 'failed' }
     }
-    // Collision → fall through to get-then-act
-    console.error(`[slack] spawnForRoute: ErrInstanceIdCollision for channel=${channelId} — fetching current state`)
   }
 
   // Collision-handling: get-then-act ---
   let state: string
   try {
-    const r = await client.get({ claude_instance_id: instanceIdFor(channelId, normalizedName) })
+    const r = await withOutageDetection(channelId, undefined, (client) => client.get({ claude_instance_id: instanceIdFor(channelId, normalizedName) }))
     state = r.state
   } catch (err) {
     if (err instanceof ErrSpawnNotFound) {
       // Race: row deleted between spawn-collision and get. Retry spawn once.
       console.error(`[slack] spawnForRoute: ErrSpawnNotFound after collision for channel=${channelId} — retrying spawn (single retry)`)
       try {
-        const r = await client.spawn(params)
+        const r = await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
         console.error(`[slack] spawnForRoute: retry-spawn succeeded for channel=${channelId} instanceId=${r.claude_instance_id}`)
         await approveTrustFolderDialog(channelId, web, isStartup, normalizedName)
         await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
         return { channelId, action: 'spawned' }
       } catch (err2) {
+        if (
+          err2 instanceof ErrSystemInstallDisappeared ||
+          err2 instanceof ErrTmuxNotAvailable ||
+          err2 instanceof ErrCwdNotFound ||
+          err2 instanceof ErrCwdNotADirectory
+        ) {
+          return { channelId, action: 'failed' }
+        }
         const e = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('spawn', 'UnknownError', String(err2))
         console.error(`[slack] spawnForRoute: retry-spawn also failed for channel=${channelId}: ${e.errName}`)
         if (isStartup) recordStartupError('spawn-failed', `retry-spawn failed for channel=${channelId}: ${e.errName}`, e)
@@ -604,6 +639,9 @@ export async function spawnForRoute(
         return { channelId, action: 'failed' }
       }
     }
+    if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+      return { channelId, action: 'failed' }
+    }
     const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('get', 'UnknownError', String(err))
     console.error(`[slack] spawnForRoute: get failed for channel=${channelId}: ${e.errName}`)
     postSpawnFailureToChannel(channelId, e, web, isStartup)
@@ -618,12 +656,20 @@ export async function spawnForRoute(
       await tryKill(channelId, normalizedName)
       if (!(await tryDelete(channelId, normalizedName, web, isStartup))) return { channelId, action: 'failed' }
       try {
-        await client.spawn(params)
+        await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
         console.error(`[slack] spawnForRoute: fresh-spawned (after kill+delete) for channel=${channelId}`)
         await approveTrustFolderDialog(channelId, web, isStartup, normalizedName)
         await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
         return { channelId, action: 'spawned' }
       } catch (err) {
+        if (
+          err instanceof ErrSystemInstallDisappeared ||
+          err instanceof ErrTmuxNotAvailable ||
+          err instanceof ErrCwdNotFound ||
+          err instanceof ErrCwdNotADirectory
+        ) {
+          return { channelId, action: 'failed' }
+        }
         const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('spawn', 'UnknownError', String(err))
         console.error(`[slack] spawnForRoute: fresh spawn after delete failed for channel=${channelId}: ${e.errName}`)
         if (isStartup) recordStartupError('spawn-failed', `fresh spawn after delete failed for channel=${channelId}: ${e.errName}`, e)
@@ -635,7 +681,7 @@ export async function spawnForRoute(
     // resume_enabled: attempt resume
     console.error(`[slack] spawnForRoute: attempting resume for channel=${channelId}`)
     try {
-      await client.resume({ claude_instance_id: instanceIdFor(channelId, normalizedName) })
+      await withSpawnDetection(channelId, route.cwd, (client) => client.resume({ claude_instance_id: instanceIdFor(channelId, normalizedName) }))
       console.error(`[slack] spawnForRoute: resumed channel=${channelId}`)
       return { channelId, action: 'resumed' }
     } catch (err) {
@@ -643,12 +689,20 @@ export async function spawnForRoute(
         console.error(`[slack] spawnForRoute: ${err.errName} on resume for channel=${channelId} — delete+fresh`)
         if (!(await tryDelete(channelId, normalizedName, web, isStartup))) return { channelId, action: 'failed' }
         try {
-          await client.spawn(params)
+          await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
           console.error(`[slack] spawnForRoute: fresh-spawned (after delete) for channel=${channelId}`)
           await approveTrustFolderDialog(channelId, web, isStartup, normalizedName)
           await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
           return { channelId, action: 'spawned' }
         } catch (err2) {
+          if (
+            err2 instanceof ErrSystemInstallDisappeared ||
+            err2 instanceof ErrTmuxNotAvailable ||
+            err2 instanceof ErrCwdNotFound ||
+            err2 instanceof ErrCwdNotADirectory
+          ) {
+            return { channelId, action: 'failed' }
+          }
           const e = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('spawn', 'UnknownError', String(err2))
           console.error(`[slack] spawnForRoute: fresh spawn after delete failed for channel=${channelId}: ${e.errName}`)
           if (isStartup) recordStartupError('spawn-failed', `fresh spawn after delete failed for channel=${channelId}: ${e.errName}`, e)
@@ -662,17 +716,33 @@ export async function spawnForRoute(
         await tryKill(channelId, normalizedName)
         if (!(await tryDelete(channelId, normalizedName, web, isStartup))) return { channelId, action: 'failed' }
         try {
-          await client.spawn(params)
+          await withSpawnDetection(channelId, route.cwd, (client) => client.spawn(params))
           await approveTrustFolderDialog(channelId, web, isStartup, normalizedName)
           await approveDevChannelsDialog(channelId, web, isStartup, normalizedName)
           return { channelId, action: 'spawned' }
         } catch (err2) {
+          if (
+            err2 instanceof ErrSystemInstallDisappeared ||
+            err2 instanceof ErrTmuxNotAvailable ||
+            err2 instanceof ErrCwdNotFound ||
+            err2 instanceof ErrCwdNotADirectory
+          ) {
+            return { channelId, action: 'failed' }
+          }
           const e = err2 instanceof AgentDirectorError ? err2 : new AgentDirectorError('spawn', 'UnknownError', String(err2))
           if (isStartup) recordStartupError('spawn-failed', `fresh spawn failed for channel=${channelId}: ${e.errName}`, e)
           postSpawnFailureToChannel(channelId, e, web, isStartup)
           return { channelId, action: 'failed' }
         }
       }
+      if (
+        err instanceof ErrSystemInstallDisappeared ||
+        err instanceof ErrTmuxNotAvailable ||
+        err instanceof ErrCwdNotFound ||
+        err instanceof ErrCwdNotADirectory
+      ) {
+        return { channelId, action: 'failed' }
+      }
       const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('resume', 'UnknownError', String(err))
       console.error(`[slack] spawnForRoute: resume failed for channel=${channelId}: ${e.errName}`)
       postSpawnFailureToChannel(channelId, e, web, isStartup)
@@ -985,9 +1055,13 @@ export async function reconcileInstanceIds(
       `[slack] reconcileInstanceIds: deleting stale row channel=${o.channelId} instanceId=${o.oldInstanceId}`,
     )
     try {
-      await client.delete({ claude_instance_id: [o.oldInstanceId] })
+      await withOutageDetection(o.channelId, undefined, (client) => client.delete({ claude_instance_id: [o.oldInstanceId] }))
       deleted++
     } catch (err) {
+      if (err instanceof ErrSystemInstallDisappeared || err instanceof ErrTmuxNotAvailable) {
+        failed++
+        continue
+      }
       const e = err instanceof AgentDirectorError ? err : new AgentDirectorError('delete', 'UnknownError', String(err))
       console.error(
         `[slack] reconcileInstanceIds: delete failed for channel=${o.channelId} instanceId=${o.oldInstanceId}: ${e.errName}`,