@proletariat/cli 0.3.95 → 0.3.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. package/dist/commands/config/index.js +1 -1
  2. package/dist/commands/config/index.js.map +1 -1
  3. package/dist/commands/db/repair.js +160 -35
  4. package/dist/commands/db/repair.js.map +1 -1
  5. package/dist/commands/gc.d.ts +14 -0
  6. package/dist/commands/gc.js +208 -0
  7. package/dist/commands/gc.js.map +1 -0
  8. package/dist/commands/linear/connect.d.ts +5 -0
  9. package/dist/commands/linear/connect.js +84 -0
  10. package/dist/commands/linear/connect.js.map +1 -1
  11. package/dist/commands/pr/merge.js +1 -1
  12. package/dist/commands/pr/merge.js.map +1 -1
  13. package/dist/commands/session/attach.d.ts +3 -3
  14. package/dist/commands/session/attach.js +37 -79
  15. package/dist/commands/session/attach.js.map +1 -1
  16. package/dist/commands/session/exec.js +1 -1
  17. package/dist/commands/session/exec.js.map +1 -1
  18. package/dist/commands/session/health.js +1 -1
  19. package/dist/commands/session/health.js.map +1 -1
  20. package/dist/commands/session/inspect.js +1 -1
  21. package/dist/commands/session/inspect.js.map +1 -1
  22. package/dist/commands/session/list.d.ts +1 -0
  23. package/dist/commands/session/list.js +111 -78
  24. package/dist/commands/session/list.js.map +1 -1
  25. package/dist/commands/session/peek.js +1 -1
  26. package/dist/commands/session/peek.js.map +1 -1
  27. package/dist/commands/session/poke.js +1 -1
  28. package/dist/commands/session/poke.js.map +1 -1
  29. package/dist/commands/session/prune.js +50 -11
  30. package/dist/commands/session/prune.js.map +1 -1
  31. package/dist/commands/session/restart.js +1 -1
  32. package/dist/commands/session/restart.js.map +1 -1
  33. package/dist/commands/session/watch.d.ts +1 -0
  34. package/dist/commands/session/watch.js +46 -2
  35. package/dist/commands/session/watch.js.map +1 -1
  36. package/dist/commands/{action → sync}/index.d.ts +2 -6
  37. package/dist/commands/sync/index.js +68 -0
  38. package/dist/commands/sync/index.js.map +1 -0
  39. package/dist/commands/sync/pause.d.ts +10 -0
  40. package/dist/commands/sync/pause.js +35 -0
  41. package/dist/commands/sync/pause.js.map +1 -0
  42. package/dist/commands/sync/queue.d.ts +10 -0
  43. package/dist/commands/sync/queue.js +106 -0
  44. package/dist/commands/sync/queue.js.map +1 -0
  45. package/dist/commands/sync/resume.d.ts +10 -0
  46. package/dist/commands/sync/resume.js +34 -0
  47. package/dist/commands/sync/resume.js.map +1 -0
  48. package/dist/commands/sync/start.d.ts +11 -0
  49. package/dist/commands/sync/start.js +73 -0
  50. package/dist/commands/sync/start.js.map +1 -0
  51. package/dist/commands/sync/status.d.ts +10 -0
  52. package/dist/commands/sync/status.js +40 -0
  53. package/dist/commands/sync/status.js.map +1 -0
  54. package/dist/commands/sync/stop.d.ts +10 -0
  55. package/dist/commands/sync/stop.js +39 -0
  56. package/dist/commands/sync/stop.js.map +1 -0
  57. package/dist/commands/ticket/create.js +1 -1
  58. package/dist/commands/ticket/create.js.map +1 -1
  59. package/dist/commands/ticket/edit.js +1 -1
  60. package/dist/commands/ticket/edit.js.map +1 -1
  61. package/dist/commands/ticket/list.js +1 -1
  62. package/dist/commands/ticket/list.js.map +1 -1
  63. package/dist/commands/ticket/update.js +1 -1
  64. package/dist/commands/ticket/update.js.map +1 -1
  65. package/dist/commands/work/complete.d.ts +1 -0
  66. package/dist/commands/work/complete.js +27 -25
  67. package/dist/commands/work/complete.js.map +1 -1
  68. package/dist/commands/work/drop.d.ts +14 -0
  69. package/dist/commands/work/drop.js +215 -0
  70. package/dist/commands/work/drop.js.map +1 -0
  71. package/dist/commands/work/ready.d.ts +1 -0
  72. package/dist/commands/work/ready.js +26 -25
  73. package/dist/commands/work/ready.js.map +1 -1
  74. package/dist/commands/work/resolve.js +1 -1
  75. package/dist/commands/work/resolve.js.map +1 -1
  76. package/dist/commands/work/ship.d.ts +1 -0
  77. package/dist/commands/work/ship.js +33 -32
  78. package/dist/commands/work/ship.js.map +1 -1
  79. package/dist/commands/work/start.d.ts +2 -0
  80. package/dist/commands/work/start.js +172 -125
  81. package/dist/commands/work/start.js.map +1 -1
  82. package/dist/commands/work/stop.d.ts +1 -0
  83. package/dist/commands/work/stop.js +40 -0
  84. package/dist/commands/work/stop.js.map +1 -1
  85. package/dist/lib/agents/commands.js +7 -5
  86. package/dist/lib/agents/commands.js.map +1 -1
  87. package/dist/lib/dashboard/data.js +1 -1
  88. package/dist/lib/dashboard/data.js.map +1 -1
  89. package/dist/lib/database/db-safety.d.ts +22 -0
  90. package/dist/lib/database/db-safety.js +91 -4
  91. package/dist/lib/database/db-safety.js.map +1 -1
  92. package/dist/lib/database/drizzle-schema.d.ts +17 -0
  93. package/dist/lib/database/drizzle-schema.js +1 -0
  94. package/dist/lib/database/drizzle-schema.js.map +1 -1
  95. package/dist/lib/database/index.d.ts +1 -1
  96. package/dist/lib/database/index.js +1 -1
  97. package/dist/lib/database/index.js.map +1 -1
  98. package/dist/lib/database/migrations/0017_drop_agent_work_fk.d.ts +13 -0
  99. package/dist/lib/database/migrations/0017_drop_agent_work_fk.js +85 -0
  100. package/dist/lib/database/migrations/0017_drop_agent_work_fk.js.map +1 -0
  101. package/dist/lib/database/migrations/0018_create_ticket_refs.d.ts +11 -0
  102. package/dist/lib/database/migrations/0018_create_ticket_refs.js +40 -0
  103. package/dist/lib/database/migrations/0018_create_ticket_refs.js.map +1 -0
  104. package/dist/lib/database/migrations/0019_gc_artifact_cleanup.d.ts +9 -0
  105. package/dist/lib/database/migrations/0019_gc_artifact_cleanup.js +23 -0
  106. package/dist/lib/database/migrations/0019_gc_artifact_cleanup.js.map +1 -0
  107. package/dist/lib/database/migrations/0020_transition_map.d.ts +2 -0
  108. package/dist/lib/database/migrations/0020_transition_map.js +27 -0
  109. package/dist/lib/database/migrations/0020_transition_map.js.map +1 -0
  110. package/dist/lib/database/migrations/index.js +8 -0
  111. package/dist/lib/database/migrations/index.js.map +1 -1
  112. package/dist/lib/execution/config.d.ts +10 -0
  113. package/dist/lib/execution/config.js +24 -0
  114. package/dist/lib/execution/config.js.map +1 -1
  115. package/dist/lib/execution/devcontainer.js +1 -1
  116. package/dist/lib/execution/devcontainer.js.map +1 -1
  117. package/dist/lib/execution/preflight.d.ts +51 -0
  118. package/dist/lib/execution/preflight.js +278 -0
  119. package/dist/lib/execution/preflight.js.map +1 -0
  120. package/dist/lib/execution/runners/docker-management.d.ts +9 -0
  121. package/dist/lib/execution/runners/docker-management.js +14 -3
  122. package/dist/lib/execution/runners/docker-management.js.map +1 -1
  123. package/dist/lib/execution/runners/prompt-builder.d.ts +6 -0
  124. package/dist/lib/execution/runners/prompt-builder.js +53 -10
  125. package/dist/lib/execution/runners/prompt-builder.js.map +1 -1
  126. package/dist/lib/execution/runners/shared.d.ts +1 -1
  127. package/dist/lib/execution/runners/shared.js +1 -1
  128. package/dist/lib/execution/runners/shared.js.map +1 -1
  129. package/dist/lib/execution/session-utils.d.ts +39 -0
  130. package/dist/lib/execution/session-utils.js +111 -0
  131. package/dist/lib/execution/session-utils.js.map +1 -1
  132. package/dist/lib/execution/spawner.d.ts +11 -1
  133. package/dist/lib/execution/spawner.js +46 -17
  134. package/dist/lib/execution/spawner.js.map +1 -1
  135. package/dist/lib/execution/storage.js +2 -1
  136. package/dist/lib/execution/storage.js.map +1 -1
  137. package/dist/lib/execution/ticket-refs.d.ts +71 -0
  138. package/dist/lib/execution/ticket-refs.js +125 -0
  139. package/dist/lib/execution/ticket-refs.js.map +1 -0
  140. package/dist/lib/execution/types.d.ts +7 -2
  141. package/dist/lib/execution/types.js +5 -3
  142. package/dist/lib/execution/types.js.map +1 -1
  143. package/dist/lib/external-issues/utils.d.ts +25 -0
  144. package/dist/lib/external-issues/utils.js +32 -0
  145. package/dist/lib/external-issues/utils.js.map +1 -0
  146. package/dist/lib/gc/index.d.ts +179 -0
  147. package/dist/lib/gc/index.js +655 -0
  148. package/dist/lib/gc/index.js.map +1 -0
  149. package/dist/lib/mcp/tools/action.js +1 -1
  150. package/dist/lib/mcp/tools/action.js.map +1 -1
  151. package/dist/lib/mcp/tools/ticket.js +1 -1
  152. package/dist/lib/mcp/tools/ticket.js.map +1 -1
  153. package/dist/lib/orchestrate/actions.js +30 -0
  154. package/dist/lib/orchestrate/actions.js.map +1 -1
  155. package/dist/lib/orchestrate/poller.js +3 -3
  156. package/dist/lib/orchestrate/poller.js.map +1 -1
  157. package/dist/lib/orchestrate/presets.js +2 -1
  158. package/dist/lib/orchestrate/presets.js.map +1 -1
  159. package/dist/lib/orchestrate/types.d.ts +1 -1
  160. package/dist/lib/orchestrate/types.js +1 -0
  161. package/dist/lib/orchestrate/types.js.map +1 -1
  162. package/dist/lib/pmo/index.js +1 -1
  163. package/dist/lib/pmo/index.js.map +1 -1
  164. package/dist/lib/pmo/markdown.js +1 -1
  165. package/dist/lib/pmo/markdown.js.map +1 -1
  166. package/dist/lib/pmo/schema.d.ts +3 -1
  167. package/dist/lib/pmo/schema.js +26 -2
  168. package/dist/lib/pmo/schema.js.map +1 -1
  169. package/dist/lib/pmo/storage/actions.js +1 -1
  170. package/dist/lib/pmo/storage/actions.js.map +1 -1
  171. package/dist/lib/pmo/storage/base.js +37 -15
  172. package/dist/lib/pmo/storage/base.js.map +1 -1
  173. package/dist/lib/pmo/storage/projects.js +2 -1
  174. package/dist/lib/pmo/storage/projects.js.map +1 -1
  175. package/dist/lib/pmo/storage/statuses.js +1 -1
  176. package/dist/lib/pmo/storage/statuses.js.map +1 -1
  177. package/dist/lib/pmo/storage/subtasks.js +1 -1
  178. package/dist/lib/pmo/storage/subtasks.js.map +1 -1
  179. package/dist/lib/pmo/storage/templates.js +1 -1
  180. package/dist/lib/pmo/storage/templates.js.map +1 -1
  181. package/dist/lib/pmo/storage/tickets.js +2 -1
  182. package/dist/lib/pmo/storage/tickets.js.map +1 -1
  183. package/dist/lib/pmo/storage/workflow-rules.js +1 -1
  184. package/dist/lib/pmo/storage/workflow-rules.js.map +1 -1
  185. package/dist/lib/pmo/utils.d.ts +6 -189
  186. package/dist/lib/pmo/utils.js +6 -306
  187. package/dist/lib/pmo/utils.js.map +1 -1
  188. package/dist/lib/prompt-json.d.ts +31 -0
  189. package/dist/lib/prompt-json.js.map +1 -1
  190. package/dist/lib/providers/auto-mapper.d.ts +45 -0
  191. package/dist/lib/providers/auto-mapper.js +115 -0
  192. package/dist/lib/providers/auto-mapper.js.map +1 -0
  193. package/dist/lib/providers/state-intents.d.ts +20 -0
  194. package/dist/lib/providers/state-intents.js +61 -7
  195. package/dist/lib/providers/state-intents.js.map +1 -1
  196. package/dist/lib/providers/state-resolution.d.ts +15 -11
  197. package/dist/lib/providers/state-resolution.js +54 -48
  198. package/dist/lib/providers/state-resolution.js.map +1 -1
  199. package/dist/lib/providers/transition-map.d.ts +59 -0
  200. package/dist/lib/providers/transition-map.js +113 -0
  201. package/dist/lib/providers/transition-map.js.map +1 -0
  202. package/dist/lib/session/heartbeat.js +1 -1
  203. package/dist/lib/session/heartbeat.js.map +1 -1
  204. package/dist/lib/session/index.d.ts +3 -1
  205. package/dist/lib/session/index.js +3 -1
  206. package/dist/lib/session/index.js.map +1 -1
  207. package/dist/lib/session/tmux-watchdog.d.ts +157 -0
  208. package/dist/lib/session/tmux-watchdog.js +424 -0
  209. package/dist/lib/session/tmux-watchdog.js.map +1 -0
  210. package/dist/lib/session/watcher.d.ts +22 -4
  211. package/dist/lib/session/watcher.js +66 -8
  212. package/dist/lib/session/watcher.js.map +1 -1
  213. package/dist/lib/sync/daemon-process.d.ts +9 -0
  214. package/dist/lib/sync/daemon-process.js +184 -0
  215. package/dist/lib/sync/daemon-process.js.map +1 -0
  216. package/dist/lib/sync/daemon.d.ts +39 -0
  217. package/dist/lib/sync/daemon.js +91 -0
  218. package/dist/lib/sync/daemon.js.map +1 -0
  219. package/dist/lib/sync/engine.d.ts +38 -0
  220. package/dist/lib/sync/engine.js +145 -0
  221. package/dist/lib/sync/engine.js.map +1 -0
  222. package/dist/lib/sync/merge-queue.d.ts +116 -0
  223. package/dist/lib/sync/merge-queue.js +321 -0
  224. package/dist/lib/sync/merge-queue.js.map +1 -0
  225. package/dist/lib/sync/reconciler.d.ts +44 -0
  226. package/dist/lib/sync/reconciler.js +88 -0
  227. package/dist/lib/sync/reconciler.js.map +1 -0
  228. package/dist/lib/utils/text.d.ts +44 -0
  229. package/dist/lib/utils/text.js +72 -0
  230. package/dist/lib/utils/text.js.map +1 -0
  231. package/dist/lib/work-lifecycle/post-execution.js +1 -1
  232. package/dist/lib/work-lifecycle/post-execution.js.map +1 -1
  233. package/dist/lib/work-lifecycle/settings.d.ts +138 -0
  234. package/dist/lib/work-lifecycle/settings.js +213 -0
  235. package/dist/lib/work-lifecycle/settings.js.map +1 -0
  236. package/dist/lib/work-lifecycle/transition.d.ts +73 -0
  237. package/dist/lib/work-lifecycle/transition.js +124 -0
  238. package/dist/lib/work-lifecycle/transition.js.map +1 -0
  239. package/oclif.manifest.json +3235 -3380
  240. package/package.json +1 -1
  241. package/dist/commands/action/create.d.ts +0 -26
  242. package/dist/commands/action/create.js +0 -245
  243. package/dist/commands/action/create.js.map +0 -1
  244. package/dist/commands/action/delete.d.ts +0 -18
  245. package/dist/commands/action/delete.js +0 -78
  246. package/dist/commands/action/delete.js.map +0 -1
  247. package/dist/commands/action/index.js +0 -103
  248. package/dist/commands/action/index.js.map +0 -1
  249. package/dist/commands/action/list.d.ts +0 -15
  250. package/dist/commands/action/list.js +0 -90
  251. package/dist/commands/action/list.js.map +0 -1
  252. package/dist/commands/action/run.d.ts +0 -20
  253. package/dist/commands/action/run.js +0 -188
  254. package/dist/commands/action/run.js.map +0 -1
  255. package/dist/commands/action/show.d.ts +0 -17
  256. package/dist/commands/action/show.js +0 -78
  257. package/dist/commands/action/show.js.map +0 -1
  258. package/dist/commands/action/update.d.ts +0 -27
  259. package/dist/commands/action/update.js +0 -288
  260. package/dist/commands/action/update.js.map +0 -1
@@ -0,0 +1,157 @@
1
+ /**
2
+ * Tmux Watchdog
3
+ *
4
+ * Detects tmux server crashes and attempts automatic recovery of agent sessions.
5
+ *
6
+ * Problem: When the tmux server crashes (macOS sleep, resource pressure, OOM),
7
+ * ALL agent sessions are silently lost. The existing heartbeat system treats
8
+ * each agent as individually stale, missing the root cause. There is no
9
+ * recovery or user notification.
10
+ *
11
+ * Solution: This watchdog monitors the tmux server process itself. When a
12
+ * crash is detected (server transitions from alive→dead while agents were
13
+ * active), it:
14
+ *
15
+ * 1. Marks all affected executions with a crash-specific error
16
+ * 2. Attempts to re-create tmux sessions and restart agents
17
+ * 3. Logs crash events for user notification
18
+ *
19
+ * Architecture:
20
+ * - Runs as Phase 0 in SessionWatcher.runCycle(), before heartbeat checks
21
+ * - Maintains last-known server state per environment (host + each container)
22
+ * - Uses session-utils.ts for server status checks
23
+ * - Uses session restart logic to re-launch agents in recovered sessions
24
+ */
25
+ import { ExecutionStorage } from '../execution/storage.js';
26
+ import type { AgentWork } from '../execution/types.js';
27
+ /**
28
+ * Record of a tmux server crash event.
29
+ */
30
+ export interface TmuxCrashEvent {
31
+ /** When the crash was detected */
32
+ timestamp: Date;
33
+ /** Where the crash occurred */
34
+ environment: 'host' | 'container';
35
+ /** Container ID (only for container crashes) */
36
+ containerId?: string;
37
+ /** Executions that were active when the crash occurred */
38
+ affectedExecutions: AgentWork[];
39
+ /** Whether recovery was attempted */
40
+ recoveryAttempted: boolean;
41
+ /** Sessions that were successfully recovered */
42
+ recoveredSessions: string[];
43
+ /** Sessions that failed to recover */
44
+ failedRecoveries: string[];
45
+ }
46
+ /**
47
+ * Result of a watchdog check cycle.
48
+ */
49
+ export interface WatchdogCycleResult {
50
+ /** Whether the host tmux server is alive */
51
+ hostServerAlive: boolean;
52
+ /** Map of container ID → server alive status */
53
+ containerServersAlive: Map<string, boolean>;
54
+ /** Crash events detected in this cycle */
55
+ crashEvents: TmuxCrashEvent[];
56
+ /** Total executions affected by crashes */
57
+ totalAffected: number;
58
+ /** Total sessions recovered */
59
+ totalRecovered: number;
60
+ }
61
+ export interface TmuxWatchdogOptions {
62
+ /** Execution storage for DB operations */
63
+ storage: ExecutionStorage;
64
+ /** Whether to attempt auto-recovery (default: true) */
65
+ autoRecover?: boolean;
66
+ /** Logger function */
67
+ log?: (msg: string) => void;
68
+ /** Callback when a crash is detected */
69
+ onCrashDetected?: (event: TmuxCrashEvent) => void | Promise<void>;
70
+ }
71
+ export declare class TmuxWatchdog {
72
+ private storage;
73
+ private autoRecover;
74
+ private log;
75
+ private onCrashDetected?;
76
+ /**
77
+ * Tracks last known tmux server state.
78
+ * Key: 'host' for host server, containerId for container servers.
79
+ * Value: true = server was alive, false = server was down.
80
+ */
81
+ private lastServerState;
82
+ /**
83
+ * Tracks which executions were active when we last saw the server alive.
84
+ * Key: 'host' or containerId.
85
+ * Value: array of execution IDs that were active in that environment.
86
+ */
87
+ private activeExecutionsSnapshot;
88
+ /**
89
+ * Tracks crash events we've already handled to avoid duplicate recovery.
90
+ * Key: 'host' or containerId.
91
+ * Value: timestamp of when we detected the crash.
92
+ */
93
+ private handledCrashes;
94
+ constructor(options: TmuxWatchdogOptions);
95
+ /**
96
+ * Run a single watchdog check cycle.
97
+ *
98
+ * Checks tmux server health for host and all containers with active executions.
99
+ * Detects crashes (server alive→dead transition) and attempts recovery.
100
+ */
101
+ checkAndRecover(): Promise<WatchdogCycleResult>;
102
+ /**
103
+ * Handle a detected tmux server crash.
104
+ * Marks affected executions and attempts recovery.
105
+ */
106
+ private handleCrash;
107
+ /**
108
+ * Attempt to recover agent sessions after a tmux server crash.
109
+ *
110
+ * Recovery steps:
111
+ * 1. Wait briefly for tmux server to come back (macOS wake, Docker resume)
112
+ * 2. Create new tmux sessions with the original session names
113
+ * 3. Navigate to the working directory
114
+ * 4. Re-launch Claude Code with --resume flag
115
+ */
116
+ private attemptRecovery;
117
+ /**
118
+ * Wait for the tmux server to come back online.
119
+ * Returns true if the server recovered within the timeout.
120
+ */
121
+ private waitForServerRecovery;
122
+ /**
123
+ * Restart an agent session by creating a new tmux session and re-launching Claude Code.
124
+ */
125
+ private restartSession;
126
+ /**
127
+ * Restart a host tmux session.
128
+ */
129
+ private restartHostSession;
130
+ /**
131
+ * Restart a tmux session inside a Docker container.
132
+ */
133
+ private restartContainerSession;
134
+ /**
135
+ * Get the current known state of servers.
136
+ * Useful for diagnostics and testing.
137
+ */
138
+ getServerStates(): Map<string, boolean>;
139
+ /**
140
+ * Reset internal state. Useful for testing.
141
+ */
142
+ reset(): void;
143
+ /**
144
+ * Seed the initial server state without triggering crash detection.
145
+ * Call this once during watcher startup to establish baseline.
146
+ */
147
+ seedServerState(activeExecs: AgentWork[]): void;
148
+ }
149
+ /**
150
+ * Format a crash event into a human-readable notification string.
151
+ */
152
+ export declare function formatCrashNotification(event: TmuxCrashEvent): string;
153
+ /**
154
+ * Send a desktop notification about a tmux crash (best effort).
155
+ * Uses osascript on macOS, notify-send on Linux.
156
+ */
157
+ export declare function sendDesktopNotification(title: string, message: string): void;
@@ -0,0 +1,424 @@
1
+ /**
2
+ * Tmux Watchdog
3
+ *
4
+ * Detects tmux server crashes and attempts automatic recovery of agent sessions.
5
+ *
6
+ * Problem: When the tmux server crashes (macOS sleep, resource pressure, OOM),
7
+ * ALL agent sessions are silently lost. The existing heartbeat system treats
8
+ * each agent as individually stale, missing the root cause. There is no
9
+ * recovery or user notification.
10
+ *
11
+ * Solution: This watchdog monitors the tmux server process itself. When a
12
+ * crash is detected (server transitions from alive→dead while agents were
13
+ * active), it:
14
+ *
15
+ * 1. Marks all affected executions with a crash-specific error
16
+ * 2. Attempts to re-create tmux sessions and restart agents
17
+ * 3. Logs crash events for user notification
18
+ *
19
+ * Architecture:
20
+ * - Runs as Phase 0 in SessionWatcher.runCycle(), before heartbeat checks
21
+ * - Maintains last-known server state per environment (host + each container)
22
+ * - Uses session-utils.ts for server status checks
23
+ * - Uses session restart logic to re-launch agents in recovered sessions
24
+ */
25
+ import { execSync, execFileSync } from 'node:child_process';
26
+ import { getHostTmuxServerStatus, getContainerTmuxServerStatus, } from '../execution/session-utils.js';
27
+ // =============================================================================
28
+ // Tmux Watchdog
29
+ // =============================================================================
30
+ export class TmuxWatchdog {
31
+ storage;
32
+ autoRecover;
33
+ log;
34
+ onCrashDetected;
35
+ /**
36
+ * Tracks last known tmux server state.
37
+ * Key: 'host' for host server, containerId for container servers.
38
+ * Value: true = server was alive, false = server was down.
39
+ */
40
+ lastServerState = new Map();
41
+ /**
42
+ * Tracks which executions were active when we last saw the server alive.
43
+ * Key: 'host' or containerId.
44
+ * Value: array of execution IDs that were active in that environment.
45
+ */
46
+ activeExecutionsSnapshot = new Map();
47
+ /**
48
+ * Tracks crash events we've already handled to avoid duplicate recovery.
49
+ * Key: 'host' or containerId.
50
+ * Value: timestamp of when we detected the crash.
51
+ */
52
+ handledCrashes = new Map();
53
+ constructor(options) {
54
+ this.storage = options.storage;
55
+ this.autoRecover = options.autoRecover ?? true;
56
+ this.log = options.log ?? (() => { });
57
+ this.onCrashDetected = options.onCrashDetected;
58
+ }
59
+ /**
60
+ * Run a single watchdog check cycle.
61
+ *
62
+ * Checks tmux server health for host and all containers with active executions.
63
+ * Detects crashes (server alive→dead transition) and attempts recovery.
64
+ */
65
+ async checkAndRecover() {
66
+ const result = {
67
+ hostServerAlive: true,
68
+ containerServersAlive: new Map(),
69
+ crashEvents: [],
70
+ totalAffected: 0,
71
+ totalRecovered: 0,
72
+ };
73
+ // Get all active executions
74
+ const runningExecs = this.storage.listExecutions({ status: 'running' });
75
+ const startingExecs = this.storage.listExecutions({ status: 'starting' });
76
+ const activeExecs = [...runningExecs, ...startingExecs];
77
+ // Group by environment
78
+ const hostExecs = activeExecs.filter(e => e.environment === 'host' || e.environment === 'sandbox');
79
+ const containerExecs = activeExecs.filter(e => (e.environment === 'devcontainer' || e.environment === 'docker') && e.containerId);
80
+ // Group container executions by container ID
81
+ const containerGroups = new Map();
82
+ for (const exec of containerExecs) {
83
+ const cId = exec.containerId;
84
+ const group = containerGroups.get(cId) || [];
85
+ group.push(exec);
86
+ containerGroups.set(cId, group);
87
+ }
88
+ // === Check host tmux server ===
89
+ if (hostExecs.length > 0 || this.lastServerState.has('host')) {
90
+ const hostStatus = getHostTmuxServerStatus();
91
+ const hostAlive = hostStatus === 'running';
92
+ result.hostServerAlive = hostAlive;
93
+ const wasAlive = this.lastServerState.get('host');
94
+ // Update snapshot of active host executions when server is alive
95
+ if (hostAlive) {
96
+ this.activeExecutionsSnapshot.set('host', hostExecs.map(e => e.id));
97
+ // Clear any previous crash handling for host
98
+ this.handledCrashes.delete('host');
99
+ }
100
+ // Detect crash: server was alive, now it's dead, and there were active sessions
101
+ if (wasAlive === true && !hostAlive && !this.handledCrashes.has('host')) {
102
+ const snapshotIds = this.activeExecutionsSnapshot.get('host') || [];
103
+ // Get the actual execution objects for affected sessions
104
+ const affected = snapshotIds
105
+ .map(id => this.storage.getExecution(id))
106
+ .filter((e) => e !== null && (e.status === 'running' || e.status === 'starting'));
107
+ if (affected.length > 0) {
108
+ const event = await this.handleCrash('host', undefined, affected);
109
+ result.crashEvents.push(event);
110
+ result.totalAffected += event.affectedExecutions.length;
111
+ result.totalRecovered += event.recoveredSessions.length;
112
+ this.handledCrashes.set('host', Date.now());
113
+ }
114
+ }
115
+ this.lastServerState.set('host', hostAlive);
116
+ }
117
+ // === Check container tmux servers ===
118
+ for (const [containerId, execs] of containerGroups) {
119
+ const containerStatus = getContainerTmuxServerStatus(containerId);
120
+ const containerAlive = containerStatus === 'running';
121
+ result.containerServersAlive.set(containerId, containerAlive);
122
+ const wasAlive = this.lastServerState.get(containerId);
123
+ if (containerAlive) {
124
+ this.activeExecutionsSnapshot.set(containerId, execs.map(e => e.id));
125
+ this.handledCrashes.delete(containerId);
126
+ }
127
+ if (wasAlive === true && !containerAlive && !this.handledCrashes.has(containerId)) {
128
+ const snapshotIds = this.activeExecutionsSnapshot.get(containerId) || [];
129
+ const affected = snapshotIds
130
+ .map(id => this.storage.getExecution(id))
131
+ .filter((e) => e !== null && (e.status === 'running' || e.status === 'starting'));
132
+ if (affected.length > 0) {
133
+ const event = await this.handleCrash('container', containerId, affected);
134
+ result.crashEvents.push(event);
135
+ result.totalAffected += event.affectedExecutions.length;
136
+ result.totalRecovered += event.recoveredSessions.length;
137
+ this.handledCrashes.set(containerId, Date.now());
138
+ }
139
+ }
140
+ this.lastServerState.set(containerId, containerAlive);
141
+ }
142
+ // Clean up tracking for containers that no longer have active executions
143
+ for (const key of this.lastServerState.keys()) {
144
+ if (key !== 'host' && !containerGroups.has(key)) {
145
+ this.lastServerState.delete(key);
146
+ this.activeExecutionsSnapshot.delete(key);
147
+ this.handledCrashes.delete(key);
148
+ }
149
+ }
150
+ return result;
151
+ }
152
+ /**
153
+ * Handle a detected tmux server crash.
154
+ * Marks affected executions and attempts recovery.
155
+ */
156
+ async handleCrash(environment, containerId, affected) {
157
+ const envLabel = environment === 'host' ? 'host' : `container ${containerId?.slice(0, 12)}`;
158
+ this.log(`[watchdog] TMUX SERVER CRASH detected on ${envLabel} — ${affected.length} agent(s) affected`);
159
+ const event = {
160
+ timestamp: new Date(),
161
+ environment,
162
+ containerId,
163
+ affectedExecutions: affected,
164
+ recoveryAttempted: false,
165
+ recoveredSessions: [],
166
+ failedRecoveries: [],
167
+ };
168
+ // Mark affected executions with crash-specific error
169
+ for (const exec of affected) {
170
+ this.log(`[watchdog] Affected: ${exec.agentName} (${exec.ticketId}) — session: ${exec.sessionId || 'unknown'}`);
171
+ this.storage.updateStatus(exec.id, 'failed', undefined, `tmux server crash on ${envLabel} — all sessions lost. ` +
172
+ `${this.autoRecover ? 'Auto-recovery attempted.' : 'Manual restart required.'}`);
173
+ this.storage.updateLifecycleState(exec.id, 'died');
174
+ }
175
+ // Attempt auto-recovery if enabled
176
+ if (this.autoRecover) {
177
+ event.recoveryAttempted = true;
178
+ await this.attemptRecovery(event);
179
+ }
180
+ // Fire callback
181
+ if (this.onCrashDetected) {
182
+ await this.onCrashDetected(event);
183
+ }
184
+ // Log recovery summary
185
+ if (event.recoveryAttempted) {
186
+ if (event.recoveredSessions.length > 0) {
187
+ this.log(`[watchdog] Recovery: ${event.recoveredSessions.length}/${affected.length} sessions restored`);
188
+ }
189
+ if (event.failedRecoveries.length > 0) {
190
+ this.log(`[watchdog] Recovery failed for: ${event.failedRecoveries.join(', ')}`);
191
+ }
192
+ }
193
+ return event;
194
+ }
195
+ /**
196
+ * Attempt to recover agent sessions after a tmux server crash.
197
+ *
198
+ * Recovery steps:
199
+ * 1. Wait briefly for tmux server to come back (macOS wake, Docker resume)
200
+ * 2. Create new tmux sessions with the original session names
201
+ * 3. Navigate to the working directory
202
+ * 4. Re-launch Claude Code with --resume flag
203
+ */
204
+ async attemptRecovery(event) {
205
+ const { environment, containerId, affectedExecutions } = event;
206
+ // Wait briefly for tmux server to recover (macOS wake scenario)
207
+ const serverBack = await this.waitForServerRecovery(environment, containerId, 5000);
208
+ if (!serverBack) {
209
+ // For host: try starting a new tmux server (it auto-starts on new-session)
210
+ // For container: tmux server restarts on next tmux command
211
+ this.log('[watchdog] Tmux server did not auto-recover, will try to start new sessions');
212
+ }
213
+ for (const exec of affectedExecutions) {
214
+ const sessionName = exec.sessionId;
215
+ if (!sessionName) {
216
+ event.failedRecoveries.push(exec.agentName);
217
+ this.log(`[watchdog] Skip ${exec.agentName}: no session ID recorded`);
218
+ continue;
219
+ }
220
+ try {
221
+ const recovered = this.restartSession(exec, environment, containerId);
222
+ if (recovered) {
223
+ event.recoveredSessions.push(exec.agentName);
224
+ // Update execution back to running
225
+ this.storage.updateStatus(exec.id, 'running');
226
+ this.storage.updateLifecycleState(exec.id, 'healthy');
227
+ this.storage.updateHeartbeat(exec.id);
228
+ this.log(`[watchdog] Recovered: ${exec.agentName} (${exec.ticketId})`);
229
+ }
230
+ else {
231
+ event.failedRecoveries.push(exec.agentName);
232
+ this.log(`[watchdog] Failed to recover: ${exec.agentName} (${exec.ticketId})`);
233
+ }
234
+ }
235
+ catch (error) {
236
+ event.failedRecoveries.push(exec.agentName);
237
+ this.log(`[watchdog] Error recovering ${exec.agentName}: ${error instanceof Error ? error.message : error}`);
238
+ }
239
+ }
240
+ }
241
+ /**
242
+ * Wait for the tmux server to come back online.
243
+ * Returns true if the server recovered within the timeout.
244
+ */
245
+ async waitForServerRecovery(environment, containerId, timeoutMs) {
246
+ const start = Date.now();
247
+ const checkInterval = 1000;
248
+ while (Date.now() - start < timeoutMs) {
249
+ const status = environment === 'host'
250
+ ? getHostTmuxServerStatus()
251
+ : getContainerTmuxServerStatus(containerId);
252
+ if (status === 'running') {
253
+ this.log('[watchdog] Tmux server recovered');
254
+ return true;
255
+ }
256
+ await new Promise(resolve => setTimeout(resolve, checkInterval));
257
+ }
258
+ return false;
259
+ }
260
+ /**
261
+ * Restart an agent session by creating a new tmux session and re-launching Claude Code.
262
+ */
263
+ restartSession(exec, environment, containerId) {
264
+ const sessionName = exec.sessionId;
265
+ // Build the claude command for re-launch
266
+ let claudeCmd = 'claude --resume';
267
+ if (exec.permissionMode === 'danger') {
268
+ claudeCmd += ' --dangerously-skip-permissions';
269
+ }
270
+ if (environment === 'container' && containerId) {
271
+ return this.restartContainerSession(containerId, sessionName, claudeCmd);
272
+ }
273
+ return this.restartHostSession(sessionName, claudeCmd);
274
+ }
275
+ /**
276
+ * Restart a host tmux session.
277
+ */
278
+ restartHostSession(sessionName, command) {
279
+ try {
280
+ // Create new tmux session — this also starts the server if it's not running
281
+ execFileSync('tmux', [
282
+ 'new-session', '-d',
283
+ '-s', sessionName,
284
+ '-n', sessionName,
285
+ command,
286
+ ], {
287
+ stdio: ['pipe', 'pipe', 'pipe'],
288
+ timeout: 10000,
289
+ });
290
+ // Enable mouse support
291
+ try {
292
+ execFileSync('tmux', ['set-option', '-g', 'mouse', 'on'], {
293
+ stdio: ['pipe', 'pipe', 'pipe'],
294
+ timeout: 5000,
295
+ });
296
+ }
297
+ catch {
298
+ // Non-critical
299
+ }
300
+ return true;
301
+ }
302
+ catch {
303
+ return false;
304
+ }
305
+ }
306
+ /**
307
+ * Restart a tmux session inside a Docker container.
308
+ */
309
+ restartContainerSession(containerId, sessionName, command) {
310
+ try {
311
+ // Create new tmux session inside container
312
+ execFileSync('docker', [
313
+ 'exec', containerId,
314
+ 'tmux', 'new-session', '-d',
315
+ '-s', sessionName,
316
+ '-n', sessionName,
317
+ command,
318
+ ], {
319
+ stdio: ['pipe', 'pipe', 'pipe'],
320
+ timeout: 15000,
321
+ });
322
+ return true;
323
+ }
324
+ catch {
325
+ return false;
326
+ }
327
+ }
328
+ /**
329
+ * Get the current known state of servers.
330
+ * Useful for diagnostics and testing.
331
+ */
332
+ getServerStates() {
333
+ return new Map(this.lastServerState);
334
+ }
335
+ /**
336
+ * Reset internal state. Useful for testing.
337
+ */
338
+ reset() {
339
+ this.lastServerState.clear();
340
+ this.activeExecutionsSnapshot.clear();
341
+ this.handledCrashes.clear();
342
+ }
343
+ /**
344
+ * Seed the initial server state without triggering crash detection.
345
+ * Call this once during watcher startup to establish baseline.
346
+ */
347
+ seedServerState(activeExecs) {
348
+ // Group by environment
349
+ const hostExecs = activeExecs.filter(e => e.environment === 'host' || e.environment === 'sandbox');
350
+ const containerExecs = activeExecs.filter(e => (e.environment === 'devcontainer' || e.environment === 'docker') && e.containerId);
351
+ if (hostExecs.length > 0) {
352
+ const hostStatus = getHostTmuxServerStatus();
353
+ this.lastServerState.set('host', hostStatus === 'running');
354
+ this.activeExecutionsSnapshot.set('host', hostExecs.map(e => e.id));
355
+ }
356
+ const containerGroups = new Map();
357
+ for (const exec of containerExecs) {
358
+ const cId = exec.containerId;
359
+ const group = containerGroups.get(cId) || [];
360
+ group.push(exec);
361
+ containerGroups.set(cId, group);
362
+ }
363
+ for (const [cId, execs] of containerGroups) {
364
+ const status = getContainerTmuxServerStatus(cId);
365
+ this.lastServerState.set(cId, status === 'running');
366
+ this.activeExecutionsSnapshot.set(cId, execs.map(e => e.id));
367
+ }
368
+ }
369
+ }
370
+ // =============================================================================
371
+ // Notification Helpers
372
+ // =============================================================================
373
+ /**
374
+ * Format a crash event into a human-readable notification string.
375
+ */
376
+ export function formatCrashNotification(event) {
377
+ const envLabel = event.environment === 'host'
378
+ ? 'host tmux server'
379
+ : `container ${event.containerId?.slice(0, 12)} tmux server`;
380
+ const lines = [
381
+ `⚠️ TMUX SERVER CRASH — ${envLabel}`,
382
+ ` Time: ${event.timestamp.toLocaleString()}`,
383
+ ` Affected agents: ${event.affectedExecutions.length}`,
384
+ ];
385
+ for (const exec of event.affectedExecutions) {
386
+ lines.push(` • ${exec.agentName} (${exec.ticketId})`);
387
+ }
388
+ if (event.recoveryAttempted) {
389
+ if (event.recoveredSessions.length === event.affectedExecutions.length) {
390
+ lines.push(` Recovery: ALL ${event.recoveredSessions.length} sessions restored`);
391
+ }
392
+ else if (event.recoveredSessions.length > 0) {
393
+ lines.push(` Recovery: ${event.recoveredSessions.length}/${event.affectedExecutions.length} restored`);
394
+ lines.push(` Failed: ${event.failedRecoveries.join(', ')}`);
395
+ }
396
+ else {
397
+ lines.push(` Recovery: FAILED — all ${event.affectedExecutions.length} sessions need manual restart`);
398
+ lines.push(' Run: prlt session health');
399
+ }
400
+ }
401
+ else {
402
+ lines.push(' Auto-recovery: disabled');
403
+ lines.push(' Run: prlt session health');
404
+ }
405
+ return lines.join('\n');
406
+ }
407
+ /**
408
+ * Send a desktop notification about a tmux crash (best effort).
409
+ * Uses osascript on macOS, notify-send on Linux.
410
+ */
411
+ export function sendDesktopNotification(title, message) {
412
+ try {
413
+ if (process.platform === 'darwin') {
414
+ execSync(`osascript -e 'display notification "${message.replace(/"/g, '\\"')}" with title "${title.replace(/"/g, '\\"')}"'`, { stdio: 'pipe', timeout: 5000 });
415
+ }
416
+ else if (process.platform === 'linux') {
417
+ execSync(`notify-send "${title.replace(/"/g, '\\"')}" "${message.replace(/"/g, '\\"')}" 2>/dev/null`, { stdio: 'pipe', timeout: 5000 });
418
+ }
419
+ }
420
+ catch {
421
+ // Best effort — notification systems may not be available
422
+ }
423
+ }
424
+ //# sourceMappingURL=tmux-watchdog.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tmux-watchdog.js","sourceRoot":"","sources":["../../../src/lib/session/tmux-watchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAG3D,OAAO,EACL,uBAAuB,EACvB,4BAA4B,GAE7B,MAAM,+BAA+B,CAAA;AAqDtC,gFAAgF;AAChF,gBAAgB;AAChB,gFAAgF;AAEhF,MAAM,OAAO,YAAY;IACf,OAAO,CAAkB;IACzB,WAAW,CAAS;IACpB,GAAG,CAAuB;IAC1B,eAAe,CAAkD;IAEzE;;;;OAIG;IACK,eAAe,GAAG,IAAI,GAAG,EAAmB,CAAA;IAEpD;;;;OAIG;IACK,wBAAwB,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9D;;;;OAIG;IACK,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAA;IAElD,YAAY,OAA4B;QACtC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAA;QAC9B,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAA;QAC9C,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAA;IAChD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,MAAM,GAAwB;YAClC,eAAe,EAAE,IAAI;YACrB,qBAAqB,EAAE,IAAI,GAAG,EAAE;YAChC,WAAW,EAAE,EAAE;YACf,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;SAClB,CAAA;QAED,4BAA4B;QAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;QACvE,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;QACzE,MAAM,WAAW,GAAG,CAAC,GAAG,YAAY,EAAE,GAAG,aAAa,CAAC,CAAA;QAEvD,uBAAuB;QACvB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,CAC7D,CAAA;QACD,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,cAAc,IAAI,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CACvF,CAAA;QAED,6CAA6C;QAC7C,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAA;QACtD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAY,CAAA;YAC7B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACjC,CAAC;QAED,iCAAiC;QACjC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,uBAAuB,EAAE,CAAA;YAC5C,MAAM,SAAS,GAAG,UAAU,KAAK,SAAS,CAAA;YAC1C,MAAM,CAAC,eAAe,GAAG,SAAS,CAAA;YAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEjD,iEAAiE;YACjE,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;gBACnE,6CAA6C;gBAC7C,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YACpC,CAAC;YAED,gFAAgF;YAChF,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxE,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;gBACnE,yDAAyD;gBACzD,MAAM,QAAQ,GAAG,WAAW;qBACzB,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;qBACxC,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAA;gBAEnG,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAA;oBACjE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC9B,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAA;oBACvD,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAA;oBACvD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACH,CAAC;YAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC7C,CAAC;QAED,uCAAuC;QACvC,KAAK,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,eAAe,EAAE,CAAC;YACnD,MAAM,eAAe,GAAG,4BAA4B,CAAC,WAAW,CAAC,CAAA;YACjE,MAAM,cAAc,GAAG,eAAe,KAAK,SAAS,CAAA;YACpD,MAAM,CAAC,qBAAqB,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAA;YAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;YAEtD,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;gBACpE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YACzC,CAAC;YAED,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAClF,MAAM,WAAW,GAAG,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;gBACxE,MAAM,QAAQ,GAAG,WAAW;qBACzB,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;qBACxC,MAAM,CAAC,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAA;gBAEnG,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAA;oBACxE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC9B,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAA;oBACvD,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAA;oBACvD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBAClD,CAAC;YACH,CAAC;YAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAA;QACvD,CAAC;QAED,yEAAyE;QACzE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9C,IAAI,GAAG,KAAK,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBAChC,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACzC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CACvB,WAAiC,EACjC,WAA+B,EAC/B,QAAqB;QAErB,MAAM,QAAQ,GAAG,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;QAC3F,IAAI,CAAC,GAAG,CACN,4CAA4C,QAAQ,MAAM,QAAQ,CAAC,MAAM,oBAAoB,CAC9F,CAAA;QAED,MAAM,KAAK,GAAmB;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,WAAW;YACX,WAAW;YACX,kBAAkB,EAAE,QAAQ;YAC5B,iBAAiB,EAAE,KAAK;YACxB,iBAAiB,EAAE,EAAE;YACrB,gBAAgB,EAAE,EAAE;SACrB,CAAA;QAED,qDAAqD;QACrD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,GAAG,CACN,0BAA0B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,gBAAgB,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CACxG,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,YAAY,CACvB,IAAI,CAAC,EAAE,EACP,QAAQ,EACR,SAAS,EACT,wBAAwB,QAAQ,wBAAwB;gBACxD,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,0BAA0B,EAAE,CAChF,CAAA;YACD,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QACpD,CAAC;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAA;YAC9B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QAED,gBAAgB;QAChB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC;QAED,uBAAuB;QACvB,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,IAAI,CAAC,GAAG,CACN,wBAAwB,KAAK,CAAC,iBAAiB,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,oBAAoB,CAC9F,CAAA;YACH,CAAC;YACD,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,GAAG,CACN,mCAAmC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvE,CAAA;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,eAAe,CAAC,KAAqB;QACjD,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,kBAAkB,EAAE,GAAG,KAAK,CAAA;QAE9D,gEAAgE;QAChE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,CAAC,CAAA;QAEnF,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,2EAA2E;YAC3E,2DAA2D;YAC3D,IAAI,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAA;QACzF,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;YACtC,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAA;YAClC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC3C,IAAI,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,SAAS,0BAA0B,CAAC,CAAA;gBACvE,SAAQ;YACV,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,CAAC,CAAA;gBACrE,IAAI,SAAS,EAAE,CAAC;oBACd,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;oBAE5C,mCAAmC;oBACnC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;oBAC7C,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;oBACrD,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;oBAErC,IAAI,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;gBAC1E,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;oBAC3C,IAAI,CAAC,GAAG,CAAC,mCAAmC,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;gBAClF,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC3C,IAAI,CAAC,GAAG,CACN,iCAAiC,IAAI,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CACrG,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,qBAAqB,CACjC,WAAiC,EACjC,WAA+B,EAC/B,SAAiB;QAEjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,aAAa,GAAG,IAAI,CAAA;QAE1B,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YACtC,MAAM,MAAM,GACV,WAAW,KAAK,MAAM;gBACpB,CAAC,CAAC,uBAAuB,EAAE;gBAC3B,CAAC,CAAC,4BAA4B,CAAC,WAAY,CAAC,CAAA;YAEhD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAA;gBAC5C,OAAO,IAAI,CAAA;YACb,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC,CAAA;QAClE,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,IAAe,EACf,WAAiC,EACjC,WAA+B;QAE/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAU,CAAA;QAEnC,yCAAyC;QACzC,IAAI,SAAS,GAAG,iBAAiB,CAAA;QACjC,IAAI,IAAI,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YACrC,SAAS,IAAI,iCAAiC,CAAA;QAChD,CAAC;QAED,IAAI,WAAW,KAAK,WAAW,IAAI,WAAW,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC,uBAAuB,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAC,CAAA;QAC1E,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;IACxD,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,WAAmB,EAAE,OAAe;QAC7D,IAAI,CAAC;YACH,4EAA4E;YAC5E,YAAY,CAAC,MAAM,EAAE;gBACnB,aAAa,EAAE,IAAI;gBACnB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,WAAW;gBACjB,OAAO;aACR,EAAE;gBACD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,OAAO,EAAE,KAAK;aACf,CAAC,CAAA;YAEF,uBAAuB;YACvB,IAAI,CAAC;gBACH,YAAY,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE;oBACxD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC/B,OAAO,EAAE,IAAI;iBACd,CAAC,CAAA;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,eAAe;YACjB,CAAC;YAED,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,WAAmB,EACnB,WAAmB,EACnB,OAAe;QAEf,IAAI,CAAC;YACH,2CAA2C;YAC3C,YAAY,CAAC,QAAQ,EAAE;gBACrB,MAAM,EAAE,WAAW;gBACnB,MAAM,EAAE,aAAa,EAAE,IAAI;gBAC3B,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,WAAW;gBACjB,OAAO;aACR,EAAE;gBACD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;gBAC/B,OAAO,EAAE,KAAK;aACf,CAAC,CAAA;YACF,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,CAAA;QACrC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,WAAwB;QACtC,uBAAuB;QACvB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,CAC7D,CAAA;QACD,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,KAAK,cAAc,IAAI,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,IAAI,CAAC,CAAC,WAAW,CACvF,CAAA;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,uBAAuB,EAAE,CAAA;YAC5C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,KAAK,SAAS,CAAC,CAAA;YAC1D,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACrE,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAA;QACtD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAY,CAAA;YAC7B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACjC,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,eAAe,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAA;YAChD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,SAAS,CAAC,CAAA;YACnD,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;CACF;AAED,gFAAgF;AAChF,uBAAuB;AACvB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAqB;IAC3D,MAAM,QAAQ,GACZ,KAAK,CAAC,WAAW,KAAK,MAAM;QAC1B,CAAC,CAAC,kBAAkB;QACpB,CAAC,CAAC,aAAa,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAA;IAEhE,MAAM,KAAK,GAAa;QACtB,2BAA2B,QAAQ,EAAE;QACrC,YAAY,KAAK,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE;QAC9C,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,MAAM,EAAE;KACzD,CAAA;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;IAC3D,CAAC;IAED,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,KAAK,KAAK,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,oBAAoB,KAAK,CAAC,iBAAiB,CAAC,MAAM,oBAAoB,CAAC,CAAA;QACpF,CAAC;aAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,KAAK,CAAC,IAAI,CACR,gBAAgB,KAAK,CAAC,iBAAiB,CAAC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,MAAM,WAAW,CAC7F,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/D,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,6BAA6B,KAAK,CAAC,kBAAkB,CAAC,MAAM,+BAA+B,CAAC,CAAA;YACvG,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;QACxC,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa,EAAE,OAAe;IACpE,IAAI,CAAC;QACH,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAClC,QAAQ,CACN,uCAAuC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,iBAAiB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAClH,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAA;QACH,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACxC,QAAQ,CACN,gBAAgB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,eAAe,EAC3F,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CACjC,CAAA;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,0DAA0D;IAC5D,CAAC;AACH,CAAC"}
@@ -6,14 +6,15 @@
6
6
  * that can't self-report (OOM kills, zombie processes, container crashes).
7
7
  *
8
8
  * Architecture:
9
- * 1. Poll: Check all running executions for heartbeat timeout
10
- * 2. Update: Record heartbeats for alive agents (tmux pane inspection)
11
- * 3. Detect: Find stale executions that exceeded the timeout
12
- * 4. Act: Mark failed, kill containers, fire events
9
+ * Phase 0: Tmux server health check detect full server crashes (PRLT-1115)
10
+ * Phase 1: Record heartbeats for alive agents (tmux pane inspection)
11
+ * Phase 2: Detect stale executions that exceeded the timeout
12
+ * Phase 3: Act on stale executions — mark failed, kill containers, fire events
13
13
  */
14
14
  import type Database from 'better-sqlite3';
15
15
  import type { AgentWork } from '../execution/types.js';
16
16
  import { type StaleExecution } from './heartbeat.js';
17
+ import { TmuxWatchdog, type TmuxCrashEvent } from './tmux-watchdog.js';
17
18
  export interface WatcherOptions {
18
19
  /** Database connection */
19
20
  db: Database.Database;
@@ -23,10 +24,14 @@ export interface WatcherOptions {
23
24
  timeoutMinutes?: number;
24
25
  /** Whether to kill containers on timeout (default: true) */
25
26
  autoKill?: boolean;
27
+ /** Whether to auto-recover from tmux server crashes (default: true) */
28
+ autoRecover?: boolean;
26
29
  /** Logger function */
27
30
  log?: (msg: string) => void;
28
31
  /** Callback when a stale execution is detected and handled */
29
32
  onStaleDetected?: (execution: AgentWork, reason: string) => void | Promise<void>;
33
+ /** Callback when a tmux server crash is detected */
34
+ onCrashDetected?: (event: TmuxCrashEvent) => void | Promise<void>;
30
35
  }
31
36
  export interface WatchCycleResult {
32
37
  /** Number of active executions checked */
@@ -37,6 +42,10 @@ export interface WatchCycleResult {
37
42
  staleExecutions: StaleExecution[];
38
43
  /** Number of containers killed */
39
44
  containersKilled: number;
45
+ /** Tmux crash events detected in this cycle */
46
+ crashEvents: TmuxCrashEvent[];
47
+ /** Total sessions recovered from crashes */
48
+ crashRecoveries: number;
40
49
  }
41
50
  export declare class SessionWatcher {
42
51
  private storage;
@@ -44,10 +53,14 @@ export declare class SessionWatcher {
44
53
  private intervalMinutes;
45
54
  private timeoutMinutes;
46
55
  private autoKill;
56
+ private autoRecover;
47
57
  private log;
48
58
  private onStaleDetected?;
59
+ private onCrashDetected?;
49
60
  private timer;
50
61
  private running;
62
+ private watchdog;
63
+ private isFirstCycle;
51
64
  constructor(options: WatcherOptions);
52
65
  /**
53
66
  * Run a single watch cycle.
@@ -74,6 +87,11 @@ export declare class SessionWatcher {
74
87
  intervalMinutes: number;
75
88
  timeoutMinutes: number;
76
89
  autoKill: boolean;
90
+ autoRecover: boolean;
77
91
  };
92
+ /**
93
+ * Get the tmux watchdog instance (for testing/diagnostics).
94
+ */
95
+ getWatchdog(): TmuxWatchdog;
78
96
  private runCycleWithErrorHandling;
79
97
  }