@lumenflow/core 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (263) hide show
  1. package/LICENSE +190 -0
  2. package/README.md +119 -0
  3. package/dist/active-wu-detector.d.ts +33 -0
  4. package/dist/active-wu-detector.js +106 -0
  5. package/dist/adapters/filesystem-metrics.adapter.d.ts +108 -0
  6. package/dist/adapters/filesystem-metrics.adapter.js +519 -0
  7. package/dist/adapters/terminal-renderer.adapter.d.ts +106 -0
  8. package/dist/adapters/terminal-renderer.adapter.js +337 -0
  9. package/dist/arg-parser.d.ts +63 -0
  10. package/dist/arg-parser.js +560 -0
  11. package/dist/backlog-editor.d.ts +98 -0
  12. package/dist/backlog-editor.js +179 -0
  13. package/dist/backlog-generator.d.ts +111 -0
  14. package/dist/backlog-generator.js +381 -0
  15. package/dist/backlog-parser.d.ts +45 -0
  16. package/dist/backlog-parser.js +102 -0
  17. package/dist/backlog-sync-validator.d.ts +78 -0
  18. package/dist/backlog-sync-validator.js +294 -0
  19. package/dist/branch-drift.d.ts +34 -0
  20. package/dist/branch-drift.js +51 -0
  21. package/dist/cleanup-install-config.d.ts +33 -0
  22. package/dist/cleanup-install-config.js +37 -0
  23. package/dist/cleanup-lock.d.ts +139 -0
  24. package/dist/cleanup-lock.js +313 -0
  25. package/dist/code-path-validator.d.ts +146 -0
  26. package/dist/code-path-validator.js +537 -0
  27. package/dist/code-paths-overlap.d.ts +55 -0
  28. package/dist/code-paths-overlap.js +245 -0
  29. package/dist/commands-logger.d.ts +77 -0
  30. package/dist/commands-logger.js +254 -0
  31. package/dist/commit-message-utils.d.ts +25 -0
  32. package/dist/commit-message-utils.js +41 -0
  33. package/dist/compliance-parser.d.ts +150 -0
  34. package/dist/compliance-parser.js +507 -0
  35. package/dist/constants/backlog-patterns.d.ts +20 -0
  36. package/dist/constants/backlog-patterns.js +23 -0
  37. package/dist/constants/dora-constants.d.ts +49 -0
  38. package/dist/constants/dora-constants.js +53 -0
  39. package/dist/constants/gate-constants.d.ts +15 -0
  40. package/dist/constants/gate-constants.js +15 -0
  41. package/dist/constants/linter-constants.d.ts +16 -0
  42. package/dist/constants/linter-constants.js +16 -0
  43. package/dist/constants/tokenizer-constants.d.ts +15 -0
  44. package/dist/constants/tokenizer-constants.js +15 -0
  45. package/dist/core/scope-checker.d.ts +97 -0
  46. package/dist/core/scope-checker.js +163 -0
  47. package/dist/core/tool-runner.d.ts +161 -0
  48. package/dist/core/tool-runner.js +393 -0
  49. package/dist/core/tool.constants.d.ts +105 -0
  50. package/dist/core/tool.constants.js +101 -0
  51. package/dist/core/tool.schemas.d.ts +226 -0
  52. package/dist/core/tool.schemas.js +226 -0
  53. package/dist/core/worktree-guard.d.ts +130 -0
  54. package/dist/core/worktree-guard.js +242 -0
  55. package/dist/coverage-gate.d.ts +108 -0
  56. package/dist/coverage-gate.js +196 -0
  57. package/dist/date-utils.d.ts +75 -0
  58. package/dist/date-utils.js +140 -0
  59. package/dist/dependency-graph.d.ts +142 -0
  60. package/dist/dependency-graph.js +550 -0
  61. package/dist/dependency-guard.d.ts +54 -0
  62. package/dist/dependency-guard.js +142 -0
  63. package/dist/dependency-validator.d.ts +105 -0
  64. package/dist/dependency-validator.js +154 -0
  65. package/dist/docs-path-validator.d.ts +36 -0
  66. package/dist/docs-path-validator.js +95 -0
  67. package/dist/domain/orchestration.constants.d.ts +99 -0
  68. package/dist/domain/orchestration.constants.js +97 -0
  69. package/dist/domain/orchestration.schemas.d.ts +280 -0
  70. package/dist/domain/orchestration.schemas.js +211 -0
  71. package/dist/domain/orchestration.types.d.ts +133 -0
  72. package/dist/domain/orchestration.types.js +12 -0
  73. package/dist/error-handler.d.ts +116 -0
  74. package/dist/error-handler.js +136 -0
  75. package/dist/file-classifiers.d.ts +62 -0
  76. package/dist/file-classifiers.js +108 -0
  77. package/dist/gates-agent-mode.d.ts +81 -0
  78. package/dist/gates-agent-mode.js +94 -0
  79. package/dist/generate-traceability.d.ts +107 -0
  80. package/dist/generate-traceability.js +411 -0
  81. package/dist/git-adapter.d.ts +395 -0
  82. package/dist/git-adapter.js +649 -0
  83. package/dist/git-staged-validator.d.ts +32 -0
  84. package/dist/git-staged-validator.js +48 -0
  85. package/dist/hardcoded-strings.d.ts +61 -0
  86. package/dist/hardcoded-strings.js +270 -0
  87. package/dist/incremental-lint.d.ts +78 -0
  88. package/dist/incremental-lint.js +129 -0
  89. package/dist/incremental-test.d.ts +39 -0
  90. package/dist/incremental-test.js +61 -0
  91. package/dist/index.d.ts +42 -0
  92. package/dist/index.js +61 -0
  93. package/dist/invariants/check-automated-tests.d.ts +50 -0
  94. package/dist/invariants/check-automated-tests.js +166 -0
  95. package/dist/invariants-runner.d.ts +103 -0
  96. package/dist/invariants-runner.js +527 -0
  97. package/dist/lane-checker.d.ts +50 -0
  98. package/dist/lane-checker.js +319 -0
  99. package/dist/lane-inference.d.ts +39 -0
  100. package/dist/lane-inference.js +195 -0
  101. package/dist/lane-lock.d.ts +211 -0
  102. package/dist/lane-lock.js +474 -0
  103. package/dist/lane-validator.d.ts +48 -0
  104. package/dist/lane-validator.js +114 -0
  105. package/dist/logs-lib.d.ts +104 -0
  106. package/dist/logs-lib.js +207 -0
  107. package/dist/lumenflow-config-schema.d.ts +272 -0
  108. package/dist/lumenflow-config-schema.js +207 -0
  109. package/dist/lumenflow-config.d.ts +95 -0
  110. package/dist/lumenflow-config.js +236 -0
  111. package/dist/manual-test-validator.d.ts +80 -0
  112. package/dist/manual-test-validator.js +200 -0
  113. package/dist/merge-lock.d.ts +115 -0
  114. package/dist/merge-lock.js +251 -0
  115. package/dist/micro-worktree.d.ts +159 -0
  116. package/dist/micro-worktree.js +427 -0
  117. package/dist/migration-deployer.d.ts +69 -0
  118. package/dist/migration-deployer.js +151 -0
  119. package/dist/orchestration-advisory-loader.d.ts +28 -0
  120. package/dist/orchestration-advisory-loader.js +87 -0
  121. package/dist/orchestration-advisory.d.ts +58 -0
  122. package/dist/orchestration-advisory.js +94 -0
  123. package/dist/orchestration-di.d.ts +48 -0
  124. package/dist/orchestration-di.js +57 -0
  125. package/dist/orchestration-rules.d.ts +57 -0
  126. package/dist/orchestration-rules.js +201 -0
  127. package/dist/orphan-detector.d.ts +131 -0
  128. package/dist/orphan-detector.js +226 -0
  129. package/dist/path-classifiers.d.ts +57 -0
  130. package/dist/path-classifiers.js +93 -0
  131. package/dist/piped-command-detector.d.ts +34 -0
  132. package/dist/piped-command-detector.js +64 -0
  133. package/dist/ports/dashboard-renderer.port.d.ts +112 -0
  134. package/dist/ports/dashboard-renderer.port.js +25 -0
  135. package/dist/ports/metrics-collector.port.d.ts +132 -0
  136. package/dist/ports/metrics-collector.port.js +26 -0
  137. package/dist/process-detector.d.ts +84 -0
  138. package/dist/process-detector.js +172 -0
  139. package/dist/prompt-linter.d.ts +72 -0
  140. package/dist/prompt-linter.js +312 -0
  141. package/dist/prompt-monitor.d.ts +15 -0
  142. package/dist/prompt-monitor.js +205 -0
  143. package/dist/rebase-artifact-cleanup.d.ts +145 -0
  144. package/dist/rebase-artifact-cleanup.js +433 -0
  145. package/dist/retry-strategy.d.ts +189 -0
  146. package/dist/retry-strategy.js +283 -0
  147. package/dist/risk-detector.d.ts +108 -0
  148. package/dist/risk-detector.js +252 -0
  149. package/dist/rollback-utils.d.ts +76 -0
  150. package/dist/rollback-utils.js +104 -0
  151. package/dist/section-headings.d.ts +43 -0
  152. package/dist/section-headings.js +49 -0
  153. package/dist/spawn-escalation.d.ts +90 -0
  154. package/dist/spawn-escalation.js +253 -0
  155. package/dist/spawn-monitor.d.ts +229 -0
  156. package/dist/spawn-monitor.js +672 -0
  157. package/dist/spawn-recovery.d.ts +82 -0
  158. package/dist/spawn-recovery.js +298 -0
  159. package/dist/spawn-registry-schema.d.ts +98 -0
  160. package/dist/spawn-registry-schema.js +108 -0
  161. package/dist/spawn-registry-store.d.ts +146 -0
  162. package/dist/spawn-registry-store.js +273 -0
  163. package/dist/spawn-tree.d.ts +121 -0
  164. package/dist/spawn-tree.js +285 -0
  165. package/dist/stamp-status-validator.d.ts +84 -0
  166. package/dist/stamp-status-validator.js +134 -0
  167. package/dist/stamp-utils.d.ts +100 -0
  168. package/dist/stamp-utils.js +229 -0
  169. package/dist/state-machine.d.ts +26 -0
  170. package/dist/state-machine.js +83 -0
  171. package/dist/system-map-validator.d.ts +80 -0
  172. package/dist/system-map-validator.js +272 -0
  173. package/dist/telemetry.d.ts +80 -0
  174. package/dist/telemetry.js +213 -0
  175. package/dist/token-counter.d.ts +51 -0
  176. package/dist/token-counter.js +145 -0
  177. package/dist/usecases/get-dashboard-data.usecase.d.ts +52 -0
  178. package/dist/usecases/get-dashboard-data.usecase.js +61 -0
  179. package/dist/usecases/get-suggestions.usecase.d.ts +100 -0
  180. package/dist/usecases/get-suggestions.usecase.js +153 -0
  181. package/dist/user-normalizer.d.ts +41 -0
  182. package/dist/user-normalizer.js +141 -0
  183. package/dist/validators/phi-constants.d.ts +97 -0
  184. package/dist/validators/phi-constants.js +152 -0
  185. package/dist/validators/phi-scanner.d.ts +58 -0
  186. package/dist/validators/phi-scanner.js +215 -0
  187. package/dist/worktree-ownership.d.ts +50 -0
  188. package/dist/worktree-ownership.js +74 -0
  189. package/dist/worktree-scanner.d.ts +103 -0
  190. package/dist/worktree-scanner.js +168 -0
  191. package/dist/worktree-symlink.d.ts +99 -0
  192. package/dist/worktree-symlink.js +359 -0
  193. package/dist/wu-backlog-updater.d.ts +17 -0
  194. package/dist/wu-backlog-updater.js +37 -0
  195. package/dist/wu-checkpoint.d.ts +124 -0
  196. package/dist/wu-checkpoint.js +233 -0
  197. package/dist/wu-claim-helpers.d.ts +26 -0
  198. package/dist/wu-claim-helpers.js +63 -0
  199. package/dist/wu-claim-resume.d.ts +106 -0
  200. package/dist/wu-claim-resume.js +276 -0
  201. package/dist/wu-consistency-checker.d.ts +95 -0
  202. package/dist/wu-consistency-checker.js +567 -0
  203. package/dist/wu-constants.d.ts +1275 -0
  204. package/dist/wu-constants.js +1382 -0
  205. package/dist/wu-create-validators.d.ts +42 -0
  206. package/dist/wu-create-validators.js +93 -0
  207. package/dist/wu-done-branch-only.d.ts +63 -0
  208. package/dist/wu-done-branch-only.js +191 -0
  209. package/dist/wu-done-messages.d.ts +119 -0
  210. package/dist/wu-done-messages.js +185 -0
  211. package/dist/wu-done-pr.d.ts +72 -0
  212. package/dist/wu-done-pr.js +174 -0
  213. package/dist/wu-done-retry-helpers.d.ts +85 -0
  214. package/dist/wu-done-retry-helpers.js +172 -0
  215. package/dist/wu-done-ui.d.ts +37 -0
  216. package/dist/wu-done-ui.js +69 -0
  217. package/dist/wu-done-validators.d.ts +411 -0
  218. package/dist/wu-done-validators.js +1229 -0
  219. package/dist/wu-done-worktree.d.ts +182 -0
  220. package/dist/wu-done-worktree.js +1097 -0
  221. package/dist/wu-helpers.d.ts +128 -0
  222. package/dist/wu-helpers.js +248 -0
  223. package/dist/wu-lint.d.ts +70 -0
  224. package/dist/wu-lint.js +234 -0
  225. package/dist/wu-paths.d.ts +171 -0
  226. package/dist/wu-paths.js +178 -0
  227. package/dist/wu-preflight-validators.d.ts +86 -0
  228. package/dist/wu-preflight-validators.js +251 -0
  229. package/dist/wu-recovery.d.ts +138 -0
  230. package/dist/wu-recovery.js +341 -0
  231. package/dist/wu-repair-core.d.ts +131 -0
  232. package/dist/wu-repair-core.js +669 -0
  233. package/dist/wu-schema-normalization.d.ts +17 -0
  234. package/dist/wu-schema-normalization.js +82 -0
  235. package/dist/wu-schema.d.ts +793 -0
  236. package/dist/wu-schema.js +881 -0
  237. package/dist/wu-spawn-helpers.d.ts +121 -0
  238. package/dist/wu-spawn-helpers.js +271 -0
  239. package/dist/wu-spawn.d.ts +158 -0
  240. package/dist/wu-spawn.js +1306 -0
  241. package/dist/wu-state-schema.d.ts +213 -0
  242. package/dist/wu-state-schema.js +156 -0
  243. package/dist/wu-state-store.d.ts +264 -0
  244. package/dist/wu-state-store.js +691 -0
  245. package/dist/wu-status-transition.d.ts +63 -0
  246. package/dist/wu-status-transition.js +382 -0
  247. package/dist/wu-status-updater.d.ts +25 -0
  248. package/dist/wu-status-updater.js +116 -0
  249. package/dist/wu-transaction-collectors.d.ts +116 -0
  250. package/dist/wu-transaction-collectors.js +272 -0
  251. package/dist/wu-transaction.d.ts +170 -0
  252. package/dist/wu-transaction.js +273 -0
  253. package/dist/wu-validation-constants.d.ts +60 -0
  254. package/dist/wu-validation-constants.js +66 -0
  255. package/dist/wu-validation.d.ts +118 -0
  256. package/dist/wu-validation.js +243 -0
  257. package/dist/wu-validator.d.ts +62 -0
  258. package/dist/wu-validator.js +325 -0
  259. package/dist/wu-yaml-fixer.d.ts +97 -0
  260. package/dist/wu-yaml-fixer.js +264 -0
  261. package/dist/wu-yaml.d.ts +86 -0
  262. package/dist/wu-yaml.js +222 -0
  263. package/package.json +114 -0
@@ -0,0 +1,672 @@
1
+ /**
2
+ * Spawn Monitor Library (WU-1948, WU-1968)
3
+ *
4
+ * Core monitoring logic for detecting stuck spawns and zombie locks.
5
+ * Used by orchestrate:monitor CLI command.
6
+ *
7
+ * Features:
8
+ * - Analyzes spawn registry for status counts
9
+ * - Detects pending spawns older than threshold (stuck)
10
+ * - Checks lane locks for zombie PIDs
11
+ * - Generates recovery suggestions
12
+ * - WU-1968: Processes spawn_failure signals from memory bus
13
+ *
14
+ * Library-First Note: This is project-specific monitoring code for
15
+ * PatientPath's spawn-registry.jsonl and lane-lock files. No external
16
+ * library exists for this custom format.
17
+ *
18
+ * @see {@link tools/__tests__/orchestrate-monitor.test.mjs} - Tests
19
+ * @see {@link tools/lib/__tests__/spawn-monitor.test.mjs} - Signal handler tests
20
+ * @see {@link tools/orchestrate-monitor.mjs} - CLI entry point
21
+ * @see {@link tools/lib/spawn-registry-store.mjs} - Registry storage
22
+ */
23
+ import fs from 'node:fs/promises';
24
+ import path from 'node:path';
25
+ import { SpawnStatus } from './spawn-registry-schema.js';
26
+ import { isZombieLock, readLockMetadata } from './lane-lock.js';
27
+ import { recoverStuckSpawn, RecoveryAction } from './spawn-recovery.js';
28
+ import { escalateStuckSpawn, SPAWN_FAILURE_SIGNAL_TYPE, SuggestedAction, } from './spawn-escalation.js';
29
+ let loadSignals = null;
30
+ let markSignalsAsRead = null;
31
+ try {
32
+ const mod = await import('@lumenflow/memory/signal');
33
+ loadSignals = mod.loadSignals;
34
+ markSignalsAsRead = mod.markSignalsAsRead;
35
+ }
36
+ catch {
37
+ // @lumenflow/memory not available - signal features disabled
38
+ }
39
+ /**
40
+ * Default threshold for stuck spawn detection (in minutes)
41
+ */
42
+ export const DEFAULT_THRESHOLD_MINUTES = 30;
43
+ /**
44
+ * Log prefix for spawn-monitor messages
45
+ */
46
+ export const LOG_PREFIX = '[spawn-monitor]';
47
+ /**
48
+ * @typedef {Object} SpawnAnalysis
49
+ * @property {number} pending - Count of pending spawns
50
+ * @property {number} completed - Count of completed spawns
51
+ * @property {number} timeout - Count of timed out spawns
52
+ * @property {number} crashed - Count of crashed spawns
53
+ * @property {number} total - Total spawn count
54
+ */
55
+ /**
56
+ * @typedef {Object} StuckSpawnInfo
57
+ * @property {import('./spawn-registry-schema.js').SpawnEvent} spawn - The stuck spawn event
58
+ * @property {number} ageMinutes - Age of spawn in minutes
59
+ * @property {string|null} lastCheckpoint - Last checkpoint timestamp (if available from memory layer)
60
+ */
61
+ /**
62
+ * @typedef {Object} ZombieLockInfo
63
+ * @property {string} wuId - WU ID that holds the zombie lock
64
+ * @property {string} lane - Lane name
65
+ * @property {number} pid - Process ID (no longer running)
66
+ * @property {string} timestamp - When lock was acquired
67
+ */
68
+ /**
69
+ * @typedef {Object} Suggestion
70
+ * @property {string} command - Suggested command to run
71
+ * @property {string} reason - Explanation of why this is suggested
72
+ */
73
+ /**
74
+ * @typedef {Object} MonitorResult
75
+ * @property {SpawnAnalysis} analysis - Spawn status counts
76
+ * @property {StuckSpawnInfo[]} stuckSpawns - List of stuck spawns
77
+ * @property {ZombieLockInfo[]} zombieLocks - List of zombie locks
78
+ * @property {Suggestion[]} suggestions - Recovery suggestions
79
+ */
80
+ /**
81
+ * @typedef {Object} RecoveryResultInfo
82
+ * @property {string} spawnId - ID of the spawn that was processed
83
+ * @property {string} targetWuId - Target WU ID for the spawn
84
+ * @property {string} action - Recovery action taken (from RecoveryAction)
85
+ * @property {boolean} recovered - Whether auto-recovery was successful
86
+ * @property {string} reason - Human-readable explanation
87
+ * @property {Object} [escalation] - Escalation info if action is ESCALATED_STUCK
88
+ * @property {string} [escalation.bugWuId] - Bug WU ID created for escalation
89
+ * @property {string} [escalation.title] - Bug WU title
90
+ */
91
+ /**
92
+ * Analyzes spawn events and returns status counts.
93
+ *
94
+ * @param {import('./spawn-registry-schema.js').SpawnEvent[]} spawns - Array of spawn events
95
+ * @returns {SpawnAnalysis} Status counts
96
+ *
97
+ * @example
98
+ * const analysis = analyzeSpawns(spawns);
99
+ * console.log(`Pending: ${analysis.pending}, Completed: ${analysis.completed}`);
100
+ */
101
+ export function analyzeSpawns(spawns) {
102
+ const counts = {
103
+ pending: 0,
104
+ completed: 0,
105
+ timeout: 0,
106
+ crashed: 0,
107
+ total: spawns.length,
108
+ };
109
+ for (const spawn of spawns) {
110
+ switch (spawn.status) {
111
+ case SpawnStatus.PENDING:
112
+ counts.pending++;
113
+ break;
114
+ case SpawnStatus.COMPLETED:
115
+ counts.completed++;
116
+ break;
117
+ case SpawnStatus.TIMEOUT:
118
+ counts.timeout++;
119
+ break;
120
+ case SpawnStatus.CRASHED:
121
+ counts.crashed++;
122
+ break;
123
+ }
124
+ }
125
+ return counts;
126
+ }
127
+ /**
128
+ * Detects pending spawns that have been running longer than the threshold.
129
+ *
130
+ * @param {import('./spawn-registry-schema.js').SpawnEvent[]} spawns - Array of spawn events
131
+ * @param {number} [thresholdMinutes=DEFAULT_THRESHOLD_MINUTES] - Threshold in minutes
132
+ * @returns {StuckSpawnInfo[]} Array of stuck spawn info
133
+ *
134
+ * @example
135
+ * const stuck = detectStuckSpawns(spawns, 30);
136
+ * for (const info of stuck) {
137
+ * console.log(`${info.spawn.targetWuId} stuck for ${info.ageMinutes} minutes`);
138
+ * }
139
+ */
140
+ export function detectStuckSpawns(spawns, thresholdMinutes = DEFAULT_THRESHOLD_MINUTES) {
141
+ const now = Date.now();
142
+ const thresholdMs = thresholdMinutes * 60 * 1000;
143
+ const stuck = [];
144
+ for (const spawn of spawns) {
145
+ // Only check pending spawns
146
+ if (spawn.status !== SpawnStatus.PENDING) {
147
+ continue;
148
+ }
149
+ const spawnedAt = new Date(spawn.spawnedAt).getTime();
150
+ const ageMs = now - spawnedAt;
151
+ const ageMinutes = Math.floor(ageMs / (60 * 1000));
152
+ if (ageMs > thresholdMs) {
153
+ stuck.push({
154
+ spawn,
155
+ ageMinutes,
156
+ lastCheckpoint: spawn.lastCheckpoint ?? null,
157
+ });
158
+ }
159
+ }
160
+ // Sort by age descending (oldest first)
161
+ stuck.sort((a, b) => b.ageMinutes - a.ageMinutes);
162
+ return stuck;
163
+ }
164
+ export async function checkZombieLocks(options = {}) {
165
+ const { baseDir = process.cwd() } = options;
166
+ const locksDir = path.join(baseDir, '.beacon', 'locks');
167
+ const zombies = [];
168
+ try {
169
+ // Check if locks directory exists
170
+ await fs.access(locksDir);
171
+ }
172
+ catch {
173
+ // Directory doesn't exist - no locks
174
+ return zombies;
175
+ }
176
+ try {
177
+ const files = await fs.readdir(locksDir);
178
+ for (const file of files) {
179
+ if (!file.endsWith('.lock')) {
180
+ continue;
181
+ }
182
+ const lockPath = path.join(locksDir, file);
183
+ const metadata = readLockMetadata(lockPath);
184
+ if (metadata && isZombieLock(metadata)) {
185
+ zombies.push({
186
+ wuId: metadata.wuId,
187
+ lane: metadata.lane,
188
+ pid: metadata.pid,
189
+ timestamp: metadata.timestamp,
190
+ });
191
+ }
192
+ }
193
+ }
194
+ catch {
195
+ // Error reading directory - return empty
196
+ }
197
+ return zombies;
198
+ }
199
+ /**
200
+ * Generates recovery suggestions for stuck spawns and zombie locks.
201
+ *
202
+ * @param {StuckSpawnInfo[]} stuckSpawns - Array of stuck spawn info
203
+ * @param {ZombieLockInfo[]} zombieLocks - Array of zombie lock info
204
+ * @returns {Suggestion[]} Array of suggestions
205
+ *
206
+ * @example
207
+ * const suggestions = generateSuggestions(stuckSpawns, zombieLocks);
208
+ * for (const s of suggestions) {
209
+ * console.log(`${s.reason}\n ${s.command}`);
210
+ * }
211
+ */
212
+ export function generateSuggestions(stuckSpawns, zombieLocks) {
213
+ const suggestions = [];
214
+ // Suggestions for stuck spawns
215
+ for (const info of stuckSpawns) {
216
+ const wuId = info.spawn.targetWuId;
217
+ const age = info.ageMinutes;
218
+ suggestions.push({
219
+ command: `pnpm wu:block --id ${wuId} --reason "Spawn stuck for ${age} minutes"`,
220
+ reason: `Spawn for ${wuId} has been pending for ${age} minutes (threshold exceeded)`,
221
+ });
222
+ }
223
+ // Suggestions for zombie locks
224
+ for (const lock of zombieLocks) {
225
+ suggestions.push({
226
+ command: `pnpm lane:unlock "${lock.lane}" --reason "Zombie lock (PID ${lock.pid} not running)"`,
227
+ reason: `Zombie lock detected for lane "${lock.lane}" (PID ${lock.pid} is not running)`,
228
+ });
229
+ }
230
+ return suggestions;
231
+ }
232
+ /**
233
+ * Formats monitor output for display.
234
+ *
235
+ * @param {MonitorResult} result - Monitor result to format
236
+ * @returns {string} Formatted output string
237
+ *
238
+ * @example
239
+ * const output = formatMonitorOutput(result);
240
+ * console.log(output);
241
+ */
242
+ export function formatMonitorOutput(result) {
243
+ const { analysis, stuckSpawns, zombieLocks, suggestions } = result;
244
+ const lines = [];
245
+ // Header
246
+ lines.push('=== Spawn Status Summary ===');
247
+ lines.push('');
248
+ // Status counts table
249
+ lines.push(` Pending: ${analysis.pending}`);
250
+ lines.push(` Completed: ${analysis.completed}`);
251
+ lines.push(` Timeout: ${analysis.timeout}`);
252
+ lines.push(` Crashed: ${analysis.crashed}`);
253
+ lines.push(' ─────────────────');
254
+ lines.push(` Total: ${analysis.total}`);
255
+ lines.push('');
256
+ // Stuck spawns section
257
+ if (stuckSpawns.length > 0) {
258
+ lines.push('=== Stuck Spawns ===');
259
+ lines.push('');
260
+ for (const info of stuckSpawns) {
261
+ lines.push(` ${info.spawn.targetWuId}`);
262
+ lines.push(` Lane: ${info.spawn.lane}`);
263
+ lines.push(` Age: ${info.ageMinutes} minutes`);
264
+ lines.push(` Parent: ${info.spawn.parentWuId}`);
265
+ if (info.lastCheckpoint) {
266
+ lines.push(` Last Checkpoint: ${info.lastCheckpoint}`);
267
+ }
268
+ lines.push('');
269
+ }
270
+ }
271
+ // Zombie locks section
272
+ if (zombieLocks.length > 0) {
273
+ lines.push('=== Zombie Locks ===');
274
+ lines.push('');
275
+ for (const lock of zombieLocks) {
276
+ lines.push(` ${lock.lane}`);
277
+ lines.push(` WU: ${lock.wuId}`);
278
+ lines.push(` PID: ${lock.pid} (not running)`);
279
+ lines.push(` Since: ${lock.timestamp}`);
280
+ lines.push('');
281
+ }
282
+ }
283
+ // Suggestions section
284
+ if (suggestions.length > 0) {
285
+ lines.push('=== Suggestions ===');
286
+ lines.push('');
287
+ for (const s of suggestions) {
288
+ lines.push(` ${s.reason}`);
289
+ lines.push(` $ ${s.command}`);
290
+ lines.push('');
291
+ }
292
+ }
293
+ // Health status
294
+ if (stuckSpawns.length === 0 && zombieLocks.length === 0) {
295
+ lines.push('No issues detected. All spawns healthy.');
296
+ }
297
+ return lines.join('\n');
298
+ }
299
+ export async function runRecovery(stuckSpawns, options = {}) {
300
+ const { baseDir = process.cwd(), dryRun = false } = options;
301
+ const results = [];
302
+ for (const { spawn } of stuckSpawns) {
303
+ const recoveryResult = await recoverStuckSpawn(spawn.id, { baseDir });
304
+ const resultInfo = {
305
+ spawnId: spawn.id,
306
+ targetWuId: spawn.targetWuId,
307
+ action: recoveryResult.action,
308
+ recovered: recoveryResult.recovered,
309
+ reason: recoveryResult.reason,
310
+ };
311
+ // Chain to escalation if action is ESCALATED_STUCK
312
+ if (recoveryResult.action === RecoveryAction.ESCALATED_STUCK) {
313
+ try {
314
+ const escalationResult = await escalateStuckSpawn(spawn.id, { baseDir, dryRun });
315
+ // escalationResult contains signalId, signal payload, and spawnStatus
316
+ // The signal payload has target_wu_id which represents the stuck WU
317
+ resultInfo.escalation = {
318
+ bugWuId: escalationResult.signalId,
319
+ title: `Escalation signal for ${spawn.targetWuId}`,
320
+ };
321
+ }
322
+ catch (error) {
323
+ // Escalation failed, but we still want to report the recovery result
324
+ const message = error instanceof Error ? error.message : String(error);
325
+ console.log(`${LOG_PREFIX} Escalation failed for ${spawn.id}: ${message}`);
326
+ }
327
+ }
328
+ results.push(resultInfo);
329
+ }
330
+ return results;
331
+ }
332
+ /**
333
+ * Formats recovery results for display.
334
+ *
335
+ * @param {RecoveryResultInfo[]} results - Array of recovery results
336
+ * @returns {string} Formatted output string
337
+ *
338
+ * @example
339
+ * const output = formatRecoveryResults(results);
340
+ * console.log(output);
341
+ */
342
+ export function formatRecoveryResults(results) {
343
+ if (results.length === 0) {
344
+ return 'No recovery actions taken.';
345
+ }
346
+ const lines = [];
347
+ // Header
348
+ lines.push('=== Recovery Results ===');
349
+ lines.push('');
350
+ // Count statistics
351
+ let recoveredCount = 0;
352
+ let escalatedCount = 0;
353
+ let noActionCount = 0;
354
+ for (const result of results) {
355
+ if (result.recovered) {
356
+ recoveredCount++;
357
+ }
358
+ else if (result.action === RecoveryAction.ESCALATED_STUCK) {
359
+ escalatedCount++;
360
+ }
361
+ else {
362
+ noActionCount++;
363
+ }
364
+ }
365
+ // Individual results
366
+ for (const result of results) {
367
+ lines.push(` ${result.targetWuId} (${result.spawnId})`);
368
+ lines.push(` Action: ${result.action}`);
369
+ lines.push(` Status: ${result.recovered ? 'Recovered' : 'Not auto-recovered'}`);
370
+ lines.push(` Reason: ${result.reason}`);
371
+ if (result.escalation) {
372
+ lines.push(` Escalation: Created ${result.escalation.bugWuId}`);
373
+ lines.push(` Title: ${result.escalation.title}`);
374
+ }
375
+ lines.push('');
376
+ }
377
+ // Summary
378
+ lines.push('--- Summary ---');
379
+ lines.push(` Recovered: ${recoveredCount}`);
380
+ lines.push(` Escalated: ${escalatedCount}`);
381
+ if (noActionCount > 0) {
382
+ lines.push(` No action: ${noActionCount}`);
383
+ }
384
+ return lines.join('\n');
385
+ }
386
+ // ============================================================================
387
+ // WU-1968: Spawn Failure Signal Handler
388
+ // ============================================================================
389
+ /**
390
+ * Log prefix for signal handler messages
391
+ */
392
+ export const SIGNAL_HANDLER_LOG_PREFIX = '[spawn-signal-handler]';
393
+ /**
394
+ * Response actions for spawn failure signals
395
+ */
396
+ export const SignalResponseAction = Object.freeze({
397
+ RETRY: 'retry',
398
+ BLOCK: 'block',
399
+ BUG_WU: 'bug_wu',
400
+ NONE: 'none',
401
+ });
402
+ /**
403
+ * @typedef {Object} SignalResponse
404
+ * @property {string} signalId - Signal ID that was processed
405
+ * @property {string} spawnId - Spawn ID from the signal
406
+ * @property {string} targetWuId - Target WU ID from the signal
407
+ * @property {string} action - Response action taken
408
+ * @property {string} reason - Human-readable reason for the action
409
+ * @property {string} severity - Original signal severity
410
+ * @property {boolean} wuBlocked - Whether the WU was blocked
411
+ * @property {string|null} bugWuCreated - Bug WU ID if created, null otherwise
412
+ * @property {string} [blockReason] - Reason used for blocking (if applicable)
413
+ * @property {Object} [bugWuSpec] - Bug WU spec (if applicable)
414
+ */
415
+ /**
416
+ * @typedef {Object} SignalProcessingResult
417
+ * @property {SignalResponse[]} processed - Array of processed signal responses
418
+ * @property {number} signalCount - Total number of spawn_failure signals found
419
+ * @property {number} retryCount - Number of retry actions
420
+ * @property {number} blockCount - Number of block actions
421
+ * @property {number} bugWuCount - Number of Bug WU creations
422
+ */
423
+ /**
424
+ * Parses a signal message to extract spawn_failure payload.
425
+ *
426
+ * @param {string} message - Signal message (may be JSON or plain text)
427
+ * @returns {Object|null} Parsed payload or null if not a spawn_failure signal
428
+ */
429
+ function parseSpawnFailurePayload(message) {
430
+ try {
431
+ const parsed = JSON.parse(message);
432
+ if (parsed.type === SPAWN_FAILURE_SIGNAL_TYPE) {
433
+ return parsed;
434
+ }
435
+ return null;
436
+ }
437
+ catch {
438
+ // Not JSON or invalid JSON - not a spawn_failure signal
439
+ return null;
440
+ }
441
+ }
442
+ /**
443
+ * Determines the response action based on signal severity and suggested_action.
444
+ *
445
+ * Escalation levels:
446
+ * - First failure (severity=warning, suggested_action=retry): RETRY
447
+ * - Second failure (severity=error, suggested_action=block): BLOCK
448
+ * - Third+ failure (severity=critical, suggested_action=human_escalate): BUG_WU
449
+ *
450
+ * @param {Object} payload - Spawn failure signal payload
451
+ * @returns {{ action: string, reason: string }}
452
+ */
453
+ function determineResponseAction(payload) {
454
+ const { suggested_action, recovery_attempts } = payload;
455
+ if (suggested_action === SuggestedAction.RETRY) {
456
+ return {
457
+ action: SignalResponseAction.RETRY,
458
+ reason: `First failure (attempt ${recovery_attempts}): suggest retry spawn`,
459
+ };
460
+ }
461
+ if (suggested_action === SuggestedAction.BLOCK) {
462
+ return {
463
+ action: SignalResponseAction.BLOCK,
464
+ reason: `Second failure (attempt ${recovery_attempts}): blocking WU`,
465
+ };
466
+ }
467
+ if (suggested_action === SuggestedAction.HUMAN_ESCALATE) {
468
+ return {
469
+ action: SignalResponseAction.BUG_WU,
470
+ reason: `Critical failure (attempt ${recovery_attempts}): creating Bug WU for human review`,
471
+ };
472
+ }
473
+ // Unknown suggested_action - default based on severity
474
+ if (payload.severity === 'critical') {
475
+ return {
476
+ action: SignalResponseAction.BUG_WU,
477
+ reason: `Critical severity: creating Bug WU`,
478
+ };
479
+ }
480
+ return {
481
+ action: SignalResponseAction.NONE,
482
+ reason: `Unknown suggested action: ${suggested_action}`,
483
+ };
484
+ }
485
+ /**
486
+ * Generates a Bug WU spec for critical failures.
487
+ *
488
+ * @param {Object} payload - Spawn failure signal payload
489
+ * @returns {Object} Bug WU specification
490
+ */
491
+ function generateBugWuSpec(payload) {
492
+ const { spawn_id, target_wu_id, lane, recovery_attempts, message, last_checkpoint } = payload;
493
+ const checkpointInfo = last_checkpoint
494
+ ? `Last checkpoint: ${last_checkpoint}`
495
+ : 'No checkpoint recorded';
496
+ return {
497
+ title: `Bug: Stuck spawn for ${target_wu_id} (${spawn_id}) after ${recovery_attempts} attempts`,
498
+ lane,
499
+ description: [
500
+ `Context: Spawn ${spawn_id} for WU ${target_wu_id} has failed ${recovery_attempts} times.`,
501
+ ``,
502
+ `Problem: ${message}`,
503
+ `${checkpointInfo}`,
504
+ ``,
505
+ `Solution: Investigate root cause of repeated spawn failures.`,
506
+ `Consider: prompt issues, tool availability, WU spec clarity, or external dependencies.`,
507
+ ].join('\n'),
508
+ type: 'bug',
509
+ priority: 'P1',
510
+ };
511
+ }
512
+ /**
513
+ * Generates a block reason for second failure.
514
+ *
515
+ * @param {Object} payload - Spawn failure signal payload
516
+ * @returns {string} Block reason
517
+ */
518
+ function generateBlockReason(payload) {
519
+ const { spawn_id, target_wu_id, recovery_attempts, message } = payload;
520
+ return `Spawn ${spawn_id} for ${target_wu_id} failed ${recovery_attempts} times: ${message}`;
521
+ }
522
+ /**
523
+ * Processes spawn_failure signals from the memory bus.
524
+ *
525
+ * WU-1968: Orchestrator signal handler for spawn_failure signals.
526
+ *
527
+ * Response logic:
528
+ * - First failure (suggested_action=retry): logs warning, suggests retry
529
+ * - Second failure (suggested_action=block): marks WU blocked with reason
530
+ * - Third+ failure (suggested_action=human_escalate): creates Bug WU
531
+ *
532
+ * @param {RunRecoveryOptions} options - Options
533
+ * @returns {Promise<SignalProcessingResult>} Processing result
534
+ *
535
+ * @example
536
+ * const result = await processSpawnFailureSignals({ baseDir: '/path/to/project' });
537
+ * console.log(`Processed ${result.signalCount} signals`);
538
+ * for (const response of result.processed) {
539
+ * console.log(`${response.targetWuId}: ${response.action}`);
540
+ * }
541
+ */
542
+ export async function processSpawnFailureSignals(options = {}) {
543
+ const { baseDir = process.cwd(), dryRun = false } = options;
544
+ // Check if signal module is available
545
+ if (!loadSignals) {
546
+ return {
547
+ processed: [],
548
+ signalCount: 0,
549
+ retryCount: 0,
550
+ blockCount: 0,
551
+ bugWuCount: 0,
552
+ };
553
+ }
554
+ // Load unread signals
555
+ const signals = await loadSignals(baseDir, { unreadOnly: true });
556
+ // Filter for spawn_failure signals
557
+ const spawnFailureSignals = [];
558
+ for (const signal of signals) {
559
+ const payload = parseSpawnFailurePayload(signal.message);
560
+ if (payload) {
561
+ spawnFailureSignals.push({ signal, payload });
562
+ }
563
+ }
564
+ const processed = [];
565
+ let retryCount = 0;
566
+ let blockCount = 0;
567
+ let bugWuCount = 0;
568
+ for (const { signal, payload } of spawnFailureSignals) {
569
+ const { action, reason } = determineResponseAction(payload);
570
+ const response = {
571
+ signalId: signal.id,
572
+ spawnId: payload.spawn_id,
573
+ targetWuId: payload.target_wu_id,
574
+ action,
575
+ reason,
576
+ severity: payload.severity,
577
+ wuBlocked: false,
578
+ bugWuCreated: null,
579
+ };
580
+ // Process based on action
581
+ switch (action) {
582
+ case SignalResponseAction.RETRY:
583
+ retryCount++;
584
+ // Log warning only - no state change
585
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [WARNING] ${reason}`);
586
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Spawn: ${payload.spawn_id}`);
587
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
588
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Suggestion: Re-spawn with pnpm wu:spawn`);
589
+ break;
590
+ case SignalResponseAction.BLOCK:
591
+ blockCount++;
592
+ response.blockReason = generateBlockReason(payload);
593
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [BLOCK] ${reason}`);
594
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Spawn: ${payload.spawn_id}`);
595
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
596
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Reason: ${response.blockReason}`);
597
+ if (!dryRun) {
598
+ // In non-dry-run, would call wu:block here
599
+ // For now, just set the flag - actual blocking done by caller
600
+ response.wuBlocked = true;
601
+ }
602
+ break;
603
+ case SignalResponseAction.BUG_WU:
604
+ bugWuCount++;
605
+ response.bugWuSpec = generateBugWuSpec(payload);
606
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [BUG WU] ${reason}`);
607
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Spawn: ${payload.spawn_id}`);
608
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
609
+ console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Bug WU title: ${response.bugWuSpec.title}`);
610
+ if (!dryRun) {
611
+ // In non-dry-run, would create Bug WU here
612
+ // For now, just set the spec - actual creation done by caller
613
+ // response.bugWuCreated = 'WU-XXXX' (set by caller after creation)
614
+ }
615
+ break;
616
+ default:
617
+ // NONE - no action
618
+ break;
619
+ }
620
+ processed.push(response);
621
+ }
622
+ // Mark processed signals as read (unless dry-run)
623
+ if (!dryRun && spawnFailureSignals.length > 0 && markSignalsAsRead) {
624
+ const signalIds = spawnFailureSignals.map((s) => s.signal.id);
625
+ await markSignalsAsRead(baseDir, signalIds);
626
+ }
627
+ return {
628
+ processed,
629
+ signalCount: spawnFailureSignals.length,
630
+ retryCount,
631
+ blockCount,
632
+ bugWuCount,
633
+ };
634
+ }
635
+ /**
636
+ * Formats signal handler output for display.
637
+ *
638
+ * @param {SignalProcessingResult} result - Processing result
639
+ * @returns {string} Formatted output string
640
+ *
641
+ * @example
642
+ * const output = formatSignalHandlerOutput(result);
643
+ * console.log(output);
644
+ */
645
+ export function formatSignalHandlerOutput(result) {
646
+ const { processed, signalCount, retryCount, blockCount, bugWuCount } = result;
647
+ const lines = [];
648
+ if (signalCount === 0) {
649
+ lines.push(`${SIGNAL_HANDLER_LOG_PREFIX} No spawn_failure signals in inbox.`);
650
+ return lines.join('\n');
651
+ }
652
+ lines.push(`${SIGNAL_HANDLER_LOG_PREFIX} Processed ${signalCount} spawn_failure signal(s):`);
653
+ lines.push('');
654
+ for (const response of processed) {
655
+ lines.push(` ${response.targetWuId} (${response.spawnId})`);
656
+ lines.push(` Action: ${response.action}`);
657
+ lines.push(` Severity: ${response.severity}`);
658
+ lines.push(` Reason: ${response.reason}`);
659
+ if (response.blockReason) {
660
+ lines.push(` Block reason: ${response.blockReason}`);
661
+ }
662
+ if (response.bugWuSpec) {
663
+ lines.push(` Bug WU: ${response.bugWuSpec.title}`);
664
+ }
665
+ lines.push('');
666
+ }
667
+ lines.push('--- Summary ---');
668
+ lines.push(` Retry suggestions: ${retryCount}`);
669
+ lines.push(` WUs blocked: ${blockCount}`);
670
+ lines.push(` Bug WUs created: ${bugWuCount}`);
671
+ return lines.join('\n');
672
+ }