@lumenflow/core 2.18.2 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/terminal-renderer.adapter.d.ts.map +1 -1
- package/dist/adapters/terminal-renderer.adapter.js +6 -4
- package/dist/adapters/terminal-renderer.adapter.js.map +1 -1
- package/dist/atomic-merge.d.ts +21 -0
- package/dist/atomic-merge.d.ts.map +1 -0
- package/dist/atomic-merge.js +83 -0
- package/dist/atomic-merge.js.map +1 -0
- package/dist/delegation-escalation.d.ts +91 -0
- package/dist/delegation-escalation.d.ts.map +1 -0
- package/dist/delegation-escalation.js +258 -0
- package/dist/delegation-escalation.js.map +1 -0
- package/dist/delegation-monitor.d.ts +230 -0
- package/dist/delegation-monitor.d.ts.map +1 -0
- package/dist/delegation-monitor.js +675 -0
- package/dist/delegation-monitor.js.map +1 -0
- package/dist/delegation-recovery.d.ts +83 -0
- package/dist/delegation-recovery.d.ts.map +1 -0
- package/dist/delegation-recovery.js +299 -0
- package/dist/delegation-recovery.js.map +1 -0
- package/dist/delegation-registry-schema.d.ts +80 -0
- package/dist/delegation-registry-schema.d.ts.map +1 -0
- package/dist/delegation-registry-schema.js +91 -0
- package/dist/delegation-registry-schema.js.map +1 -0
- package/dist/delegation-registry-store.d.ts +159 -0
- package/dist/delegation-registry-store.d.ts.map +1 -0
- package/dist/delegation-registry-store.js +299 -0
- package/dist/delegation-registry-store.js.map +1 -0
- package/dist/delegation-tree.d.ts +57 -0
- package/dist/delegation-tree.d.ts.map +1 -0
- package/dist/delegation-tree.js +203 -0
- package/dist/delegation-tree.js.map +1 -0
- package/dist/gates-agent-mode.d.ts +25 -0
- package/dist/gates-agent-mode.d.ts.map +1 -1
- package/dist/gates-agent-mode.js +41 -0
- package/dist/gates-agent-mode.js.map +1 -1
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -9
- package/dist/index.js.map +1 -1
- package/dist/lumenflow-config-schema.d.ts +9 -3
- package/dist/lumenflow-config-schema.d.ts.map +1 -1
- package/dist/lumenflow-config-schema.js +18 -3
- package/dist/lumenflow-config-schema.js.map +1 -1
- package/dist/lumenflow-config.d.ts +2 -0
- package/dist/lumenflow-config.d.ts.map +1 -1
- package/dist/lumenflow-config.js +1 -0
- package/dist/lumenflow-config.js.map +1 -1
- package/dist/micro-worktree-shared.d.ts +134 -0
- package/dist/micro-worktree-shared.d.ts.map +1 -0
- package/dist/micro-worktree-shared.js +350 -0
- package/dist/micro-worktree-shared.js.map +1 -0
- package/dist/micro-worktree.d.ts +4 -273
- package/dist/micro-worktree.d.ts.map +1 -1
- package/dist/micro-worktree.js +17 -549
- package/dist/micro-worktree.js.map +1 -1
- package/dist/rollback-utils.d.ts +52 -0
- package/dist/rollback-utils.d.ts.map +1 -1
- package/dist/rollback-utils.js +111 -0
- package/dist/rollback-utils.js.map +1 -1
- package/dist/schemas/index.d.ts +3 -3
- package/dist/schemas/index.d.ts.map +1 -1
- package/dist/schemas/index.js +6 -6
- package/dist/schemas/index.js.map +1 -1
- package/dist/schemas/initiative-arg-validators.d.ts +1 -0
- package/dist/schemas/initiative-arg-validators.d.ts.map +1 -1
- package/dist/schemas/initiative-schemas.d.ts +3 -1
- package/dist/schemas/initiative-schemas.d.ts.map +1 -1
- package/dist/schemas/initiative-schemas.js +2 -1
- package/dist/schemas/initiative-schemas.js.map +1 -1
- package/dist/schemas/setup-arg-validators.d.ts +4 -4
- package/dist/schemas/setup-arg-validators.d.ts.map +1 -1
- package/dist/schemas/setup-arg-validators.js +6 -6
- package/dist/schemas/setup-arg-validators.js.map +1 -1
- package/dist/schemas/setup-schemas.d.ts +7 -7
- package/dist/schemas/setup-schemas.d.ts.map +1 -1
- package/dist/schemas/setup-schemas.js +10 -10
- package/dist/schemas/setup-schemas.js.map +1 -1
- package/dist/schemas/wu-lifecycle-arg-validators.d.ts +2 -1
- package/dist/schemas/wu-lifecycle-arg-validators.d.ts.map +1 -1
- package/dist/schemas/wu-lifecycle-schemas.d.ts +5 -3
- package/dist/schemas/wu-lifecycle-schemas.d.ts.map +1 -1
- package/dist/schemas/wu-lifecycle-schemas.js +5 -1
- package/dist/schemas/wu-lifecycle-schemas.js.map +1 -1
- package/dist/template-loader.d.ts +7 -3
- package/dist/template-loader.d.ts.map +1 -1
- package/dist/template-loader.js +22 -6
- package/dist/template-loader.js.map +1 -1
- package/dist/wu-consistency-checker.d.ts +1 -0
- package/dist/wu-consistency-checker.d.ts.map +1 -1
- package/dist/wu-consistency-checker.js +31 -2
- package/dist/wu-consistency-checker.js.map +1 -1
- package/dist/wu-context-constants.d.ts +0 -2
- package/dist/wu-context-constants.d.ts.map +1 -1
- package/dist/wu-context-constants.js +0 -2
- package/dist/wu-context-constants.js.map +1 -1
- package/dist/wu-done-branch-only.d.ts +2 -11
- package/dist/wu-done-branch-only.d.ts.map +1 -1
- package/dist/wu-done-branch-only.js +81 -45
- package/dist/wu-done-branch-only.js.map +1 -1
- package/dist/wu-done-cleanup.js +33 -1
- package/dist/wu-done-cleanup.js.map +1 -1
- package/dist/wu-done-initiative-sync.d.ts.map +1 -1
- package/dist/wu-done-initiative-sync.js +20 -5
- package/dist/wu-done-initiative-sync.js.map +1 -1
- package/dist/wu-done-machine.d.ts +175 -0
- package/dist/wu-done-machine.d.ts.map +1 -0
- package/dist/wu-done-machine.js +225 -0
- package/dist/wu-done-machine.js.map +1 -0
- package/dist/wu-done-metadata.d.ts.map +1 -1
- package/dist/wu-done-metadata.js +3 -1
- package/dist/wu-done-metadata.js.map +1 -1
- package/dist/wu-done-validation.d.ts +0 -37
- package/dist/wu-done-validation.d.ts.map +1 -1
- package/dist/wu-done-validation.js +1 -155
- package/dist/wu-done-validation.js.map +1 -1
- package/dist/wu-done-validators.d.ts +1 -2
- package/dist/wu-done-validators.d.ts.map +1 -1
- package/dist/wu-done-validators.js +1 -3
- package/dist/wu-done-validators.js.map +1 -1
- package/dist/wu-done-worktree-services.d.ts +191 -0
- package/dist/wu-done-worktree-services.d.ts.map +1 -0
- package/dist/wu-done-worktree-services.js +273 -0
- package/dist/wu-done-worktree-services.js.map +1 -0
- package/dist/wu-done-worktree.d.ts +0 -19
- package/dist/wu-done-worktree.d.ts.map +1 -1
- package/dist/wu-done-worktree.js +165 -118
- package/dist/wu-done-worktree.js.map +1 -1
- package/dist/wu-git-constants.d.ts +4 -0
- package/dist/wu-git-constants.d.ts.map +1 -1
- package/dist/wu-git-constants.js +4 -0
- package/dist/wu-git-constants.js.map +1 -1
- package/dist/wu-helpers.d.ts +5 -1
- package/dist/wu-helpers.d.ts.map +1 -1
- package/dist/wu-helpers.js +5 -1
- package/dist/wu-helpers.js.map +1 -1
- package/dist/wu-lint.d.ts +24 -0
- package/dist/wu-lint.d.ts.map +1 -1
- package/dist/wu-lint.js +48 -1
- package/dist/wu-lint.js.map +1 -1
- package/dist/wu-paths-constants.d.ts +3 -3
- package/dist/wu-paths-constants.d.ts.map +1 -1
- package/dist/wu-paths-constants.js +3 -3
- package/dist/wu-paths-constants.js.map +1 -1
- package/dist/wu-recovery.d.ts +89 -0
- package/dist/wu-recovery.d.ts.map +1 -1
- package/dist/wu-recovery.js +118 -0
- package/dist/wu-recovery.js.map +1 -1
- package/dist/wu-schema.d.ts +6 -6
- package/dist/wu-spawn-context.d.ts +1 -1
- package/dist/wu-spawn-context.d.ts.map +1 -1
- package/dist/wu-spawn-context.js +8 -2
- package/dist/wu-spawn-context.js.map +1 -1
- package/dist/wu-spawn-helpers.js +2 -2
- package/dist/wu-spawn-helpers.js.map +1 -1
- package/dist/wu-state-schema.d.ts +12 -12
- package/dist/wu-state-schema.d.ts.map +1 -1
- package/dist/wu-state-schema.js +10 -10
- package/dist/wu-state-schema.js.map +1 -1
- package/dist/wu-state-store.d.ts +10 -4
- package/dist/wu-state-store.d.ts.map +1 -1
- package/dist/wu-state-store.js +309 -11
- package/dist/wu-state-store.js.map +1 -1
- package/dist/wu-transaction.d.ts +21 -0
- package/dist/wu-transaction.d.ts.map +1 -1
- package/dist/wu-transaction.js +17 -0
- package/dist/wu-transaction.js.map +1 -1
- package/package.json +15 -9
|
@@ -0,0 +1,675 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delegation Monitor Library (WU-1948, WU-1968)
|
|
3
|
+
*
|
|
4
|
+
* Core monitoring logic for detecting stuck delegations and zombie locks.
|
|
5
|
+
* Used by orchestrate:monitor CLI command.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Analyzes delegation registry for status counts
|
|
9
|
+
* - Detects pending delegations older than threshold (stuck)
|
|
10
|
+
* - Checks lane locks for zombie PIDs
|
|
11
|
+
* - Generates recovery suggestions
|
|
12
|
+
* - WU-1968: Processes delegation_failure signals from memory bus
|
|
13
|
+
*
|
|
14
|
+
* Library-First Note: This is project-specific monitoring code for
|
|
15
|
+
* PatientPath's delegation-registry.jsonl and lane-lock files. No external
|
|
16
|
+
* library exists for this custom format.
|
|
17
|
+
*
|
|
18
|
+
* @see {@link packages/@lumenflow/cli/src/__tests__/orchestrate-monitor.test.ts} - Tests
|
|
19
|
+
* @see {@link packages/@lumenflow/cli/src/lib/__tests__/delegation-monitor.test.ts} - Signal handler tests
|
|
20
|
+
* @see {@link packages/@lumenflow/cli/src/orchestrate-monitor.ts} - CLI entry point
|
|
21
|
+
* @see {@link packages/@lumenflow/cli/src/lib/delegation-registry-store.ts} - Registry storage
|
|
22
|
+
*/
|
|
23
|
+
import fs from 'node:fs/promises';
|
|
24
|
+
import path from 'node:path';
|
|
25
|
+
import { DelegationStatus } from './delegation-registry-schema.js';
|
|
26
|
+
import { isZombieLock, readLockMetadata } from './lane-lock.js';
|
|
27
|
+
import { recoverStuckDelegation, RecoveryAction } from './delegation-recovery.js';
|
|
28
|
+
import { escalateStuckDelegation, DELEGATION_FAILURE_SIGNAL_TYPE, SuggestedAction, } from './delegation-escalation.js';
|
|
29
|
+
import { LUMENFLOW_PATHS } from './wu-constants.js';
|
|
30
|
+
let loadSignals = null;
|
|
31
|
+
let markSignalsAsRead = null;
|
|
32
|
+
try {
|
|
33
|
+
const mod = await import('@lumenflow/memory/signal');
|
|
34
|
+
loadSignals = mod.loadSignals;
|
|
35
|
+
markSignalsAsRead = mod.markSignalsAsRead;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
// @lumenflow/memory not available - signal features disabled
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Default threshold for stuck delegation detection (in minutes)
|
|
42
|
+
*/
|
|
43
|
+
export const DEFAULT_THRESHOLD_MINUTES = 30;
|
|
44
|
+
/**
|
|
45
|
+
* Log prefix for delegation-monitor messages
|
|
46
|
+
*/
|
|
47
|
+
export const LOG_PREFIX = '[delegation-monitor]';
|
|
48
|
+
/**
|
|
49
|
+
* @typedef {Object} SpawnAnalysis
|
|
50
|
+
* @property {number} pending - Count of pending delegations
|
|
51
|
+
* @property {number} completed - Count of completed delegations
|
|
52
|
+
* @property {number} timeout - Count of timed out delegations
|
|
53
|
+
* @property {number} crashed - Count of crashed delegations
|
|
54
|
+
* @property {number} total - Total delegation count
|
|
55
|
+
*/
|
|
56
|
+
/**
|
|
57
|
+
* @typedef {Object} StuckSpawnInfo
|
|
58
|
+
* @property {import('./delegation-registry-schema.js').DelegationEvent} delegation - The stuck delegation event
|
|
59
|
+
* @property {number} ageMinutes - Age of delegation in minutes
|
|
60
|
+
* @property {string|null} lastCheckpoint - Last checkpoint timestamp (if available from memory layer)
|
|
61
|
+
*/
|
|
62
|
+
/**
|
|
63
|
+
* @typedef {Object} ZombieLockInfo
|
|
64
|
+
* @property {string} wuId - WU ID that holds the zombie lock
|
|
65
|
+
* @property {string} lane - Lane name
|
|
66
|
+
* @property {number} pid - Process ID (no longer running)
|
|
67
|
+
* @property {string} timestamp - When lock was acquired
|
|
68
|
+
*/
|
|
69
|
+
/**
|
|
70
|
+
* @typedef {Object} Suggestion
|
|
71
|
+
* @property {string} command - Suggested command to run
|
|
72
|
+
* @property {string} reason - Explanation of why this is suggested
|
|
73
|
+
*/
|
|
74
|
+
/**
|
|
75
|
+
* @typedef {Object} MonitorResult
|
|
76
|
+
* @property {SpawnAnalysis} analysis - Delegation status counts
|
|
77
|
+
* @property {StuckSpawnInfo[]} stuckDelegations - List of stuck delegations
|
|
78
|
+
* @property {ZombieLockInfo[]} zombieLocks - List of zombie locks
|
|
79
|
+
* @property {Suggestion[]} suggestions - Recovery suggestions
|
|
80
|
+
*/
|
|
81
|
+
/**
|
|
82
|
+
* @typedef {Object} RecoveryResultInfo
|
|
83
|
+
* @property {string} delegationId - ID of the delegation that was processed
|
|
84
|
+
* @property {string} targetWuId - Target WU ID for the delegation
|
|
85
|
+
* @property {string} action - Recovery action taken (from RecoveryAction)
|
|
86
|
+
* @property {boolean} recovered - Whether auto-recovery was successful
|
|
87
|
+
* @property {string} reason - Human-readable explanation
|
|
88
|
+
* @property {Object} [escalation] - Escalation info if action is ESCALATED_STUCK
|
|
89
|
+
* @property {string} [escalation.bugWuId] - Bug WU ID created for escalation
|
|
90
|
+
* @property {string} [escalation.title] - Bug WU title
|
|
91
|
+
*/
|
|
92
|
+
/**
|
|
93
|
+
* Analyzes delegation events and returns status counts.
|
|
94
|
+
*
|
|
95
|
+
* @param {import('./delegation-registry-schema.js').DelegationEvent[]} delegations - Array of delegation events
|
|
96
|
+
* @returns {SpawnAnalysis} Status counts
|
|
97
|
+
*
|
|
98
|
+
* @example
|
|
99
|
+
* const analysis = analyzeDelegations(delegations);
|
|
100
|
+
* console.log(`Pending: ${analysis.pending}, Completed: ${analysis.completed}`);
|
|
101
|
+
*/
|
|
102
|
+
export function analyzeDelegations(delegations) {
|
|
103
|
+
const counts = {
|
|
104
|
+
pending: 0,
|
|
105
|
+
completed: 0,
|
|
106
|
+
timeout: 0,
|
|
107
|
+
crashed: 0,
|
|
108
|
+
total: delegations.length,
|
|
109
|
+
};
|
|
110
|
+
for (const delegation of delegations) {
|
|
111
|
+
switch (delegation.status) {
|
|
112
|
+
case DelegationStatus.PENDING:
|
|
113
|
+
counts.pending++;
|
|
114
|
+
break;
|
|
115
|
+
case DelegationStatus.COMPLETED:
|
|
116
|
+
counts.completed++;
|
|
117
|
+
break;
|
|
118
|
+
case DelegationStatus.TIMEOUT:
|
|
119
|
+
counts.timeout++;
|
|
120
|
+
break;
|
|
121
|
+
case DelegationStatus.CRASHED:
|
|
122
|
+
counts.crashed++;
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return counts;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Detects pending delegations that have been running longer than the threshold.
|
|
130
|
+
*
|
|
131
|
+
* @param {import('./delegation-registry-schema.js').DelegationEvent[]} delegations - Array of delegation events
|
|
132
|
+
* @param {number} [thresholdMinutes=DEFAULT_THRESHOLD_MINUTES] - Threshold in minutes
|
|
133
|
+
* @returns {StuckSpawnInfo[]} Array of stuck delegation info
|
|
134
|
+
*
|
|
135
|
+
* @example
|
|
136
|
+
* const stuck = detectStuckDelegations(delegations, 30);
|
|
137
|
+
* for (const info of stuck) {
|
|
138
|
+
* console.log(`${info.delegation.targetWuId} stuck for ${info.ageMinutes} minutes`);
|
|
139
|
+
* }
|
|
140
|
+
*/
|
|
141
|
+
export function detectStuckDelegations(delegations, thresholdMinutes = DEFAULT_THRESHOLD_MINUTES) {
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
const thresholdMs = thresholdMinutes * 60 * 1000;
|
|
144
|
+
const stuck = [];
|
|
145
|
+
for (const delegation of delegations) {
|
|
146
|
+
// Only check pending delegations
|
|
147
|
+
if (delegation.status !== DelegationStatus.PENDING) {
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const delegatedAt = new Date(delegation.delegatedAt).getTime();
|
|
151
|
+
const ageMs = now - delegatedAt;
|
|
152
|
+
const ageMinutes = Math.floor(ageMs / (60 * 1000));
|
|
153
|
+
if (ageMs > thresholdMs) {
|
|
154
|
+
stuck.push({
|
|
155
|
+
delegation,
|
|
156
|
+
ageMinutes,
|
|
157
|
+
lastCheckpoint: delegation.lastCheckpoint ?? null,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Sort by age descending (oldest first)
|
|
162
|
+
stuck.sort((a, b) => b.ageMinutes - a.ageMinutes);
|
|
163
|
+
return stuck;
|
|
164
|
+
}
|
|
165
|
+
export async function checkZombieLocks(options = {}) {
|
|
166
|
+
const { baseDir = process.cwd() } = options;
|
|
167
|
+
// WU-1421: Use LUMENFLOW_PATHS.LOCKS_DIR (same as lane-lock.ts) for consistency
|
|
168
|
+
const locksDir = path.join(baseDir, LUMENFLOW_PATHS.LOCKS_DIR);
|
|
169
|
+
const zombies = [];
|
|
170
|
+
try {
|
|
171
|
+
// Check if locks directory exists
|
|
172
|
+
await fs.access(locksDir);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Directory doesn't exist - no locks
|
|
176
|
+
return zombies;
|
|
177
|
+
}
|
|
178
|
+
try {
|
|
179
|
+
const files = await fs.readdir(locksDir);
|
|
180
|
+
for (const file of files) {
|
|
181
|
+
if (!file.endsWith('.lock')) {
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
const lockPath = path.join(locksDir, file);
|
|
185
|
+
const metadata = readLockMetadata(lockPath);
|
|
186
|
+
if (metadata && isZombieLock(metadata)) {
|
|
187
|
+
zombies.push({
|
|
188
|
+
wuId: metadata.wuId,
|
|
189
|
+
lane: metadata.lane,
|
|
190
|
+
pid: metadata.pid,
|
|
191
|
+
timestamp: metadata.timestamp,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
// Error reading directory - return empty
|
|
198
|
+
}
|
|
199
|
+
return zombies;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Generates recovery suggestions for stuck delegations and zombie locks.
|
|
203
|
+
*
|
|
204
|
+
* @param {StuckSpawnInfo[]} stuckDelegations - Array of stuck delegation info
|
|
205
|
+
* @param {ZombieLockInfo[]} zombieLocks - Array of zombie lock info
|
|
206
|
+
* @returns {Suggestion[]} Array of suggestions
|
|
207
|
+
*
|
|
208
|
+
* @example
|
|
209
|
+
* const suggestions = generateSuggestions(stuckDelegations, zombieLocks);
|
|
210
|
+
* for (const s of suggestions) {
|
|
211
|
+
* console.log(`${s.reason}\n ${s.command}`);
|
|
212
|
+
* }
|
|
213
|
+
*/
|
|
214
|
+
export function generateSuggestions(stuckDelegations, zombieLocks) {
|
|
215
|
+
const suggestions = [];
|
|
216
|
+
// Suggestions for stuck delegations
|
|
217
|
+
for (const info of stuckDelegations) {
|
|
218
|
+
const wuId = info.delegation.targetWuId;
|
|
219
|
+
const age = info.ageMinutes;
|
|
220
|
+
suggestions.push({
|
|
221
|
+
command: `pnpm wu:block --id ${wuId} --reason "Delegation stuck for ${age} minutes"`,
|
|
222
|
+
reason: `Delegation for ${wuId} has been pending for ${age} minutes (threshold exceeded)`,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
// Suggestions for zombie locks
|
|
226
|
+
for (const lock of zombieLocks) {
|
|
227
|
+
suggestions.push({
|
|
228
|
+
command: `pnpm lane:unlock "${lock.lane}" --reason "Zombie lock (PID ${lock.pid} not running)"`,
|
|
229
|
+
reason: `Zombie lock detected for lane "${lock.lane}" (PID ${lock.pid} is not running)`,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
return suggestions;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Formats monitor output for display.
|
|
236
|
+
*
|
|
237
|
+
* @param {MonitorResult} result - Monitor result to format
|
|
238
|
+
* @returns {string} Formatted output string
|
|
239
|
+
*
|
|
240
|
+
* @example
|
|
241
|
+
* const output = formatMonitorOutput(result);
|
|
242
|
+
* console.log(output);
|
|
243
|
+
*/
|
|
244
|
+
export function formatMonitorOutput(result) {
|
|
245
|
+
const { analysis, stuckDelegations, zombieLocks, suggestions } = result;
|
|
246
|
+
const lines = [];
|
|
247
|
+
// Header
|
|
248
|
+
lines.push('=== Delegation Status Summary ===');
|
|
249
|
+
lines.push('');
|
|
250
|
+
// Status counts table
|
|
251
|
+
lines.push(` Pending: ${analysis.pending}`);
|
|
252
|
+
lines.push(` Completed: ${analysis.completed}`);
|
|
253
|
+
lines.push(` Timeout: ${analysis.timeout}`);
|
|
254
|
+
lines.push(` Crashed: ${analysis.crashed}`);
|
|
255
|
+
lines.push(' ─────────────────');
|
|
256
|
+
lines.push(` Total: ${analysis.total}`);
|
|
257
|
+
lines.push('');
|
|
258
|
+
// Stuck delegations section
|
|
259
|
+
if (stuckDelegations.length > 0) {
|
|
260
|
+
lines.push('=== Stuck Delegations ===');
|
|
261
|
+
lines.push('');
|
|
262
|
+
for (const info of stuckDelegations) {
|
|
263
|
+
lines.push(` ${info.delegation.targetWuId}`);
|
|
264
|
+
lines.push(` Lane: ${info.delegation.lane}`);
|
|
265
|
+
lines.push(` Age: ${info.ageMinutes} minutes`);
|
|
266
|
+
lines.push(` Parent: ${info.delegation.parentWuId}`);
|
|
267
|
+
if (info.lastCheckpoint) {
|
|
268
|
+
lines.push(` Last Checkpoint: ${info.lastCheckpoint}`);
|
|
269
|
+
}
|
|
270
|
+
lines.push('');
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
// Zombie locks section
|
|
274
|
+
if (zombieLocks.length > 0) {
|
|
275
|
+
lines.push('=== Zombie Locks ===');
|
|
276
|
+
lines.push('');
|
|
277
|
+
for (const lock of zombieLocks) {
|
|
278
|
+
lines.push(` ${lock.lane}`);
|
|
279
|
+
lines.push(` WU: ${lock.wuId}`);
|
|
280
|
+
lines.push(` PID: ${lock.pid} (not running)`);
|
|
281
|
+
lines.push(` Since: ${lock.timestamp}`);
|
|
282
|
+
lines.push('');
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
// Suggestions section
|
|
286
|
+
if (suggestions.length > 0) {
|
|
287
|
+
lines.push('=== Suggestions ===');
|
|
288
|
+
lines.push('');
|
|
289
|
+
for (const s of suggestions) {
|
|
290
|
+
lines.push(` ${s.reason}`);
|
|
291
|
+
lines.push(` $ ${s.command}`);
|
|
292
|
+
lines.push('');
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
// Health status
|
|
296
|
+
if (stuckDelegations.length === 0 && zombieLocks.length === 0) {
|
|
297
|
+
lines.push('No issues detected. All delegations healthy.');
|
|
298
|
+
}
|
|
299
|
+
return lines.join('\n');
|
|
300
|
+
}
|
|
301
|
+
export async function runRecovery(stuckDelegations, options = {}) {
|
|
302
|
+
const { baseDir = process.cwd(), dryRun = false } = options;
|
|
303
|
+
const results = [];
|
|
304
|
+
for (const { delegation } of stuckDelegations) {
|
|
305
|
+
const recoveryResult = await recoverStuckDelegation(delegation.id, { baseDir });
|
|
306
|
+
const resultInfo = {
|
|
307
|
+
delegationId: delegation.id,
|
|
308
|
+
targetWuId: delegation.targetWuId,
|
|
309
|
+
action: recoveryResult.action,
|
|
310
|
+
recovered: recoveryResult.recovered,
|
|
311
|
+
reason: recoveryResult.reason,
|
|
312
|
+
};
|
|
313
|
+
// Chain to escalation if action is ESCALATED_STUCK
|
|
314
|
+
if (recoveryResult.action === RecoveryAction.ESCALATED_STUCK) {
|
|
315
|
+
try {
|
|
316
|
+
const escalationResult = await escalateStuckDelegation(delegation.id, { baseDir, dryRun });
|
|
317
|
+
// escalationResult contains signalId, signal payload, and delegationStatus
|
|
318
|
+
// The signal payload has target_wu_id which represents the stuck WU
|
|
319
|
+
resultInfo.escalation = {
|
|
320
|
+
bugWuId: escalationResult.signalId,
|
|
321
|
+
title: `Escalation signal for ${delegation.targetWuId}`,
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
catch (error) {
|
|
325
|
+
// Escalation failed, but we still want to report the recovery result
|
|
326
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
327
|
+
console.log(`${LOG_PREFIX} Escalation failed for ${delegation.id}: ${message}`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
results.push(resultInfo);
|
|
331
|
+
}
|
|
332
|
+
return results;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Formats recovery results for display.
|
|
336
|
+
*
|
|
337
|
+
* @param {RecoveryResultInfo[]} results - Array of recovery results
|
|
338
|
+
* @returns {string} Formatted output string
|
|
339
|
+
*
|
|
340
|
+
* @example
|
|
341
|
+
* const output = formatRecoveryResults(results);
|
|
342
|
+
* console.log(output);
|
|
343
|
+
*/
|
|
344
|
+
export function formatRecoveryResults(results) {
|
|
345
|
+
if (results.length === 0) {
|
|
346
|
+
return 'No recovery actions taken.';
|
|
347
|
+
}
|
|
348
|
+
const lines = [];
|
|
349
|
+
// Header
|
|
350
|
+
lines.push('=== Recovery Results ===');
|
|
351
|
+
lines.push('');
|
|
352
|
+
// Count statistics
|
|
353
|
+
let recoveredCount = 0;
|
|
354
|
+
let escalatedCount = 0;
|
|
355
|
+
let noActionCount = 0;
|
|
356
|
+
for (const result of results) {
|
|
357
|
+
if (result.recovered) {
|
|
358
|
+
recoveredCount++;
|
|
359
|
+
}
|
|
360
|
+
else if (result.action === RecoveryAction.ESCALATED_STUCK) {
|
|
361
|
+
escalatedCount++;
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
noActionCount++;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
// Individual results
|
|
368
|
+
for (const result of results) {
|
|
369
|
+
lines.push(` ${result.targetWuId} (${result.delegationId})`);
|
|
370
|
+
lines.push(` Action: ${result.action}`);
|
|
371
|
+
lines.push(` Status: ${result.recovered ? 'Recovered' : 'Not auto-recovered'}`);
|
|
372
|
+
lines.push(` Reason: ${result.reason}`);
|
|
373
|
+
if (result.escalation) {
|
|
374
|
+
lines.push(` Escalation: Created ${result.escalation.bugWuId}`);
|
|
375
|
+
lines.push(` Title: ${result.escalation.title}`);
|
|
376
|
+
}
|
|
377
|
+
lines.push('');
|
|
378
|
+
}
|
|
379
|
+
// Summary
|
|
380
|
+
lines.push('--- Summary ---');
|
|
381
|
+
lines.push(` Recovered: ${recoveredCount}`);
|
|
382
|
+
lines.push(` Escalated: ${escalatedCount}`);
|
|
383
|
+
if (noActionCount > 0) {
|
|
384
|
+
lines.push(` No action: ${noActionCount}`);
|
|
385
|
+
}
|
|
386
|
+
return lines.join('\n');
|
|
387
|
+
}
|
|
388
|
+
// ============================================================================
|
|
389
|
+
// WU-1968: Delegation Failure Signal Handler
|
|
390
|
+
// ============================================================================
|
|
391
|
+
/**
|
|
392
|
+
* Log prefix for signal handler messages
|
|
393
|
+
*/
|
|
394
|
+
export const SIGNAL_HANDLER_LOG_PREFIX = '[delegation-signal-handler]';
|
|
395
|
+
/**
|
|
396
|
+
* Response actions for delegation failure signals
|
|
397
|
+
*/
|
|
398
|
+
export const SignalResponseAction = Object.freeze({
|
|
399
|
+
RETRY: 'retry',
|
|
400
|
+
BLOCK: 'block',
|
|
401
|
+
BUG_WU: 'bug_wu',
|
|
402
|
+
NONE: 'none',
|
|
403
|
+
});
|
|
404
|
+
/**
|
|
405
|
+
* @typedef {Object} SignalResponse
|
|
406
|
+
* @property {string} signalId - Signal ID that was processed
|
|
407
|
+
* @property {string} delegationId - Delegation ID from the signal
|
|
408
|
+
* @property {string} targetWuId - Target WU ID from the signal
|
|
409
|
+
* @property {string} action - Response action taken
|
|
410
|
+
* @property {string} reason - Human-readable reason for the action
|
|
411
|
+
* @property {string} severity - Original signal severity
|
|
412
|
+
* @property {boolean} wuBlocked - Whether the WU was blocked
|
|
413
|
+
* @property {string|null} bugWuCreated - Bug WU ID if created, null otherwise
|
|
414
|
+
* @property {string} [blockReason] - Reason used for blocking (if applicable)
|
|
415
|
+
* @property {Object} [bugWuSpec] - Bug WU spec (if applicable)
|
|
416
|
+
*/
|
|
417
|
+
/**
|
|
418
|
+
* @typedef {Object} SignalProcessingResult
|
|
419
|
+
* @property {SignalResponse[]} processed - Array of processed signal responses
|
|
420
|
+
* @property {number} signalCount - Total number of delegation_failure signals found
|
|
421
|
+
* @property {number} retryCount - Number of retry actions
|
|
422
|
+
* @property {number} blockCount - Number of block actions
|
|
423
|
+
* @property {number} bugWuCount - Number of Bug WU creations
|
|
424
|
+
*/
|
|
425
|
+
/**
|
|
426
|
+
* Parses a signal message to extract delegation_failure payload.
|
|
427
|
+
*
|
|
428
|
+
* @param {string} message - Signal message (may be JSON or plain text)
|
|
429
|
+
* @returns {Object|null} Parsed payload or null if not a delegation_failure signal
|
|
430
|
+
*/
|
|
431
|
+
function parseDelegationFailurePayload(message) {
|
|
432
|
+
try {
|
|
433
|
+
const parsed = JSON.parse(message);
|
|
434
|
+
if (parsed.type === DELEGATION_FAILURE_SIGNAL_TYPE) {
|
|
435
|
+
return parsed;
|
|
436
|
+
}
|
|
437
|
+
return null;
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
// Not JSON or invalid JSON - not a delegation_failure signal
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Determines the response action based on signal severity and suggested_action.
|
|
446
|
+
*
|
|
447
|
+
* Escalation levels:
|
|
448
|
+
* - First failure (severity=warning, suggested_action=retry): RETRY
|
|
449
|
+
* - Second failure (severity=error, suggested_action=block): BLOCK
|
|
450
|
+
* - Third+ failure (severity=critical, suggested_action=human_escalate): BUG_WU
|
|
451
|
+
*
|
|
452
|
+
* @param {Object} payload - Delegation failure signal payload
|
|
453
|
+
* @returns {{ action: string, reason: string }}
|
|
454
|
+
*/
|
|
455
|
+
function determineResponseAction(payload) {
|
|
456
|
+
const { suggested_action, recovery_attempts } = payload;
|
|
457
|
+
if (suggested_action === SuggestedAction.RETRY) {
|
|
458
|
+
return {
|
|
459
|
+
action: SignalResponseAction.RETRY,
|
|
460
|
+
reason: `First failure (attempt ${recovery_attempts}): suggest retry delegation`,
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
if (suggested_action === SuggestedAction.BLOCK) {
|
|
464
|
+
return {
|
|
465
|
+
action: SignalResponseAction.BLOCK,
|
|
466
|
+
reason: `Second failure (attempt ${recovery_attempts}): blocking WU`,
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
if (suggested_action === SuggestedAction.HUMAN_ESCALATE) {
|
|
470
|
+
return {
|
|
471
|
+
action: SignalResponseAction.BUG_WU,
|
|
472
|
+
reason: `Critical failure (attempt ${recovery_attempts}): creating Bug WU for human review`,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
// Unknown suggested_action - default based on severity
|
|
476
|
+
if (payload.severity === 'critical') {
|
|
477
|
+
return {
|
|
478
|
+
action: SignalResponseAction.BUG_WU,
|
|
479
|
+
reason: `Critical severity: creating Bug WU`,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
return {
|
|
483
|
+
action: SignalResponseAction.NONE,
|
|
484
|
+
reason: `Unknown suggested action: ${suggested_action}`,
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Generates a Bug WU spec for critical failures.
|
|
489
|
+
*
|
|
490
|
+
* @param {Object} payload - Delegation failure signal payload
|
|
491
|
+
* @returns {Object} Bug WU specification
|
|
492
|
+
*/
|
|
493
|
+
function generateBugWuSpec(payload) {
|
|
494
|
+
const { delegation_id, target_wu_id, lane, recovery_attempts, message, last_checkpoint } = payload;
|
|
495
|
+
const checkpointInfo = last_checkpoint
|
|
496
|
+
? `Last checkpoint: ${last_checkpoint}`
|
|
497
|
+
: 'No checkpoint recorded';
|
|
498
|
+
return {
|
|
499
|
+
title: `Bug: Stuck delegation for ${target_wu_id} (${delegation_id}) after ${recovery_attempts} attempts`,
|
|
500
|
+
lane,
|
|
501
|
+
description: [
|
|
502
|
+
`Context: Delegation ${delegation_id} for WU ${target_wu_id} has failed ${recovery_attempts} times.`,
|
|
503
|
+
``,
|
|
504
|
+
`Problem: ${message}`,
|
|
505
|
+
`${checkpointInfo}`,
|
|
506
|
+
``,
|
|
507
|
+
`Solution: Investigate root cause of repeated delegation failures.`,
|
|
508
|
+
`Consider: prompt issues, tool availability, WU spec clarity, or external dependencies.`,
|
|
509
|
+
].join('\n'),
|
|
510
|
+
type: 'bug',
|
|
511
|
+
priority: 'P1',
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Generates a block reason for second failure.
|
|
516
|
+
*
|
|
517
|
+
* @param {Object} payload - Delegation failure signal payload
|
|
518
|
+
* @returns {string} Block reason
|
|
519
|
+
*/
|
|
520
|
+
function generateBlockReason(payload) {
|
|
521
|
+
const { delegation_id, target_wu_id, recovery_attempts, message } = payload;
|
|
522
|
+
return `Delegation ${delegation_id} for ${target_wu_id} failed ${recovery_attempts} times: ${message}`;
|
|
523
|
+
}
|
|
524
|
+
/**
|
|
525
|
+
* Processes delegation_failure signals from the memory bus.
|
|
526
|
+
*
|
|
527
|
+
* WU-1968: Orchestrator signal handler for delegation_failure signals.
|
|
528
|
+
*
|
|
529
|
+
* Response logic:
|
|
530
|
+
* - First failure (suggested_action=retry): logs warning, suggests retry
|
|
531
|
+
* - Second failure (suggested_action=block): marks WU blocked with reason
|
|
532
|
+
* - Third+ failure (suggested_action=human_escalate): creates Bug WU
|
|
533
|
+
*
|
|
534
|
+
* @param {RunRecoveryOptions} options - Options
|
|
535
|
+
* @returns {Promise<SignalProcessingResult>} Processing result
|
|
536
|
+
*
|
|
537
|
+
* @example
|
|
538
|
+
* const result = await processDelegationFailureSignals({ baseDir: '/path/to/project' });
|
|
539
|
+
* console.log(`Processed ${result.signalCount} signals`);
|
|
540
|
+
* for (const response of result.processed) {
|
|
541
|
+
* console.log(`${response.targetWuId}: ${response.action}`);
|
|
542
|
+
* }
|
|
543
|
+
*/
|
|
544
|
+
export async function processDelegationFailureSignals(options = {}) {
|
|
545
|
+
const { baseDir = process.cwd(), dryRun = false } = options;
|
|
546
|
+
// Check if signal module is available
|
|
547
|
+
if (!loadSignals) {
|
|
548
|
+
return {
|
|
549
|
+
processed: [],
|
|
550
|
+
signalCount: 0,
|
|
551
|
+
retryCount: 0,
|
|
552
|
+
blockCount: 0,
|
|
553
|
+
bugWuCount: 0,
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
// Load unread signals
|
|
557
|
+
const signals = await loadSignals(baseDir, { unreadOnly: true });
|
|
558
|
+
// Filter for delegation_failure signals
|
|
559
|
+
const delegationFailureSignals = [];
|
|
560
|
+
for (const signal of signals) {
|
|
561
|
+
const payload = parseDelegationFailurePayload(signal.message);
|
|
562
|
+
if (payload) {
|
|
563
|
+
delegationFailureSignals.push({ signal, payload });
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
const processed = [];
|
|
567
|
+
let retryCount = 0;
|
|
568
|
+
let blockCount = 0;
|
|
569
|
+
let bugWuCount = 0;
|
|
570
|
+
for (const { signal, payload } of delegationFailureSignals) {
|
|
571
|
+
const { action, reason } = determineResponseAction(payload);
|
|
572
|
+
const response = {
|
|
573
|
+
signalId: signal.id,
|
|
574
|
+
delegationId: payload.delegation_id,
|
|
575
|
+
targetWuId: payload.target_wu_id,
|
|
576
|
+
action,
|
|
577
|
+
reason,
|
|
578
|
+
severity: payload.severity,
|
|
579
|
+
wuBlocked: false,
|
|
580
|
+
bugWuCreated: null,
|
|
581
|
+
};
|
|
582
|
+
// Process based on action
|
|
583
|
+
switch (action) {
|
|
584
|
+
case SignalResponseAction.RETRY:
|
|
585
|
+
retryCount++;
|
|
586
|
+
// Log warning only - no state change
|
|
587
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [WARNING] ${reason}`);
|
|
588
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Delegation: ${payload.delegation_id}`);
|
|
589
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
|
|
590
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Suggestion: Re-generate with pnpm wu:brief --id ${payload.target_wu_id} --client claude-code`);
|
|
591
|
+
break;
|
|
592
|
+
case SignalResponseAction.BLOCK:
|
|
593
|
+
blockCount++;
|
|
594
|
+
response.blockReason = generateBlockReason(payload);
|
|
595
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [BLOCK] ${reason}`);
|
|
596
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Delegation: ${payload.delegation_id}`);
|
|
597
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
|
|
598
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Reason: ${response.blockReason}`);
|
|
599
|
+
if (!dryRun) {
|
|
600
|
+
// In non-dry-run, would call wu:block here
|
|
601
|
+
// For now, just set the flag - actual blocking done by caller
|
|
602
|
+
response.wuBlocked = true;
|
|
603
|
+
}
|
|
604
|
+
break;
|
|
605
|
+
case SignalResponseAction.BUG_WU:
|
|
606
|
+
bugWuCount++;
|
|
607
|
+
response.bugWuSpec = generateBugWuSpec(payload);
|
|
608
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} [BUG WU] ${reason}`);
|
|
609
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Delegation: ${payload.delegation_id}`);
|
|
610
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Target: ${payload.target_wu_id}`);
|
|
611
|
+
console.log(`${SIGNAL_HANDLER_LOG_PREFIX} Bug WU title: ${response.bugWuSpec.title}`);
|
|
612
|
+
if (!dryRun) {
|
|
613
|
+
// In non-dry-run, would create Bug WU here
|
|
614
|
+
// For now, just set the spec - actual creation done by caller
|
|
615
|
+
// response.bugWuCreated = 'WU-XXXX' (set by caller after creation)
|
|
616
|
+
}
|
|
617
|
+
break;
|
|
618
|
+
default:
|
|
619
|
+
// NONE - no action
|
|
620
|
+
break;
|
|
621
|
+
}
|
|
622
|
+
processed.push(response);
|
|
623
|
+
}
|
|
624
|
+
// Mark processed signals as read (unless dry-run)
|
|
625
|
+
if (!dryRun && delegationFailureSignals.length > 0 && markSignalsAsRead) {
|
|
626
|
+
const signalIds = delegationFailureSignals.map((s) => s.signal.id);
|
|
627
|
+
await markSignalsAsRead(baseDir, signalIds);
|
|
628
|
+
}
|
|
629
|
+
return {
|
|
630
|
+
processed,
|
|
631
|
+
signalCount: delegationFailureSignals.length,
|
|
632
|
+
retryCount,
|
|
633
|
+
blockCount,
|
|
634
|
+
bugWuCount,
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Formats signal handler output for display.
|
|
639
|
+
*
|
|
640
|
+
* @param {SignalProcessingResult} result - Processing result
|
|
641
|
+
* @returns {string} Formatted output string
|
|
642
|
+
*
|
|
643
|
+
* @example
|
|
644
|
+
* const output = formatSignalHandlerOutput(result);
|
|
645
|
+
* console.log(output);
|
|
646
|
+
*/
|
|
647
|
+
export function formatSignalHandlerOutput(result) {
|
|
648
|
+
const { processed, signalCount, retryCount, blockCount, bugWuCount } = result;
|
|
649
|
+
const lines = [];
|
|
650
|
+
if (signalCount === 0) {
|
|
651
|
+
lines.push(`${SIGNAL_HANDLER_LOG_PREFIX} No delegation_failure signals in inbox.`);
|
|
652
|
+
return lines.join('\n');
|
|
653
|
+
}
|
|
654
|
+
lines.push(`${SIGNAL_HANDLER_LOG_PREFIX} Processed ${signalCount} delegation_failure signal(s):`);
|
|
655
|
+
lines.push('');
|
|
656
|
+
for (const response of processed) {
|
|
657
|
+
lines.push(` ${response.targetWuId} (${response.delegationId})`);
|
|
658
|
+
lines.push(` Action: ${response.action}`);
|
|
659
|
+
lines.push(` Severity: ${response.severity}`);
|
|
660
|
+
lines.push(` Reason: ${response.reason}`);
|
|
661
|
+
if (response.blockReason) {
|
|
662
|
+
lines.push(` Block reason: ${response.blockReason}`);
|
|
663
|
+
}
|
|
664
|
+
if (response.bugWuSpec) {
|
|
665
|
+
lines.push(` Bug WU: ${response.bugWuSpec.title}`);
|
|
666
|
+
}
|
|
667
|
+
lines.push('');
|
|
668
|
+
}
|
|
669
|
+
lines.push('--- Summary ---');
|
|
670
|
+
lines.push(` Retry suggestions: ${retryCount}`);
|
|
671
|
+
lines.push(` WUs blocked: ${blockCount}`);
|
|
672
|
+
lines.push(` Bug WUs created: ${bugWuCount}`);
|
|
673
|
+
return lines.join('\n');
|
|
674
|
+
}
|
|
675
|
+
//# sourceMappingURL=delegation-monitor.js.map
|