@covibes/zeroshot 5.4.0 → 5.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +8 -465
  3. package/cli/index.js +101 -5231
  4. package/package.json +16 -130
  5. package/CHANGELOG.md +0 -537
  6. package/cli/commands/providers.js +0 -149
  7. package/cli/lib/first-run.js +0 -211
  8. package/cli/lib/update-checker.js +0 -281
  9. package/cli/message-formatter-utils.js +0 -75
  10. package/cli/message-formatters-normal.js +0 -275
  11. package/cli/message-formatters-watch.js +0 -181
  12. package/cluster-templates/base-templates/debug-workflow.json +0 -375
  13. package/cluster-templates/base-templates/full-workflow.json +0 -581
  14. package/cluster-templates/base-templates/single-worker.json +0 -63
  15. package/cluster-templates/base-templates/worker-validator.json +0 -208
  16. package/cluster-templates/conductor-bootstrap.json +0 -124
  17. package/docker/zeroshot-cluster/Dockerfile +0 -152
  18. package/docker/zeroshot-cluster/pre-baked-deps.json +0 -28
  19. package/lib/completion.js +0 -174
  20. package/lib/docker-config.js +0 -220
  21. package/lib/git-remote-utils.js +0 -165
  22. package/lib/id-detector.js +0 -56
  23. package/lib/provider-defaults.js +0 -62
  24. package/lib/provider-detection.js +0 -59
  25. package/lib/provider-names.js +0 -57
  26. package/lib/settings/claude-auth.js +0 -78
  27. package/lib/settings.js +0 -503
  28. package/lib/stream-json-parser.js +0 -10
  29. package/scripts/fix-node-pty-permissions.js +0 -75
  30. package/scripts/record-demo.sh +0 -279
  31. package/scripts/setup-merge-queue.sh +0 -170
  32. package/scripts/test-install.sh +0 -40
  33. package/scripts/validate-templates.js +0 -100
  34. package/src/agent/agent-config.js +0 -246
  35. package/src/agent/agent-context-builder.js +0 -444
  36. package/src/agent/agent-hook-executor.js +0 -599
  37. package/src/agent/agent-lifecycle.js +0 -743
  38. package/src/agent/agent-stuck-detector.js +0 -256
  39. package/src/agent/agent-task-executor.js +0 -1690
  40. package/src/agent/agent-trigger-evaluator.js +0 -67
  41. package/src/agent/output-extraction.js +0 -217
  42. package/src/agent/output-reformatter.js +0 -175
  43. package/src/agent/schema-utils.js +0 -146
  44. package/src/agent-wrapper.js +0 -579
  45. package/src/agents/git-pusher-template.js +0 -285
  46. package/src/attach/attach-client.js +0 -438
  47. package/src/attach/attach-server.js +0 -543
  48. package/src/attach/index.js +0 -35
  49. package/src/attach/protocol.js +0 -220
  50. package/src/attach/ring-buffer.js +0 -121
  51. package/src/attach/socket-discovery.js +0 -267
  52. package/src/claude-task-runner.js +0 -574
  53. package/src/config-router.js +0 -80
  54. package/src/config-validator.js +0 -1925
  55. package/src/copy-worker.js +0 -43
  56. package/src/input-helpers.js +0 -65
  57. package/src/isolation-manager.js +0 -1471
  58. package/src/issue-providers/README.md +0 -305
  59. package/src/issue-providers/azure-devops-provider.js +0 -273
  60. package/src/issue-providers/base-provider.js +0 -232
  61. package/src/issue-providers/github-provider.js +0 -179
  62. package/src/issue-providers/gitlab-provider.js +0 -241
  63. package/src/issue-providers/index.js +0 -196
  64. package/src/issue-providers/jira-provider.js +0 -239
  65. package/src/ledger.js +0 -631
  66. package/src/lib/safe-exec.js +0 -88
  67. package/src/logic-engine.js +0 -223
  68. package/src/message-bus-bridge.js +0 -139
  69. package/src/message-bus.js +0 -249
  70. package/src/name-generator.js +0 -232
  71. package/src/orchestrator.js +0 -2815
  72. package/src/preflight.js +0 -639
  73. package/src/process-metrics.js +0 -608
  74. package/src/providers/anthropic/cli-builder.js +0 -73
  75. package/src/providers/anthropic/index.js +0 -204
  76. package/src/providers/anthropic/models.js +0 -23
  77. package/src/providers/anthropic/output-parser.js +0 -177
  78. package/src/providers/base-provider.js +0 -251
  79. package/src/providers/capabilities.js +0 -60
  80. package/src/providers/google/cli-builder.js +0 -55
  81. package/src/providers/google/index.js +0 -116
  82. package/src/providers/google/models.js +0 -24
  83. package/src/providers/google/output-parser.js +0 -101
  84. package/src/providers/index.js +0 -91
  85. package/src/providers/openai/cli-builder.js +0 -133
  86. package/src/providers/openai/index.js +0 -136
  87. package/src/providers/openai/models.js +0 -21
  88. package/src/providers/openai/output-parser.js +0 -143
  89. package/src/providers/opencode/cli-builder.js +0 -42
  90. package/src/providers/opencode/index.js +0 -103
  91. package/src/providers/opencode/models.js +0 -55
  92. package/src/providers/opencode/output-parser.js +0 -122
  93. package/src/schemas/sub-cluster.js +0 -185
  94. package/src/status-footer.js +0 -1026
  95. package/src/sub-cluster-wrapper.js +0 -601
  96. package/src/task-runner.js +0 -30
  97. package/src/template-resolver.js +0 -372
  98. package/src/tui/CHANGES.txt +0 -133
  99. package/src/tui/LAYOUT.md +0 -261
  100. package/src/tui/README.txt +0 -192
  101. package/src/tui/TWO-LEVEL-NAVIGATION.md +0 -186
  102. package/src/tui/data-poller.js +0 -349
  103. package/src/tui/demo.js +0 -208
  104. package/src/tui/formatters.js +0 -129
  105. package/src/tui/index.js +0 -193
  106. package/src/tui/keybindings.js +0 -324
  107. package/src/tui/layout.js +0 -334
  108. package/src/tui/renderer.js +0 -184
  109. package/task-lib/attachable-watcher.js +0 -309
  110. package/task-lib/claude-recovery.js +0 -156
  111. package/task-lib/commands/clean.js +0 -50
  112. package/task-lib/commands/episodes.js +0 -105
  113. package/task-lib/commands/get-log-path.js +0 -23
  114. package/task-lib/commands/kill.js +0 -32
  115. package/task-lib/commands/list.js +0 -105
  116. package/task-lib/commands/logs.js +0 -453
  117. package/task-lib/commands/resume.js +0 -42
  118. package/task-lib/commands/run.js +0 -57
  119. package/task-lib/commands/schedule.js +0 -105
  120. package/task-lib/commands/scheduler-cmd.js +0 -96
  121. package/task-lib/commands/schedules.js +0 -148
  122. package/task-lib/commands/status.js +0 -44
  123. package/task-lib/commands/unschedule.js +0 -16
  124. package/task-lib/completion.js +0 -9
  125. package/task-lib/config.js +0 -8
  126. package/task-lib/name-generator.js +0 -230
  127. package/task-lib/package.json +0 -3
  128. package/task-lib/runner.js +0 -201
  129. package/task-lib/scheduler.js +0 -252
  130. package/task-lib/store.js +0 -529
  131. package/task-lib/tui/formatters.js +0 -215
  132. package/task-lib/tui/index.js +0 -197
  133. package/task-lib/tui/layout.js +0 -111
  134. package/task-lib/tui/renderer.js +0 -129
  135. package/task-lib/tui.js +0 -425
  136. package/task-lib/watcher.js +0 -226
@@ -1,743 +0,0 @@
1
- // @ts-nocheck
2
- /**
3
- * AgentLifecycle - Agent state machine and lifecycle management
4
- *
5
- * Provides:
6
- * - Agent startup and shutdown
7
- * - Message handling and routing
8
- * - Trigger action execution (execute_task, stop_cluster)
9
- * - Task execution with retry logic
10
- * - Liveness monitoring with multi-indicator stuck detection
11
- *
12
- * State machine: idle → evaluating → building_context → executing → idle
13
- */
14
-
15
- const { findMatchingTrigger, evaluateTrigger } = require('./agent-trigger-evaluator');
16
- const { executeHook } = require('./agent-hook-executor');
17
- const {
18
- analyzeProcessHealth,
19
- isPlatformSupported,
20
- STUCK_THRESHOLD,
21
- } = require('./agent-stuck-detector');
22
-
23
- /**
24
- * Start the agent (begin listening for triggers)
25
- * @param {AgentWrapper} agent - Agent instance
26
- */
27
- function start(agent) {
28
- if (agent.running) {
29
- throw new Error(`Agent ${agent.id} is already running`);
30
- }
31
-
32
- agent.running = true;
33
- agent.state = 'idle';
34
-
35
- // Subscribe to all messages for this cluster
36
- agent.unsubscribe = agent.messageBus.subscribe((message) => {
37
- if (message.cluster_id === agent.cluster.id) {
38
- handleMessage(agent, message).catch((error) => {
39
- // FATAL: Message handling failed - crash loud
40
- console.error(`\n${'='.repeat(80)}`);
41
- console.error(`🔴 FATAL: Agent ${agent.id} message handler crashed`);
42
- console.error(`${'='.repeat(80)}`);
43
- console.error(`Topic: ${message.topic}`);
44
- console.error(`Error: ${error.message}`);
45
- console.error(`Stack: ${error.stack}`);
46
- console.error(`${'='.repeat(80)}\n`);
47
- // Re-throw to crash the process - DO NOT SILENTLY CONTINUE
48
- throw error;
49
- });
50
- }
51
- });
52
-
53
- agent._log(`Agent ${agent.id} started (role: ${agent.role})`);
54
- agent._publishLifecycle('STARTED', {
55
- triggers: agent.config.triggers?.map((t) => t.topic) || [],
56
- });
57
- }
58
-
59
- /**
60
- * Stop the agent
61
- * Waits for any in-flight execution to complete before returning.
62
- * @param {AgentWrapper} agent - Agent instance
63
- * @returns {Promise<void>}
64
- */
65
- async function stop(agent) {
66
- if (!agent.running) {
67
- return;
68
- }
69
-
70
- agent.running = false;
71
- agent.state = 'stopped';
72
-
73
- if (agent.unsubscribe) {
74
- agent.unsubscribe();
75
- agent.unsubscribe = null;
76
- }
77
-
78
- // Kill current task if any
79
- if (agent.currentTask) {
80
- agent._killTask();
81
- }
82
-
83
- // Wait for in-flight execution to complete (up to 5 seconds)
84
- // This prevents write-after-close race conditions
85
- if (agent._currentExecution) {
86
- try {
87
- await Promise.race([
88
- agent._currentExecution,
89
- new Promise((resolve) => setTimeout(resolve, 5000)),
90
- ]);
91
- } catch {
92
- // Ignore errors from cancelled execution
93
- }
94
- agent._currentExecution = null;
95
- }
96
-
97
- agent._log(`Agent ${agent.id} stopped`);
98
- }
99
-
100
- /**
101
- * Handle incoming message
102
- * @param {AgentWrapper} agent - Agent instance
103
- * @param {Object} message - Incoming message
104
- */
105
- async function handleMessage(agent, message) {
106
- // Check if any trigger matches FIRST (before state check)
107
- const matchingTrigger = findMatchingTrigger({
108
- triggers: agent.config.triggers,
109
- message,
110
- });
111
-
112
- if (!matchingTrigger) {
113
- return; // No trigger for this message type
114
- }
115
-
116
- // Now check state - LOG if we're dropping a message we SHOULD handle
117
- if (!agent.running) {
118
- console.warn(`[${agent.id}] ⚠️ DROPPING message (not running): ${message.topic}`);
119
- return;
120
- }
121
- if (agent.state !== 'idle') {
122
- console.warn(
123
- `[${agent.id}] ⚠️ DROPPING message (busy, state=${agent.state}): ${message.topic}`
124
- );
125
- return;
126
- }
127
-
128
- // Evaluate trigger logic
129
- agent.state = 'evaluating_logic';
130
-
131
- const agentContext = {
132
- id: agent.id,
133
- role: agent.role,
134
- iteration: agent.iteration,
135
- cluster_id: agent.cluster.id,
136
- };
137
-
138
- const shouldExecute = evaluateTrigger({
139
- trigger: matchingTrigger,
140
- message,
141
- agent: agentContext,
142
- logicEngine: agent.logicEngine,
143
- });
144
-
145
- if (!shouldExecute) {
146
- agent.state = 'idle';
147
- return;
148
- }
149
-
150
- // Execute trigger action (lifecycle event published inside for execute_task)
151
- // Track execution so stop() can wait for it
152
- const executionPromise = executeTriggerAction(agent, matchingTrigger, message);
153
- agent._currentExecution = executionPromise;
154
- try {
155
- await executionPromise;
156
- } finally {
157
- // Clear only if this is still our execution (not replaced by another)
158
- if (agent._currentExecution === executionPromise) {
159
- agent._currentExecution = null;
160
- }
161
- }
162
- }
163
-
164
- /**
165
- * Execute trigger action
166
- * @param {AgentWrapper} agent - Agent instance
167
- * @param {Object} trigger - Matched trigger config
168
- * @param {Object} message - Triggering message
169
- */
170
- async function executeTriggerAction(agent, trigger, message) {
171
- const action = trigger.action || 'execute_task';
172
-
173
- if (action === 'execute_task') {
174
- await executeTask(agent, message);
175
- } else if (action === 'stop_cluster') {
176
- // Publish CLUSTER_COMPLETE message to signal successful completion
177
- agent._publish({
178
- topic: 'CLUSTER_COMPLETE',
179
- receiver: 'system',
180
- content: {
181
- text: 'All validation passed. Cluster completing successfully.',
182
- data: {
183
- reason: 'all_validators_approved',
184
- timestamp: Date.now(),
185
- },
186
- },
187
- });
188
- agent.state = 'completed';
189
- agent._log(`Agent ${agent.id}: Cluster completion triggered`);
190
- } else {
191
- console.warn(`Unknown action: ${action}`);
192
- agent.state = 'idle';
193
- }
194
- }
195
-
196
- /**
197
- * Execute claude-zeroshots with built context
198
- * Retries disabled by default. Set agent config `maxRetries` to enable (e.g., 3).
199
- * @param {AgentWrapper} agent - Agent instance
200
- * @param {Object} triggeringMessage - Message that triggered execution
201
- */
202
- function handleMaxIterations(agent) {
203
- if (agent.iteration < agent.maxIterations) {
204
- return false;
205
- }
206
-
207
- agent._log(`[Agent ${agent.id}] Hit max iterations (${agent.maxIterations}), stopping cluster`);
208
- agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
209
- iteration: agent.iteration,
210
- maxIterations: agent.maxIterations,
211
- });
212
- // Publish failure message - orchestrator watches for this and auto-stops
213
- agent._publish({
214
- topic: 'CLUSTER_FAILED',
215
- receiver: 'system',
216
- content: {
217
- text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
218
- data: {
219
- reason: 'max_iterations',
220
- iteration: agent.iteration,
221
- maxIterations: agent.maxIterations,
222
- },
223
- },
224
- });
225
- agent.state = 'failed';
226
- return true;
227
- }
228
-
229
- function logInputContext(agent, context) {
230
- if (agent.quiet) {
231
- return;
232
- }
233
-
234
- console.log(`
235
- ${'='.repeat(80)}`);
236
- console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
237
- console.log(`${'='.repeat(80)}`);
238
- console.log(context);
239
- console.log(`${'='.repeat(80)}
240
- `);
241
- }
242
-
243
- async function applyValidatorJitter(agent) {
244
- // LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
245
- // When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
246
- // they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
247
- // per workspace, so only one can run. Adding jitter staggers their starts.
248
- // SKIP in testMode - tests use mocks and don't need jitter
249
- if (agent.role !== 'validator' || agent.testMode) {
250
- return;
251
- }
252
-
253
- const jitterMs = Math.floor(Math.random() * 15000); // 0-15 seconds
254
- if (!agent.quiet) {
255
- agent._log(
256
- `[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`
257
- );
258
- }
259
- await new Promise((resolve) => setTimeout(resolve, jitterMs));
260
- }
261
-
262
- function publishTaskStarted(agent, triggeringMessage) {
263
- const modelSpec = agent._resolveModelSpec ? agent._resolveModelSpec() : null;
264
- agent._publishLifecycle('TASK_STARTED', {
265
- iteration: agent.iteration,
266
- model: agent._selectModel(),
267
- provider: agent._resolveProvider ? agent._resolveProvider() : 'claude',
268
- modelSpec,
269
- triggeredBy: triggeringMessage.topic,
270
- triggerFrom: triggeringMessage.sender,
271
- });
272
- }
273
-
274
- function attachResultMetadata(agent, result) {
275
- // Add task ID to result for debugging and hooks
276
- result.taskId = agent.currentTaskId;
277
- result.agentId = agent.id;
278
- result.iteration = agent.iteration;
279
- }
280
-
281
- function publishTaskCompleted(agent, result) {
282
- agent._publishLifecycle('TASK_COMPLETED', {
283
- iteration: agent.iteration,
284
- success: true,
285
- taskId: agent.currentTaskId,
286
- tokenUsage: result.tokenUsage || null,
287
- });
288
- }
289
-
290
- function publishTokenUsage(agent, result) {
291
- // Publish TOKEN_USAGE event for aggregation and tracking
292
- // CRITICAL: Include taskId for causal linking - allows consumers to group
293
- // messages by task regardless of interleaved timing from async hooks
294
- if (!result.tokenUsage) {
295
- return;
296
- }
297
-
298
- agent.messageBus.publish({
299
- cluster_id: agent.cluster.id,
300
- topic: 'TOKEN_USAGE',
301
- sender: agent.id,
302
- content: {
303
- text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
304
- data: {
305
- agentId: agent.id,
306
- role: agent.role,
307
- model: agent._selectModel(),
308
- iteration: agent.iteration,
309
- taskId: agent.currentTaskId, // Causal linking for message ordering
310
- ...result.tokenUsage,
311
- },
312
- },
313
- });
314
- }
315
-
316
- async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
317
- // Execute onComplete hook WITH RETRY
318
- // Hook failure shouldn't retry the entire task - just the hook
319
- const hookMaxRetries = 3;
320
- const hookBaseDelay = 1000;
321
- let hookSuccess = false;
322
-
323
- for (let hookAttempt = 1; hookAttempt <= hookMaxRetries && !hookSuccess; hookAttempt++) {
324
- try {
325
- await executeHook({
326
- hook: agent.config.hooks?.onComplete,
327
- agent: agent,
328
- message: triggeringMessage,
329
- result: result,
330
- messageBus: agent.messageBus,
331
- cluster: agent.cluster,
332
- orchestrator: agent.orchestrator,
333
- });
334
- hookSuccess = true;
335
- } catch (hookError) {
336
- console.error(`
337
- ${'='.repeat(80)}`);
338
- console.error(
339
- `🔴 HOOK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${hookAttempt}/${hookMaxRetries})`
340
- );
341
- console.error(`${'='.repeat(80)}`);
342
- console.error(`Error: ${hookError.message}`);
343
-
344
- if (hookAttempt < hookMaxRetries) {
345
- const delay = hookBaseDelay * Math.pow(2, hookAttempt - 1);
346
- console.error(`Will retry hook in ${delay}ms...`);
347
- console.error(`${'='.repeat(80)}
348
- `);
349
- await new Promise((resolve) => setTimeout(resolve, delay));
350
- } else {
351
- console.error(`${'='.repeat(80)}
352
- `);
353
- // All hook retries exhausted - throw to trigger task-level handling
354
- throw new Error(
355
- `Hook execution failed after ${hookMaxRetries} attempts. ` +
356
- `Task completed successfully but hook could not publish result. ` +
357
- `Original error: ${hookError.message}`
358
- );
359
- }
360
- }
361
- }
362
- }
363
-
364
- async function runTaskAttempt(agent, triggeringMessage) {
365
- // Execute onStart hook
366
- await executeHook({
367
- hook: agent.config.hooks?.onStart,
368
- agent: agent,
369
- message: triggeringMessage,
370
- result: undefined,
371
- messageBus: agent.messageBus,
372
- cluster: agent.cluster,
373
- orchestrator: agent.orchestrator,
374
- });
375
-
376
- // Check max iterations limit BEFORE incrementing (prevents infinite rejection loops)
377
- if (handleMaxIterations(agent)) {
378
- return;
379
- }
380
-
381
- // Increment iteration BEFORE building context so worker knows current iteration
382
- agent.iteration++;
383
-
384
- // Build context
385
- agent.state = 'building_context';
386
- const context = agent._buildContext(triggeringMessage);
387
-
388
- // Log input context (helps debug what each agent sees)
389
- logInputContext(agent, context);
390
-
391
- // Spawn provider task
392
- agent.state = 'executing_task';
393
- await applyValidatorJitter(agent);
394
- publishTaskStarted(agent, triggeringMessage);
395
-
396
- const result = await agent._spawnClaudeTask(context);
397
- attachResultMetadata(agent, result);
398
-
399
- // Check if task execution failed
400
- if (!result.success) {
401
- throw new Error(result.error || 'Task execution failed');
402
- }
403
-
404
- // Set state to idle BEFORE publishing lifecycle event
405
- // (so lifecycle message includes correct state)
406
- agent.state = 'idle';
407
-
408
- // Track completion time for context filtering (used by "since: last_task_end")
409
- agent.lastTaskEndTime = Date.now();
410
-
411
- publishTaskCompleted(agent, result);
412
- publishTokenUsage(agent, result);
413
- await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
414
- }
415
-
416
- function logTaskAttemptFailure(agent, attempt, maxRetries, error) {
417
- // Log attempt failure
418
- console.error(`
419
- ${'='.repeat(80)}`);
420
- console.error(`🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`);
421
- console.error(`${'='.repeat(80)}`);
422
- console.error(`Error: ${error.message}`);
423
- }
424
-
425
- async function handleLockContention() {
426
- // Lock contention - add significant jittered delay
427
- const lockDelay = 10000 + Math.floor(Math.random() * 20000); // 10-30 seconds
428
- console.error(
429
- `⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`
430
- );
431
- await new Promise((resolve) => setTimeout(resolve, lockDelay));
432
- }
433
-
434
- async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
435
- console.error(`
436
- ${'='.repeat(80)}`);
437
- console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
438
- console.error(`${'='.repeat(80)}`);
439
- console.error(`All ${maxRetries} attempts failed`);
440
- console.error(`Final error: ${error.message}`);
441
- console.error(`Stack: ${error.stack}`);
442
- console.error(`${'='.repeat(80)}
443
- `);
444
-
445
- // CRITICAL FIX: Validator crash = REJECTION (not auto-approval)
446
- // Auto-approval on crash allowed broken code to be merged - unacceptable!
447
- // If validator crashed 3x, something is fundamentally wrong - REJECT and investigate
448
- if (agent.role === 'validator') {
449
- console.error(`
450
- ${'='.repeat(80)}`);
451
- console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
452
- console.error(`${'='.repeat(80)}`);
453
- console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
454
- console.error(`Error: ${error.message}`);
455
- console.error(`REJECTING validation - broken code will NOT be merged`);
456
- console.error(`Investigation required before retry`);
457
- console.error(`${'='.repeat(80)}
458
- `);
459
-
460
- // Publish REJECTION message (NOT approval!)
461
- const hook = agent.config.hooks?.onComplete;
462
- if (hook && hook.action === 'publish_message') {
463
- agent._publish({
464
- topic: hook.config.topic,
465
- receiver: hook.config.receiver || 'broadcast',
466
- content: {
467
- text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
468
- data: {
469
- approved: false, // REJECT!
470
- crashedAfterRetries: true,
471
- errors: JSON.stringify([
472
- `VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
473
- `Validation could not be performed - REJECTING to prevent broken code merge`,
474
- `Investigation required before retry`,
475
- ]),
476
- attempts: maxRetries,
477
- requiresInvestigation: true,
478
- },
479
- },
480
- });
481
- }
482
-
483
- agent.state = 'error';
484
- // Don't return - fall through to publish AGENT_ERROR and save failure info
485
- // This allows the cluster to stop and be resumed after investigation
486
- }
487
-
488
- // Non-validator agents: publish error and stop
489
- agent.state = 'error';
490
-
491
- // Save failure info to cluster for resume capability
492
- agent.cluster.failureInfo = {
493
- agentId: agent.id,
494
- taskId: agent.currentTaskId,
495
- iteration: agent.iteration,
496
- error: error.message,
497
- attempts: maxRetries,
498
- timestamp: Date.now(),
499
- };
500
-
501
- // Publish error to message bus for visibility in logs
502
- agent._publish({
503
- topic: 'AGENT_ERROR',
504
- receiver: 'broadcast',
505
- content: {
506
- text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
507
- data: {
508
- error: error.message,
509
- stack: error.stack,
510
- agent: agent.id,
511
- role: agent.role,
512
- iteration: agent.iteration,
513
- taskId: agent.currentTaskId,
514
- attempts: maxRetries,
515
- hookFailureContext: error.message.includes('Hook uses result')
516
- ? {
517
- taskId: agent.currentTaskId || 'UNKNOWN',
518
- retrieveLogs: agent.currentTaskId
519
- ? `zeroshot task logs ${agent.currentTaskId}`
520
- : 'N/A',
521
- }
522
- : undefined,
523
- },
524
- },
525
- metadata: {
526
- triggeringTopic: triggeringMessage.topic,
527
- },
528
- });
529
-
530
- // Execute onError hook
531
- await executeHook({
532
- hook: agent.config.hooks?.onError,
533
- agent: agent,
534
- message: triggeringMessage,
535
- result: { error },
536
- messageBus: agent.messageBus,
537
- cluster: agent.cluster,
538
- orchestrator: agent.orchestrator,
539
- });
540
-
541
- agent.state = 'idle';
542
- }
543
-
544
- async function scheduleRetry(agent, error, attempt, maxRetries, baseDelay) {
545
- const delay = baseDelay * Math.pow(2, attempt - 1); // 2s, 4s, 8s
546
-
547
- agent._publishLifecycle('RETRY_SCHEDULED', {
548
- attempt,
549
- maxRetries,
550
- delayMs: delay,
551
- error: error.message,
552
- });
553
-
554
- agent._log(`[${agent.id}] ⚠️ Retrying in ${delay}ms... (${attempt + 1}/${maxRetries})`);
555
-
556
- // Exponential backoff
557
- await new Promise((resolve) => setTimeout(resolve, delay));
558
-
559
- agent._log(`[${agent.id}] 🔄 Starting retry attempt ${attempt + 1}/${maxRetries}`);
560
- }
561
-
562
- async function handleTaskAttemptFailure({
563
- agent,
564
- triggeringMessage,
565
- error,
566
- attempt,
567
- maxRetries,
568
- baseDelay,
569
- }) {
570
- // LOCK CONTENTION: Add extra jittered delay for lock file errors
571
- // This happens when multiple validators try to run Claude CLI in the same workspace
572
- const isLockError = error.message && error.message.includes('Lock file');
573
-
574
- logTaskAttemptFailure(agent, attempt, maxRetries, error);
575
-
576
- if (isLockError) {
577
- await handleLockContention();
578
- } else if (attempt < maxRetries) {
579
- console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
580
- }
581
- console.error(`${'='.repeat(80)}
582
- `);
583
-
584
- if (attempt >= maxRetries) {
585
- await handleFinalFailure(agent, triggeringMessage, error, maxRetries);
586
- return true;
587
- }
588
-
589
- await scheduleRetry(agent, error, attempt, maxRetries, baseDelay);
590
- return false;
591
- }
592
-
593
- /**
594
- * Execute claude-zeroshots with built context
595
- * Retries disabled by default. Set agent config `maxRetries` to enable (e.g., 3).
596
- * @param {AgentWrapper} agent - Agent instance
597
- * @param {Object} triggeringMessage - Message that triggered execution
598
- */
599
- async function executeTask(agent, triggeringMessage) {
600
- // Early exit if agent was stopped
601
- if (!agent.running) {
602
- return;
603
- }
604
-
605
- // Default: no retries (maxRetries=1 means 1 attempt only)
606
- // Set agent config `maxRetries: 3` to enable exponential backoff retries
607
- const maxRetries = agent.config.maxRetries ?? 1;
608
- const baseDelay = 2000; // 2 seconds
609
-
610
- for (let attempt = 1; attempt <= maxRetries; attempt++) {
611
- // Check if agent was stopped between retries
612
- if (!agent.running) {
613
- return;
614
- }
615
-
616
- try {
617
- await runTaskAttempt(agent, triggeringMessage);
618
- return;
619
- } catch (error) {
620
- const shouldStop = await handleTaskAttemptFailure({
621
- agent,
622
- triggeringMessage,
623
- error,
624
- attempt,
625
- maxRetries,
626
- baseDelay,
627
- });
628
- if (shouldStop) {
629
- return;
630
- }
631
- }
632
- }
633
- }
634
- function startLivenessCheck(agent) {
635
- if (agent.livenessCheckInterval) {
636
- clearInterval(agent.livenessCheckInterval);
637
- }
638
-
639
- // Check if platform supports /proc filesystem (Linux only)
640
- if (!isPlatformSupported()) {
641
- agent._log(
642
- `[${agent.id}] Liveness check disabled: /proc filesystem not available (non-Linux platform)`
643
- );
644
- return;
645
- }
646
-
647
- // Check every 60 seconds (gives time for multi-indicator analysis)
648
- const CHECK_INTERVAL_MS = 60 * 1000;
649
- const ANALYSIS_SAMPLE_MS = 5000; // Sample CPU/context switches over 5 seconds
650
-
651
- agent.livenessCheckInterval = setInterval(async () => {
652
- // Skip if no task running or no PID tracked
653
- if (!agent.currentTask || !agent.processPid) {
654
- return;
655
- }
656
-
657
- // Skip if output is recent (process is clearly active)
658
- if (agent.lastOutputTime) {
659
- const timeSinceLastOutput = Date.now() - agent.lastOutputTime;
660
- if (timeSinceLastOutput < agent.staleDuration) {
661
- return; // Output is recent, definitely not stuck
662
- }
663
- }
664
-
665
- // Output is stale - run multi-indicator analysis to confirm
666
- agent._log(
667
- `[${agent.id}] Output stale for ${Math.round((Date.now() - (agent.lastOutputTime || 0)) / 1000)}s, running multi-indicator analysis...`
668
- );
669
-
670
- try {
671
- const analysis = await analyzeProcessHealth(agent.processPid, ANALYSIS_SAMPLE_MS);
672
-
673
- // Process died during analysis
674
- if (analysis.isLikelyStuck === null) {
675
- agent._log(`[${agent.id}] Process analysis inconclusive: ${analysis.reason}`);
676
- return;
677
- }
678
-
679
- // Log analysis details for debugging
680
- agent._log(
681
- `[${agent.id}] Analysis: score=${analysis.stuckScore}/${STUCK_THRESHOLD}, ` +
682
- `state=${analysis.state}, wchan=${analysis.wchan}, ` +
683
- `CPU=${analysis.cpuPercent}%, ctxSwitches=${analysis.ctxSwitchesDelta}`
684
- );
685
-
686
- if (analysis.isLikelyStuck) {
687
- agent._log(`⚠️ Agent ${agent.id}: CONFIRMED STUCK (confidence: ${analysis.confidence})`);
688
- agent._log(` ${analysis.analysis}`);
689
-
690
- // CHANGED: Stale detection is informational only - never kills tasks
691
- // Publish stale detection event with full analysis (for logging/monitoring)
692
- agent._publishLifecycle('AGENT_STALE_WARNING', {
693
- timeSinceLastOutput: Date.now() - (agent.lastOutputTime || 0),
694
- staleDuration: agent.staleDuration,
695
- lastOutputTime: agent.lastOutputTime,
696
- // Multi-indicator analysis results
697
- stuckScore: analysis.stuckScore,
698
- confidence: analysis.confidence,
699
- processState: analysis.state,
700
- wchan: analysis.wchan,
701
- cpuPercent: analysis.cpuPercent,
702
- ctxSwitchesDelta: analysis.ctxSwitchesDelta,
703
- indicators: analysis.indicators,
704
- analysis: analysis.analysis,
705
- });
706
-
707
- // Keep monitoring - do NOT stop the agent
708
- // User can manually intervene with 'zeroshot resume' if needed
709
- // stopLivenessCheck(agent); // REMOVED - keep monitoring
710
- } else {
711
- agent._log(
712
- `[${agent.id}] Process appears WORKING despite stale output (score: ${analysis.stuckScore})`
713
- );
714
- agent._log(` ${analysis.analysis}`);
715
- // Don't flag as stuck - process is legitimately working
716
- }
717
- } catch (err) {
718
- agent._log(`[${agent.id}] Error during stuck analysis: ${err.message}`);
719
- // Don't flag as stuck on analysis error
720
- }
721
- }, CHECK_INTERVAL_MS);
722
- }
723
-
724
- /**
725
- * Stop liveness monitoring
726
- * @param {AgentWrapper} agent - Agent instance
727
- */
728
- function stopLivenessCheck(agent) {
729
- if (agent.livenessCheckInterval) {
730
- clearInterval(agent.livenessCheckInterval);
731
- agent.livenessCheckInterval = null;
732
- }
733
- }
734
-
735
- module.exports = {
736
- start,
737
- stop,
738
- handleMessage,
739
- executeTriggerAction,
740
- executeTask,
741
- startLivenessCheck,
742
- stopLivenessCheck,
743
- };