@covibes/zeroshot 5.2.1 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +174 -189
  2. package/README.md +226 -195
  3. package/cli/commands/providers.js +149 -0
  4. package/cli/index.js +3145 -2366
  5. package/cli/lib/first-run.js +40 -3
  6. package/cli/message-formatters-normal.js +28 -6
  7. package/cluster-templates/base-templates/debug-workflow.json +24 -78
  8. package/cluster-templates/base-templates/full-workflow.json +99 -316
  9. package/cluster-templates/base-templates/single-worker.json +23 -15
  10. package/cluster-templates/base-templates/worker-validator.json +105 -36
  11. package/cluster-templates/conductor-bootstrap.json +9 -7
  12. package/lib/docker-config.js +14 -1
  13. package/lib/git-remote-utils.js +165 -0
  14. package/lib/id-detector.js +10 -7
  15. package/lib/provider-defaults.js +62 -0
  16. package/lib/provider-detection.js +59 -0
  17. package/lib/provider-names.js +57 -0
  18. package/lib/settings/claude-auth.js +78 -0
  19. package/lib/settings.js +298 -15
  20. package/lib/stream-json-parser.js +4 -238
  21. package/package.json +27 -6
  22. package/scripts/setup-merge-queue.sh +170 -0
  23. package/scripts/validate-templates.js +100 -0
  24. package/src/agent/agent-config.js +140 -63
  25. package/src/agent/agent-context-builder.js +336 -165
  26. package/src/agent/agent-hook-executor.js +337 -67
  27. package/src/agent/agent-lifecycle.js +386 -287
  28. package/src/agent/agent-stuck-detector.js +7 -7
  29. package/src/agent/agent-task-executor.js +944 -683
  30. package/src/agent/output-extraction.js +217 -0
  31. package/src/agent/output-reformatter.js +175 -0
  32. package/src/agent/schema-utils.js +146 -0
  33. package/src/agent-wrapper.js +112 -31
  34. package/src/agents/git-pusher-template.js +285 -0
  35. package/src/claude-task-runner.js +145 -44
  36. package/src/config-router.js +13 -13
  37. package/src/config-validator.js +1049 -563
  38. package/src/input-helpers.js +65 -0
  39. package/src/isolation-manager.js +499 -320
  40. package/src/issue-providers/README.md +305 -0
  41. package/src/issue-providers/azure-devops-provider.js +273 -0
  42. package/src/issue-providers/base-provider.js +232 -0
  43. package/src/issue-providers/github-provider.js +179 -0
  44. package/src/issue-providers/gitlab-provider.js +241 -0
  45. package/src/issue-providers/index.js +196 -0
  46. package/src/issue-providers/jira-provider.js +239 -0
  47. package/src/ledger.js +50 -11
  48. package/src/lib/safe-exec.js +88 -0
  49. package/src/orchestrator.js +1348 -757
  50. package/src/preflight.js +306 -149
  51. package/src/process-metrics.js +98 -56
  52. package/src/providers/anthropic/cli-builder.js +73 -0
  53. package/src/providers/anthropic/index.js +204 -0
  54. package/src/providers/anthropic/models.js +23 -0
  55. package/src/providers/anthropic/output-parser.js +177 -0
  56. package/src/providers/base-provider.js +251 -0
  57. package/src/providers/capabilities.js +60 -0
  58. package/src/providers/google/cli-builder.js +55 -0
  59. package/src/providers/google/index.js +116 -0
  60. package/src/providers/google/models.js +24 -0
  61. package/src/providers/google/output-parser.js +101 -0
  62. package/src/providers/index.js +91 -0
  63. package/src/providers/openai/cli-builder.js +133 -0
  64. package/src/providers/openai/index.js +136 -0
  65. package/src/providers/openai/models.js +21 -0
  66. package/src/providers/openai/output-parser.js +143 -0
  67. package/src/providers/opencode/cli-builder.js +42 -0
  68. package/src/providers/opencode/index.js +103 -0
  69. package/src/providers/opencode/models.js +55 -0
  70. package/src/providers/opencode/output-parser.js +122 -0
  71. package/src/schemas/sub-cluster.js +68 -39
  72. package/src/status-footer.js +94 -48
  73. package/src/sub-cluster-wrapper.js +92 -36
  74. package/src/task-runner.js +8 -6
  75. package/src/template-resolver.js +12 -9
  76. package/src/tui/data-poller.js +123 -99
  77. package/src/tui/formatters.js +4 -3
  78. package/src/tui/keybindings.js +259 -318
  79. package/src/tui/layout.js +20 -3
  80. package/src/tui/renderer.js +11 -21
  81. package/task-lib/attachable-watcher.js +150 -111
  82. package/task-lib/claude-recovery.js +156 -0
  83. package/task-lib/commands/episodes.js +105 -0
  84. package/task-lib/commands/list.js +3 -3
  85. package/task-lib/commands/logs.js +231 -189
  86. package/task-lib/commands/resume.js +3 -2
  87. package/task-lib/commands/run.js +12 -3
  88. package/task-lib/commands/schedules.js +111 -61
  89. package/task-lib/config.js +0 -2
  90. package/task-lib/runner.js +128 -50
  91. package/task-lib/scheduler.js +3 -3
  92. package/task-lib/store.js +464 -152
  93. package/task-lib/tui/formatters.js +94 -45
  94. package/task-lib/tui/renderer.js +13 -3
  95. package/task-lib/tui.js +73 -32
  96. package/task-lib/watcher.js +157 -100
  97. package/src/agents/git-pusher-agent.json +0 -20
  98. package/src/github.js +0 -103
@@ -12,7 +12,6 @@
12
12
  * State machine: idle → evaluating → building_context → executing → idle
13
13
  */
14
14
 
15
- const { buildContext } = require('./agent-context-builder');
16
15
  const { findMatchingTrigger, evaluateTrigger } = require('./agent-trigger-evaluator');
17
16
  const { executeHook } = require('./agent-hook-executor');
18
17
  const {
@@ -200,338 +199,438 @@ async function executeTriggerAction(agent, trigger, message) {
200
199
  * @param {AgentWrapper} agent - Agent instance
201
200
  * @param {Object} triggeringMessage - Message that triggered execution
202
201
  */
203
- async function executeTask(agent, triggeringMessage) {
204
- // Early exit if agent was stopped
205
- if (!agent.running) {
202
+ function handleMaxIterations(agent) {
203
+ if (agent.iteration < agent.maxIterations) {
204
+ return false;
205
+ }
206
+
207
+ agent._log(`[Agent ${agent.id}] Hit max iterations (${agent.maxIterations}), stopping cluster`);
208
+ agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
209
+ iteration: agent.iteration,
210
+ maxIterations: agent.maxIterations,
211
+ });
212
+ // Publish failure message - orchestrator watches for this and auto-stops
213
+ agent._publish({
214
+ topic: 'CLUSTER_FAILED',
215
+ receiver: 'system',
216
+ content: {
217
+ text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
218
+ data: {
219
+ reason: 'max_iterations',
220
+ iteration: agent.iteration,
221
+ maxIterations: agent.maxIterations,
222
+ },
223
+ },
224
+ });
225
+ agent.state = 'failed';
226
+ return true;
227
+ }
228
+
229
+ function logInputContext(agent, context) {
230
+ if (agent.quiet) {
206
231
  return;
207
232
  }
208
233
 
209
- // Default: no retries (maxRetries=1 means 1 attempt only)
210
- // Set agent config `maxRetries: 3` to enable exponential backoff retries
211
- const maxRetries = agent.config.maxRetries ?? 1;
212
- const baseDelay = 2000; // 2 seconds
234
+ console.log(`
235
+ ${'='.repeat(80)}`);
236
+ console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
237
+ console.log(`${'='.repeat(80)}`);
238
+ console.log(context);
239
+ console.log(`${'='.repeat(80)}
240
+ `);
241
+ }
213
242
 
214
- for (let attempt = 1; attempt <= maxRetries; attempt++) {
215
- // Check if agent was stopped between retries
216
- if (!agent.running) {
217
- return;
218
- }
243
+ async function applyValidatorJitter(agent) {
244
+ // LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
245
+ // When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
246
+ // they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
247
+ // per workspace, so only one can run. Adding jitter staggers their starts.
248
+ // SKIP in testMode - tests use mocks and don't need jitter
249
+ if (agent.role !== 'validator' || agent.testMode) {
250
+ return;
251
+ }
252
+
253
+ const jitterMs = Math.floor(Math.random() * 15000); // 0-15 seconds
254
+ if (!agent.quiet) {
255
+ agent._log(
256
+ `[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`
257
+ );
258
+ }
259
+ await new Promise((resolve) => setTimeout(resolve, jitterMs));
260
+ }
261
+
262
+ function publishTaskStarted(agent, triggeringMessage) {
263
+ const modelSpec = agent._resolveModelSpec ? agent._resolveModelSpec() : null;
264
+ agent._publishLifecycle('TASK_STARTED', {
265
+ iteration: agent.iteration,
266
+ model: agent._selectModel(),
267
+ provider: agent._resolveProvider ? agent._resolveProvider() : 'claude',
268
+ modelSpec,
269
+ triggeredBy: triggeringMessage.topic,
270
+ triggerFrom: triggeringMessage.sender,
271
+ });
272
+ }
273
+
274
+ function attachResultMetadata(agent, result) {
275
+ // Add task ID to result for debugging and hooks
276
+ result.taskId = agent.currentTaskId;
277
+ result.agentId = agent.id;
278
+ result.iteration = agent.iteration;
279
+ }
280
+
281
+ function publishTaskCompleted(agent, result) {
282
+ agent._publishLifecycle('TASK_COMPLETED', {
283
+ iteration: agent.iteration,
284
+ success: true,
285
+ taskId: agent.currentTaskId,
286
+ tokenUsage: result.tokenUsage || null,
287
+ });
288
+ }
219
289
 
290
+ function publishTokenUsage(agent, result) {
291
+ // Publish TOKEN_USAGE event for aggregation and tracking
292
+ // CRITICAL: Include taskId for causal linking - allows consumers to group
293
+ // messages by task regardless of interleaved timing from async hooks
294
+ if (!result.tokenUsage) {
295
+ return;
296
+ }
297
+
298
+ agent.messageBus.publish({
299
+ cluster_id: agent.cluster.id,
300
+ topic: 'TOKEN_USAGE',
301
+ sender: agent.id,
302
+ content: {
303
+ text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
304
+ data: {
305
+ agentId: agent.id,
306
+ role: agent.role,
307
+ model: agent._selectModel(),
308
+ iteration: agent.iteration,
309
+ taskId: agent.currentTaskId, // Causal linking for message ordering
310
+ ...result.tokenUsage,
311
+ },
312
+ },
313
+ });
314
+ }
315
+
316
+ async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
317
+ // Execute onComplete hook WITH RETRY
318
+ // Hook failure shouldn't retry the entire task - just the hook
319
+ const hookMaxRetries = 3;
320
+ const hookBaseDelay = 1000;
321
+ let hookSuccess = false;
322
+
323
+ for (let hookAttempt = 1; hookAttempt <= hookMaxRetries && !hookSuccess; hookAttempt++) {
220
324
  try {
221
- // Execute onStart hook
222
325
  await executeHook({
223
- hook: agent.config.hooks?.onStart,
326
+ hook: agent.config.hooks?.onComplete,
224
327
  agent: agent,
225
328
  message: triggeringMessage,
226
- result: undefined,
329
+ result: result,
227
330
  messageBus: agent.messageBus,
228
331
  cluster: agent.cluster,
229
332
  orchestrator: agent.orchestrator,
230
333
  });
231
-
232
- // Check max iterations limit BEFORE incrementing (prevents infinite rejection loops)
233
- if (agent.iteration >= agent.maxIterations) {
234
- agent._log(
235
- `[Agent ${agent.id}] Hit max iterations (${agent.maxIterations}), stopping cluster`
334
+ hookSuccess = true;
335
+ } catch (hookError) {
336
+ console.error(`
337
+ ${'='.repeat(80)}`);
338
+ console.error(
339
+ `🔴 HOOK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${hookAttempt}/${hookMaxRetries})`
340
+ );
341
+ console.error(`${'='.repeat(80)}`);
342
+ console.error(`Error: ${hookError.message}`);
343
+
344
+ if (hookAttempt < hookMaxRetries) {
345
+ const delay = hookBaseDelay * Math.pow(2, hookAttempt - 1);
346
+ console.error(`Will retry hook in ${delay}ms...`);
347
+ console.error(`${'='.repeat(80)}
348
+ `);
349
+ await new Promise((resolve) => setTimeout(resolve, delay));
350
+ } else {
351
+ console.error(`${'='.repeat(80)}
352
+ `);
353
+ // All hook retries exhausted - throw to trigger task-level handling
354
+ throw new Error(
355
+ `Hook execution failed after ${hookMaxRetries} attempts. ` +
356
+ `Task completed successfully but hook could not publish result. ` +
357
+ `Original error: ${hookError.message}`
236
358
  );
237
- agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
238
- iteration: agent.iteration,
239
- maxIterations: agent.maxIterations,
240
- });
241
- // Publish failure message - orchestrator watches for this and auto-stops
242
- agent._publish({
243
- topic: 'CLUSTER_FAILED',
244
- receiver: 'system',
245
- content: {
246
- text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
247
- data: {
248
- reason: 'max_iterations',
249
- iteration: agent.iteration,
250
- maxIterations: agent.maxIterations,
251
- },
252
- },
253
- });
254
- agent.state = 'failed';
255
- return;
256
359
  }
360
+ }
361
+ }
362
+ }
257
363
 
258
- // Increment iteration BEFORE building context so worker knows current iteration
259
- agent.iteration++;
364
+ async function runTaskAttempt(agent, triggeringMessage) {
365
+ // Execute onStart hook
366
+ await executeHook({
367
+ hook: agent.config.hooks?.onStart,
368
+ agent: agent,
369
+ message: triggeringMessage,
370
+ result: undefined,
371
+ messageBus: agent.messageBus,
372
+ cluster: agent.cluster,
373
+ orchestrator: agent.orchestrator,
374
+ });
260
375
 
261
- // Build context
262
- agent.state = 'building_context';
263
- const context = buildContext({
264
- id: agent.id,
265
- role: agent.role,
266
- iteration: agent.iteration,
267
- config: agent.config,
268
- messageBus: agent.messageBus,
269
- cluster: agent.cluster,
270
- lastTaskEndTime: agent.lastTaskEndTime,
271
- triggeringMessage,
272
- selectedPrompt: agent._selectPrompt(),
273
- });
376
+ // Check max iterations limit BEFORE incrementing (prevents infinite rejection loops)
377
+ if (handleMaxIterations(agent)) {
378
+ return;
379
+ }
274
380
 
275
- // Log input context (helps debug what each agent sees)
276
- if (!agent.quiet) {
277
- console.log(`\n${'='.repeat(80)}`);
278
- console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
279
- console.log(`${'='.repeat(80)}`);
280
- console.log(context);
281
- console.log(`${'='.repeat(80)}\n`);
282
- }
381
+ // Increment iteration BEFORE building context so worker knows current iteration
382
+ agent.iteration++;
283
383
 
284
- // Spawn claude-zeroshots
285
- agent.state = 'executing_task';
286
-
287
- // LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
288
- // When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
289
- // they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
290
- // per workspace, so only one can run. Adding jitter staggers their starts.
291
- // SKIP in testMode - tests use mocks and don't need jitter
292
- if (agent.role === 'validator' && !agent.testMode) {
293
- const jitterMs = Math.floor(Math.random() * 15000); // 0-15 seconds
294
- if (!agent.quiet) {
295
- agent._log(`[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`);
296
- }
297
- await new Promise((resolve) => setTimeout(resolve, jitterMs));
298
- }
384
+ // Build context
385
+ agent.state = 'building_context';
386
+ const context = agent._buildContext(triggeringMessage);
299
387
 
300
- agent._publishLifecycle('TASK_STARTED', {
301
- iteration: agent.iteration,
302
- model: agent._selectModel(),
303
- triggeredBy: triggeringMessage.topic,
304
- triggerFrom: triggeringMessage.sender,
305
- });
388
+ // Log input context (helps debug what each agent sees)
389
+ logInputContext(agent, context);
306
390
 
307
- const result = await agent._spawnClaudeTask(context);
391
+ // Spawn provider task
392
+ agent.state = 'executing_task';
393
+ await applyValidatorJitter(agent);
394
+ publishTaskStarted(agent, triggeringMessage);
308
395
 
309
- // Add task ID to result for debugging and hooks
310
- result.taskId = agent.currentTaskId;
311
- result.agentId = agent.id;
312
- result.iteration = agent.iteration;
396
+ const result = await agent._spawnClaudeTask(context);
397
+ attachResultMetadata(agent, result);
313
398
 
314
- // Check if task execution failed
315
- if (!result.success) {
316
- throw new Error(result.error || 'Task execution failed');
317
- }
399
+ // Check if task execution failed
400
+ if (!result.success) {
401
+ throw new Error(result.error || 'Task execution failed');
402
+ }
318
403
 
319
- // Set state to idle BEFORE publishing lifecycle event
320
- // (so lifecycle message includes correct state)
321
- agent.state = 'idle';
404
+ // Set state to idle BEFORE publishing lifecycle event
405
+ // (so lifecycle message includes correct state)
406
+ agent.state = 'idle';
322
407
 
323
- // Track completion time for context filtering (used by "since: last_task_end")
324
- agent.lastTaskEndTime = Date.now();
408
+ // Track completion time for context filtering (used by "since: last_task_end")
409
+ agent.lastTaskEndTime = Date.now();
325
410
 
326
- agent._publishLifecycle('TASK_COMPLETED', {
327
- iteration: agent.iteration,
328
- success: true,
329
- taskId: agent.currentTaskId,
330
- tokenUsage: result.tokenUsage || null,
331
- });
411
+ publishTaskCompleted(agent, result);
412
+ publishTokenUsage(agent, result);
413
+ await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
414
+ }
332
415
 
333
- // Publish TOKEN_USAGE event for aggregation and tracking
334
- // CRITICAL: Include taskId for causal linking - allows consumers to group
335
- // messages by task regardless of interleaved timing from async hooks
336
- if (result.tokenUsage) {
337
- agent.messageBus.publish({
338
- cluster_id: agent.cluster.id,
339
- topic: 'TOKEN_USAGE',
340
- sender: agent.id,
341
- content: {
342
- text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
343
- data: {
344
- agentId: agent.id,
345
- role: agent.role,
346
- model: agent._selectModel(),
347
- iteration: agent.iteration,
348
- taskId: agent.currentTaskId, // Causal linking for message ordering
349
- ...result.tokenUsage,
350
- },
351
- },
352
- });
353
- }
416
+ function logTaskAttemptFailure(agent, attempt, maxRetries, error) {
417
+ // Log attempt failure
418
+ console.error(`
419
+ ${'='.repeat(80)}`);
420
+ console.error(`🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`);
421
+ console.error(`${'='.repeat(80)}`);
422
+ console.error(`Error: ${error.message}`);
423
+ }
354
424
 
355
- // Execute onComplete hook
356
- await executeHook({
357
- hook: agent.config.hooks?.onComplete,
358
- agent: agent,
359
- message: triggeringMessage,
360
- result: result,
361
- messageBus: agent.messageBus,
362
- cluster: agent.cluster,
363
- orchestrator: agent.orchestrator,
425
+ async function handleLockContention() {
426
+ // Lock contention - add significant jittered delay
427
+ const lockDelay = 10000 + Math.floor(Math.random() * 20000); // 10-30 seconds
428
+ console.error(
429
+ `⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`
430
+ );
431
+ await new Promise((resolve) => setTimeout(resolve, lockDelay));
432
+ }
433
+
434
+ async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
435
+ console.error(`
436
+ ${'='.repeat(80)}`);
437
+ console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
438
+ console.error(`${'='.repeat(80)}`);
439
+ console.error(`All ${maxRetries} attempts failed`);
440
+ console.error(`Final error: ${error.message}`);
441
+ console.error(`Stack: ${error.stack}`);
442
+ console.error(`${'='.repeat(80)}
443
+ `);
444
+
445
+ // CRITICAL FIX: Validator crash = REJECTION (not auto-approval)
446
+ // Auto-approval on crash allowed broken code to be merged - unacceptable!
447
+ // If validator crashed 3x, something is fundamentally wrong - REJECT and investigate
448
+ if (agent.role === 'validator') {
449
+ console.error(`
450
+ ${'='.repeat(80)}`);
451
+ console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
452
+ console.error(`${'='.repeat(80)}`);
453
+ console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
454
+ console.error(`Error: ${error.message}`);
455
+ console.error(`REJECTING validation - broken code will NOT be merged`);
456
+ console.error(`Investigation required before retry`);
457
+ console.error(`${'='.repeat(80)}
458
+ `);
459
+
460
+ // Publish REJECTION message (NOT approval!)
461
+ const hook = agent.config.hooks?.onComplete;
462
+ if (hook && hook.action === 'publish_message') {
463
+ agent._publish({
464
+ topic: hook.config.topic,
465
+ receiver: hook.config.receiver || 'broadcast',
466
+ content: {
467
+ text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
468
+ data: {
469
+ approved: false, // REJECT!
470
+ crashedAfterRetries: true,
471
+ errors: JSON.stringify([
472
+ `VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
473
+ `Validation could not be performed - REJECTING to prevent broken code merge`,
474
+ `Investigation required before retry`,
475
+ ]),
476
+ attempts: maxRetries,
477
+ requiresInvestigation: true,
478
+ },
479
+ },
364
480
  });
481
+ }
365
482
 
366
- // SUCCESS - exit retry loop
367
- return;
368
- } catch (error) {
369
- // LOCK CONTENTION: Add extra jittered delay for lock file errors
370
- // This happens when multiple validators try to run Claude CLI in the same workspace
371
- const isLockError = error.message && error.message.includes('Lock file');
483
+ agent.state = 'error';
484
+ // Don't return - fall through to publish AGENT_ERROR and save failure info
485
+ // This allows the cluster to stop and be resumed after investigation
486
+ }
372
487
 
373
- // Log attempt failure
374
- console.error(`\n${'='.repeat(80)}`);
375
- console.error(
376
- `🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`
377
- );
378
- console.error(`${'='.repeat(80)}`);
379
- console.error(`Error: ${error.message}`);
380
-
381
- if (isLockError) {
382
- // Lock contention - add significant jittered delay
383
- const lockDelay = 10000 + Math.floor(Math.random() * 20000); // 10-30 seconds
384
- console.error(`⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`);
385
- await new Promise(resolve => setTimeout(resolve, lockDelay));
386
- } else if (attempt < maxRetries) {
387
- console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
388
- }
389
- console.error(`${'='.repeat(80)}\n`);
488
+ // Non-validator agents: publish error and stop
489
+ agent.state = 'error';
390
490
 
391
- // Last attempt - give up
392
- if (attempt >= maxRetries) {
393
- console.error(`\n${'='.repeat(80)}`);
394
- console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
395
- console.error(`${'='.repeat(80)}`);
396
- console.error(`All ${maxRetries} attempts failed`);
397
- console.error(`Final error: ${error.message}`);
398
- console.error(`Stack: ${error.stack}`);
399
- console.error(`${'='.repeat(80)}\n`);
491
+ // Save failure info to cluster for resume capability
492
+ agent.cluster.failureInfo = {
493
+ agentId: agent.id,
494
+ taskId: agent.currentTaskId,
495
+ iteration: agent.iteration,
496
+ error: error.message,
497
+ attempts: maxRetries,
498
+ timestamp: Date.now(),
499
+ };
400
500
 
401
- // CRITICAL FIX: Validator crash = REJECTION (not auto-approval)
402
- // Auto-approval on crash allowed broken code to be merged - unacceptable!
403
- // If validator crashed 3x, something is fundamentally wrong - REJECT and investigate
404
- if (agent.role === 'validator') {
405
- console.error(`\n${'='.repeat(80)}`);
406
- console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
407
- console.error(`${'='.repeat(80)}`);
408
- console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
409
- console.error(`Error: ${error.message}`);
410
- console.error(`REJECTING validation - broken code will NOT be merged`);
411
- console.error(`Investigation required before retry`);
412
- console.error(`${'='.repeat(80)}\n`);
413
-
414
- // Publish REJECTION message (NOT approval!)
415
- const hook = agent.config.hooks?.onComplete;
416
- if (hook && hook.action === 'publish_message') {
417
- agent._publish({
418
- topic: hook.config.topic,
419
- receiver: hook.config.receiver || 'broadcast',
420
- content: {
421
- text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
422
- data: {
423
- approved: false, // REJECT!
424
- crashedAfterRetries: true,
425
- errors: JSON.stringify([
426
- `VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
427
- `Validation could not be performed - REJECTING to prevent broken code merge`,
428
- `Investigation required before retry`,
429
- ]),
430
- attempts: maxRetries,
431
- requiresInvestigation: true,
432
- },
433
- },
434
- });
435
- }
436
-
437
- agent.state = 'error';
438
- // Don't return - fall through to publish AGENT_ERROR and save failure info
439
- // This allows the cluster to stop and be resumed after investigation
440
- }
441
-
442
- // Non-validator agents: publish error and stop
443
- agent.state = 'error';
444
-
445
- // Save failure info to cluster for resume capability
446
- agent.cluster.failureInfo = {
447
- agentId: agent.id,
448
- taskId: agent.currentTaskId,
449
- iteration: agent.iteration,
450
- error: error.message,
451
- attempts: maxRetries,
452
- timestamp: Date.now(),
453
- };
454
-
455
- // Publish error to message bus for visibility in logs
456
- agent._publish({
457
- topic: 'AGENT_ERROR',
458
- receiver: 'broadcast',
459
- content: {
460
- text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
461
- data: {
462
- error: error.message,
463
- stack: error.stack,
464
- agent: agent.id,
465
- role: agent.role,
466
- iteration: agent.iteration,
467
- taskId: agent.currentTaskId,
468
- attempts: maxRetries,
469
- hookFailureContext: error.message.includes('Hook uses result')
470
- ? {
471
- taskId: agent.currentTaskId || 'UNKNOWN',
472
- retrieveLogs: agent.currentTaskId
473
- ? `zeroshot task logs ${agent.currentTaskId}`
474
- : 'N/A',
475
- }
476
- : undefined,
477
- },
478
- },
479
- metadata: {
480
- triggeringTopic: triggeringMessage.topic,
481
- },
482
- });
501
+ // Publish error to message bus for visibility in logs
502
+ agent._publish({
503
+ topic: 'AGENT_ERROR',
504
+ receiver: 'broadcast',
505
+ content: {
506
+ text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
507
+ data: {
508
+ error: error.message,
509
+ stack: error.stack,
510
+ agent: agent.id,
511
+ role: agent.role,
512
+ iteration: agent.iteration,
513
+ taskId: agent.currentTaskId,
514
+ attempts: maxRetries,
515
+ hookFailureContext: error.message.includes('Hook uses result')
516
+ ? {
517
+ taskId: agent.currentTaskId || 'UNKNOWN',
518
+ retrieveLogs: agent.currentTaskId
519
+ ? `zeroshot task logs ${agent.currentTaskId}`
520
+ : 'N/A',
521
+ }
522
+ : undefined,
523
+ },
524
+ },
525
+ metadata: {
526
+ triggeringTopic: triggeringMessage.topic,
527
+ },
528
+ });
483
529
 
484
- // Execute onError hook
485
- await executeHook({
486
- hook: agent.config.hooks?.onError,
487
- agent: agent,
488
- message: triggeringMessage,
489
- result: { error },
490
- messageBus: agent.messageBus,
491
- cluster: agent.cluster,
492
- orchestrator: agent.orchestrator,
493
- });
530
+ // Execute onError hook
531
+ await executeHook({
532
+ hook: agent.config.hooks?.onError,
533
+ agent: agent,
534
+ message: triggeringMessage,
535
+ result: { error },
536
+ messageBus: agent.messageBus,
537
+ cluster: agent.cluster,
538
+ orchestrator: agent.orchestrator,
539
+ });
494
540
 
495
- agent.state = 'idle';
496
- return; // Give up
497
- }
541
+ agent.state = 'idle';
542
+ }
498
543
 
499
- // Not the last attempt - prepare for retry
500
- const delay = baseDelay * Math.pow(2, attempt - 1); // 2s, 4s, 8s
544
+ async function scheduleRetry(agent, error, attempt, maxRetries, baseDelay) {
545
+ const delay = baseDelay * Math.pow(2, attempt - 1); // 2s, 4s, 8s
501
546
 
502
- agent._publishLifecycle('RETRY_SCHEDULED', {
503
- attempt,
504
- maxRetries,
505
- delayMs: delay,
506
- error: error.message,
507
- });
547
+ agent._publishLifecycle('RETRY_SCHEDULED', {
548
+ attempt,
549
+ maxRetries,
550
+ delayMs: delay,
551
+ error: error.message,
552
+ });
508
553
 
509
- agent._log(`[${agent.id}] ⚠️ Retrying in ${delay}ms... (${attempt + 1}/${maxRetries})`);
554
+ agent._log(`[${agent.id}] ⚠️ Retrying in ${delay}ms... (${attempt + 1}/${maxRetries})`);
510
555
 
511
- // Exponential backoff
512
- await new Promise((resolve) => setTimeout(resolve, delay));
556
+ // Exponential backoff
557
+ await new Promise((resolve) => setTimeout(resolve, delay));
513
558
 
514
- agent._log(`[${agent.id}] 🔄 Starting retry attempt ${attempt + 1}/${maxRetries}`);
515
- // Continue to next iteration of for loop
516
- }
559
+ agent._log(`[${agent.id}] 🔄 Starting retry attempt ${attempt + 1}/${maxRetries}`);
560
+ }
561
+
562
+ async function handleTaskAttemptFailure({
563
+ agent,
564
+ triggeringMessage,
565
+ error,
566
+ attempt,
567
+ maxRetries,
568
+ baseDelay,
569
+ }) {
570
+ // LOCK CONTENTION: Add extra jittered delay for lock file errors
571
+ // This happens when multiple validators try to run Claude CLI in the same workspace
572
+ const isLockError = error.message && error.message.includes('Lock file');
573
+
574
+ logTaskAttemptFailure(agent, attempt, maxRetries, error);
575
+
576
+ if (isLockError) {
577
+ await handleLockContention();
578
+ } else if (attempt < maxRetries) {
579
+ console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
517
580
  }
581
+ console.error(`${'='.repeat(80)}
582
+ `);
583
+
584
+ if (attempt >= maxRetries) {
585
+ await handleFinalFailure(agent, triggeringMessage, error, maxRetries);
586
+ return true;
587
+ }
588
+
589
+ await scheduleRetry(agent, error, attempt, maxRetries, baseDelay);
590
+ return false;
518
591
  }
519
592
 
520
593
  /**
521
- * Start monitoring agent output liveness using multi-indicator stuck detection
522
- *
523
- * SAFE DETECTION: Only flags as stuck when MULTIPLE indicators agree:
524
- * - Process sleeping (state=S)
525
- * - Blocked on epoll/poll wait
526
- * - Low CPU usage (<1%)
527
- * - Low context switches (<10)
528
- * - No network data in flight
529
- *
530
- * Single-indicator detection (just output freshness) has HIGH false positive risk.
531
- * This multi-indicator approach eliminates false positives.
532
- *
594
+ * Execute claude-zeroshots with built context
595
+ * Retries disabled by default. Set agent config `maxRetries` to enable (e.g., 3).
533
596
  * @param {AgentWrapper} agent - Agent instance
597
+ * @param {Object} triggeringMessage - Message that triggered execution
534
598
  */
599
+ async function executeTask(agent, triggeringMessage) {
600
+ // Early exit if agent was stopped
601
+ if (!agent.running) {
602
+ return;
603
+ }
604
+
605
+ // Default: no retries (maxRetries=1 means 1 attempt only)
606
+ // Set agent config `maxRetries: 3` to enable exponential backoff retries
607
+ const maxRetries = agent.config.maxRetries ?? 1;
608
+ const baseDelay = 2000; // 2 seconds
609
+
610
+ for (let attempt = 1; attempt <= maxRetries; attempt++) {
611
+ // Check if agent was stopped between retries
612
+ if (!agent.running) {
613
+ return;
614
+ }
615
+
616
+ try {
617
+ await runTaskAttempt(agent, triggeringMessage);
618
+ return;
619
+ } catch (error) {
620
+ const shouldStop = await handleTaskAttemptFailure({
621
+ agent,
622
+ triggeringMessage,
623
+ error,
624
+ attempt,
625
+ maxRetries,
626
+ baseDelay,
627
+ });
628
+ if (shouldStop) {
629
+ return;
630
+ }
631
+ }
632
+ }
633
+ }
535
634
  function startLivenessCheck(agent) {
536
635
  if (agent.livenessCheckInterval) {
537
636
  clearInterval(agent.livenessCheckInterval);