@covibes/zeroshot 5.2.1 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +174 -189
- package/README.md +226 -195
- package/cli/commands/providers.js +149 -0
- package/cli/index.js +3145 -2366
- package/cli/lib/first-run.js +40 -3
- package/cli/message-formatters-normal.js +28 -6
- package/cluster-templates/base-templates/debug-workflow.json +24 -78
- package/cluster-templates/base-templates/full-workflow.json +99 -316
- package/cluster-templates/base-templates/single-worker.json +23 -15
- package/cluster-templates/base-templates/worker-validator.json +105 -36
- package/cluster-templates/conductor-bootstrap.json +9 -7
- package/lib/docker-config.js +14 -1
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +10 -7
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-detection.js +59 -0
- package/lib/provider-names.js +57 -0
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +298 -15
- package/lib/stream-json-parser.js +4 -238
- package/package.json +27 -6
- package/scripts/setup-merge-queue.sh +170 -0
- package/scripts/validate-templates.js +100 -0
- package/src/agent/agent-config.js +140 -63
- package/src/agent/agent-context-builder.js +336 -165
- package/src/agent/agent-hook-executor.js +337 -67
- package/src/agent/agent-lifecycle.js +386 -287
- package/src/agent/agent-stuck-detector.js +7 -7
- package/src/agent/agent-task-executor.js +944 -683
- package/src/agent/output-extraction.js +217 -0
- package/src/agent/output-reformatter.js +175 -0
- package/src/agent/schema-utils.js +146 -0
- package/src/agent-wrapper.js +112 -31
- package/src/agents/git-pusher-template.js +285 -0
- package/src/claude-task-runner.js +145 -44
- package/src/config-router.js +13 -13
- package/src/config-validator.js +1049 -563
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +499 -320
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +273 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +179 -0
- package/src/issue-providers/gitlab-provider.js +241 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +239 -0
- package/src/ledger.js +50 -11
- package/src/lib/safe-exec.js +88 -0
- package/src/orchestrator.js +1348 -757
- package/src/preflight.js +306 -149
- package/src/process-metrics.js +98 -56
- package/src/providers/anthropic/cli-builder.js +73 -0
- package/src/providers/anthropic/index.js +204 -0
- package/src/providers/anthropic/models.js +23 -0
- package/src/providers/anthropic/output-parser.js +177 -0
- package/src/providers/base-provider.js +251 -0
- package/src/providers/capabilities.js +60 -0
- package/src/providers/google/cli-builder.js +55 -0
- package/src/providers/google/index.js +116 -0
- package/src/providers/google/models.js +24 -0
- package/src/providers/google/output-parser.js +101 -0
- package/src/providers/index.js +91 -0
- package/src/providers/openai/cli-builder.js +133 -0
- package/src/providers/openai/index.js +136 -0
- package/src/providers/openai/models.js +21 -0
- package/src/providers/openai/output-parser.js +143 -0
- package/src/providers/opencode/cli-builder.js +42 -0
- package/src/providers/opencode/index.js +103 -0
- package/src/providers/opencode/models.js +55 -0
- package/src/providers/opencode/output-parser.js +122 -0
- package/src/schemas/sub-cluster.js +68 -39
- package/src/status-footer.js +94 -48
- package/src/sub-cluster-wrapper.js +92 -36
- package/src/task-runner.js +8 -6
- package/src/template-resolver.js +12 -9
- package/src/tui/data-poller.js +123 -99
- package/src/tui/formatters.js +4 -3
- package/src/tui/keybindings.js +259 -318
- package/src/tui/layout.js +20 -3
- package/src/tui/renderer.js +11 -21
- package/task-lib/attachable-watcher.js +150 -111
- package/task-lib/claude-recovery.js +156 -0
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/list.js +3 -3
- package/task-lib/commands/logs.js +231 -189
- package/task-lib/commands/resume.js +3 -2
- package/task-lib/commands/run.js +12 -3
- package/task-lib/commands/schedules.js +111 -61
- package/task-lib/config.js +0 -2
- package/task-lib/runner.js +128 -50
- package/task-lib/scheduler.js +3 -3
- package/task-lib/store.js +464 -152
- package/task-lib/tui/formatters.js +94 -45
- package/task-lib/tui/renderer.js +13 -3
- package/task-lib/tui.js +73 -32
- package/task-lib/watcher.js +157 -100
- package/src/agents/git-pusher-agent.json +0 -20
- package/src/github.js +0 -103
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
* State machine: idle → evaluating → building_context → executing → idle
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
const { buildContext } = require('./agent-context-builder');
|
|
16
15
|
const { findMatchingTrigger, evaluateTrigger } = require('./agent-trigger-evaluator');
|
|
17
16
|
const { executeHook } = require('./agent-hook-executor');
|
|
18
17
|
const {
|
|
@@ -200,338 +199,438 @@ async function executeTriggerAction(agent, trigger, message) {
|
|
|
200
199
|
* @param {AgentWrapper} agent - Agent instance
|
|
201
200
|
* @param {Object} triggeringMessage - Message that triggered execution
|
|
202
201
|
*/
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
202
|
+
function handleMaxIterations(agent) {
|
|
203
|
+
if (agent.iteration < agent.maxIterations) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
agent._log(`[Agent ${agent.id}] Hit max iterations (${agent.maxIterations}), stopping cluster`);
|
|
208
|
+
agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
|
|
209
|
+
iteration: agent.iteration,
|
|
210
|
+
maxIterations: agent.maxIterations,
|
|
211
|
+
});
|
|
212
|
+
// Publish failure message - orchestrator watches for this and auto-stops
|
|
213
|
+
agent._publish({
|
|
214
|
+
topic: 'CLUSTER_FAILED',
|
|
215
|
+
receiver: 'system',
|
|
216
|
+
content: {
|
|
217
|
+
text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
|
|
218
|
+
data: {
|
|
219
|
+
reason: 'max_iterations',
|
|
220
|
+
iteration: agent.iteration,
|
|
221
|
+
maxIterations: agent.maxIterations,
|
|
222
|
+
},
|
|
223
|
+
},
|
|
224
|
+
});
|
|
225
|
+
agent.state = 'failed';
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function logInputContext(agent, context) {
|
|
230
|
+
if (agent.quiet) {
|
|
206
231
|
return;
|
|
207
232
|
}
|
|
208
233
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
234
|
+
console.log(`
|
|
235
|
+
${'='.repeat(80)}`);
|
|
236
|
+
console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
|
|
237
|
+
console.log(`${'='.repeat(80)}`);
|
|
238
|
+
console.log(context);
|
|
239
|
+
console.log(`${'='.repeat(80)}
|
|
240
|
+
`);
|
|
241
|
+
}
|
|
213
242
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
243
|
+
async function applyValidatorJitter(agent) {
|
|
244
|
+
// LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
|
|
245
|
+
// When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
|
|
246
|
+
// they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
|
|
247
|
+
// per workspace, so only one can run. Adding jitter staggers their starts.
|
|
248
|
+
// SKIP in testMode - tests use mocks and don't need jitter
|
|
249
|
+
if (agent.role !== 'validator' || agent.testMode) {
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const jitterMs = Math.floor(Math.random() * 15000); // 0-15 seconds
|
|
254
|
+
if (!agent.quiet) {
|
|
255
|
+
agent._log(
|
|
256
|
+
`[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
await new Promise((resolve) => setTimeout(resolve, jitterMs));
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function publishTaskStarted(agent, triggeringMessage) {
|
|
263
|
+
const modelSpec = agent._resolveModelSpec ? agent._resolveModelSpec() : null;
|
|
264
|
+
agent._publishLifecycle('TASK_STARTED', {
|
|
265
|
+
iteration: agent.iteration,
|
|
266
|
+
model: agent._selectModel(),
|
|
267
|
+
provider: agent._resolveProvider ? agent._resolveProvider() : 'claude',
|
|
268
|
+
modelSpec,
|
|
269
|
+
triggeredBy: triggeringMessage.topic,
|
|
270
|
+
triggerFrom: triggeringMessage.sender,
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function attachResultMetadata(agent, result) {
|
|
275
|
+
// Add task ID to result for debugging and hooks
|
|
276
|
+
result.taskId = agent.currentTaskId;
|
|
277
|
+
result.agentId = agent.id;
|
|
278
|
+
result.iteration = agent.iteration;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function publishTaskCompleted(agent, result) {
|
|
282
|
+
agent._publishLifecycle('TASK_COMPLETED', {
|
|
283
|
+
iteration: agent.iteration,
|
|
284
|
+
success: true,
|
|
285
|
+
taskId: agent.currentTaskId,
|
|
286
|
+
tokenUsage: result.tokenUsage || null,
|
|
287
|
+
});
|
|
288
|
+
}
|
|
219
289
|
|
|
290
|
+
function publishTokenUsage(agent, result) {
|
|
291
|
+
// Publish TOKEN_USAGE event for aggregation and tracking
|
|
292
|
+
// CRITICAL: Include taskId for causal linking - allows consumers to group
|
|
293
|
+
// messages by task regardless of interleaved timing from async hooks
|
|
294
|
+
if (!result.tokenUsage) {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
agent.messageBus.publish({
|
|
299
|
+
cluster_id: agent.cluster.id,
|
|
300
|
+
topic: 'TOKEN_USAGE',
|
|
301
|
+
sender: agent.id,
|
|
302
|
+
content: {
|
|
303
|
+
text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
|
|
304
|
+
data: {
|
|
305
|
+
agentId: agent.id,
|
|
306
|
+
role: agent.role,
|
|
307
|
+
model: agent._selectModel(),
|
|
308
|
+
iteration: agent.iteration,
|
|
309
|
+
taskId: agent.currentTaskId, // Causal linking for message ordering
|
|
310
|
+
...result.tokenUsage,
|
|
311
|
+
},
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
|
|
317
|
+
// Execute onComplete hook WITH RETRY
|
|
318
|
+
// Hook failure shouldn't retry the entire task - just the hook
|
|
319
|
+
const hookMaxRetries = 3;
|
|
320
|
+
const hookBaseDelay = 1000;
|
|
321
|
+
let hookSuccess = false;
|
|
322
|
+
|
|
323
|
+
for (let hookAttempt = 1; hookAttempt <= hookMaxRetries && !hookSuccess; hookAttempt++) {
|
|
220
324
|
try {
|
|
221
|
-
// Execute onStart hook
|
|
222
325
|
await executeHook({
|
|
223
|
-
hook: agent.config.hooks?.
|
|
326
|
+
hook: agent.config.hooks?.onComplete,
|
|
224
327
|
agent: agent,
|
|
225
328
|
message: triggeringMessage,
|
|
226
|
-
result:
|
|
329
|
+
result: result,
|
|
227
330
|
messageBus: agent.messageBus,
|
|
228
331
|
cluster: agent.cluster,
|
|
229
332
|
orchestrator: agent.orchestrator,
|
|
230
333
|
});
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
334
|
+
hookSuccess = true;
|
|
335
|
+
} catch (hookError) {
|
|
336
|
+
console.error(`
|
|
337
|
+
${'='.repeat(80)}`);
|
|
338
|
+
console.error(
|
|
339
|
+
`🔴 HOOK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${hookAttempt}/${hookMaxRetries})`
|
|
340
|
+
);
|
|
341
|
+
console.error(`${'='.repeat(80)}`);
|
|
342
|
+
console.error(`Error: ${hookError.message}`);
|
|
343
|
+
|
|
344
|
+
if (hookAttempt < hookMaxRetries) {
|
|
345
|
+
const delay = hookBaseDelay * Math.pow(2, hookAttempt - 1);
|
|
346
|
+
console.error(`Will retry hook in ${delay}ms...`);
|
|
347
|
+
console.error(`${'='.repeat(80)}
|
|
348
|
+
`);
|
|
349
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
350
|
+
} else {
|
|
351
|
+
console.error(`${'='.repeat(80)}
|
|
352
|
+
`);
|
|
353
|
+
// All hook retries exhausted - throw to trigger task-level handling
|
|
354
|
+
throw new Error(
|
|
355
|
+
`Hook execution failed after ${hookMaxRetries} attempts. ` +
|
|
356
|
+
`Task completed successfully but hook could not publish result. ` +
|
|
357
|
+
`Original error: ${hookError.message}`
|
|
236
358
|
);
|
|
237
|
-
agent._publishLifecycle('MAX_ITERATIONS_REACHED', {
|
|
238
|
-
iteration: agent.iteration,
|
|
239
|
-
maxIterations: agent.maxIterations,
|
|
240
|
-
});
|
|
241
|
-
// Publish failure message - orchestrator watches for this and auto-stops
|
|
242
|
-
agent._publish({
|
|
243
|
-
topic: 'CLUSTER_FAILED',
|
|
244
|
-
receiver: 'system',
|
|
245
|
-
content: {
|
|
246
|
-
text: `Agent ${agent.id} hit max iterations limit (${agent.maxIterations}). Stopping cluster.`,
|
|
247
|
-
data: {
|
|
248
|
-
reason: 'max_iterations',
|
|
249
|
-
iteration: agent.iteration,
|
|
250
|
-
maxIterations: agent.maxIterations,
|
|
251
|
-
},
|
|
252
|
-
},
|
|
253
|
-
});
|
|
254
|
-
agent.state = 'failed';
|
|
255
|
-
return;
|
|
256
359
|
}
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
257
363
|
|
|
258
|
-
|
|
259
|
-
|
|
364
|
+
async function runTaskAttempt(agent, triggeringMessage) {
|
|
365
|
+
// Execute onStart hook
|
|
366
|
+
await executeHook({
|
|
367
|
+
hook: agent.config.hooks?.onStart,
|
|
368
|
+
agent: agent,
|
|
369
|
+
message: triggeringMessage,
|
|
370
|
+
result: undefined,
|
|
371
|
+
messageBus: agent.messageBus,
|
|
372
|
+
cluster: agent.cluster,
|
|
373
|
+
orchestrator: agent.orchestrator,
|
|
374
|
+
});
|
|
260
375
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
role: agent.role,
|
|
266
|
-
iteration: agent.iteration,
|
|
267
|
-
config: agent.config,
|
|
268
|
-
messageBus: agent.messageBus,
|
|
269
|
-
cluster: agent.cluster,
|
|
270
|
-
lastTaskEndTime: agent.lastTaskEndTime,
|
|
271
|
-
triggeringMessage,
|
|
272
|
-
selectedPrompt: agent._selectPrompt(),
|
|
273
|
-
});
|
|
376
|
+
// Check max iterations limit BEFORE incrementing (prevents infinite rejection loops)
|
|
377
|
+
if (handleMaxIterations(agent)) {
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
274
380
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
console.log(`\n${'='.repeat(80)}`);
|
|
278
|
-
console.log(`📥 INPUT CONTEXT - Agent: ${agent.id} (Iteration: ${agent.iteration})`);
|
|
279
|
-
console.log(`${'='.repeat(80)}`);
|
|
280
|
-
console.log(context);
|
|
281
|
-
console.log(`${'='.repeat(80)}\n`);
|
|
282
|
-
}
|
|
381
|
+
// Increment iteration BEFORE building context so worker knows current iteration
|
|
382
|
+
agent.iteration++;
|
|
283
383
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
// LOCK CONTENTION FIX: Add random jitter for validators to prevent thundering herd
|
|
288
|
-
// When multiple validators wake on the same trigger (e.g., IMPLEMENTATION_READY),
|
|
289
|
-
// they all try to spawn Claude CLI at the same time. Claude CLI uses a lock file
|
|
290
|
-
// per workspace, so only one can run. Adding jitter staggers their starts.
|
|
291
|
-
// SKIP in testMode - tests use mocks and don't need jitter
|
|
292
|
-
if (agent.role === 'validator' && !agent.testMode) {
|
|
293
|
-
const jitterMs = Math.floor(Math.random() * 15000); // 0-15 seconds
|
|
294
|
-
if (!agent.quiet) {
|
|
295
|
-
agent._log(`[Agent ${agent.id}] Adding ${Math.round(jitterMs / 1000)}s jitter to prevent lock contention`);
|
|
296
|
-
}
|
|
297
|
-
await new Promise((resolve) => setTimeout(resolve, jitterMs));
|
|
298
|
-
}
|
|
384
|
+
// Build context
|
|
385
|
+
agent.state = 'building_context';
|
|
386
|
+
const context = agent._buildContext(triggeringMessage);
|
|
299
387
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
model: agent._selectModel(),
|
|
303
|
-
triggeredBy: triggeringMessage.topic,
|
|
304
|
-
triggerFrom: triggeringMessage.sender,
|
|
305
|
-
});
|
|
388
|
+
// Log input context (helps debug what each agent sees)
|
|
389
|
+
logInputContext(agent, context);
|
|
306
390
|
|
|
307
|
-
|
|
391
|
+
// Spawn provider task
|
|
392
|
+
agent.state = 'executing_task';
|
|
393
|
+
await applyValidatorJitter(agent);
|
|
394
|
+
publishTaskStarted(agent, triggeringMessage);
|
|
308
395
|
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
result.agentId = agent.id;
|
|
312
|
-
result.iteration = agent.iteration;
|
|
396
|
+
const result = await agent._spawnClaudeTask(context);
|
|
397
|
+
attachResultMetadata(agent, result);
|
|
313
398
|
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
399
|
+
// Check if task execution failed
|
|
400
|
+
if (!result.success) {
|
|
401
|
+
throw new Error(result.error || 'Task execution failed');
|
|
402
|
+
}
|
|
318
403
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
404
|
+
// Set state to idle BEFORE publishing lifecycle event
|
|
405
|
+
// (so lifecycle message includes correct state)
|
|
406
|
+
agent.state = 'idle';
|
|
322
407
|
|
|
323
|
-
|
|
324
|
-
|
|
408
|
+
// Track completion time for context filtering (used by "since: last_task_end")
|
|
409
|
+
agent.lastTaskEndTime = Date.now();
|
|
325
410
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
tokenUsage: result.tokenUsage || null,
|
|
331
|
-
});
|
|
411
|
+
publishTaskCompleted(agent, result);
|
|
412
|
+
publishTokenUsage(agent, result);
|
|
413
|
+
await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
|
|
414
|
+
}
|
|
332
415
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
content: {
|
|
342
|
-
text: `${agent.id} used ${result.tokenUsage.inputTokens} input + ${result.tokenUsage.outputTokens} output tokens`,
|
|
343
|
-
data: {
|
|
344
|
-
agentId: agent.id,
|
|
345
|
-
role: agent.role,
|
|
346
|
-
model: agent._selectModel(),
|
|
347
|
-
iteration: agent.iteration,
|
|
348
|
-
taskId: agent.currentTaskId, // Causal linking for message ordering
|
|
349
|
-
...result.tokenUsage,
|
|
350
|
-
},
|
|
351
|
-
},
|
|
352
|
-
});
|
|
353
|
-
}
|
|
416
|
+
function logTaskAttemptFailure(agent, attempt, maxRetries, error) {
|
|
417
|
+
// Log attempt failure
|
|
418
|
+
console.error(`
|
|
419
|
+
${'='.repeat(80)}`);
|
|
420
|
+
console.error(`🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`);
|
|
421
|
+
console.error(`${'='.repeat(80)}`);
|
|
422
|
+
console.error(`Error: ${error.message}`);
|
|
423
|
+
}
|
|
354
424
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
425
|
+
async function handleLockContention() {
|
|
426
|
+
// Lock contention - add significant jittered delay
|
|
427
|
+
const lockDelay = 10000 + Math.floor(Math.random() * 20000); // 10-30 seconds
|
|
428
|
+
console.error(
|
|
429
|
+
`⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`
|
|
430
|
+
);
|
|
431
|
+
await new Promise((resolve) => setTimeout(resolve, lockDelay));
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
|
|
435
|
+
console.error(`
|
|
436
|
+
${'='.repeat(80)}`);
|
|
437
|
+
console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
|
|
438
|
+
console.error(`${'='.repeat(80)}`);
|
|
439
|
+
console.error(`All ${maxRetries} attempts failed`);
|
|
440
|
+
console.error(`Final error: ${error.message}`);
|
|
441
|
+
console.error(`Stack: ${error.stack}`);
|
|
442
|
+
console.error(`${'='.repeat(80)}
|
|
443
|
+
`);
|
|
444
|
+
|
|
445
|
+
// CRITICAL FIX: Validator crash = REJECTION (not auto-approval)
|
|
446
|
+
// Auto-approval on crash allowed broken code to be merged - unacceptable!
|
|
447
|
+
// If validator crashed 3x, something is fundamentally wrong - REJECT and investigate
|
|
448
|
+
if (agent.role === 'validator') {
|
|
449
|
+
console.error(`
|
|
450
|
+
${'='.repeat(80)}`);
|
|
451
|
+
console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
|
|
452
|
+
console.error(`${'='.repeat(80)}`);
|
|
453
|
+
console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
|
|
454
|
+
console.error(`Error: ${error.message}`);
|
|
455
|
+
console.error(`REJECTING validation - broken code will NOT be merged`);
|
|
456
|
+
console.error(`Investigation required before retry`);
|
|
457
|
+
console.error(`${'='.repeat(80)}
|
|
458
|
+
`);
|
|
459
|
+
|
|
460
|
+
// Publish REJECTION message (NOT approval!)
|
|
461
|
+
const hook = agent.config.hooks?.onComplete;
|
|
462
|
+
if (hook && hook.action === 'publish_message') {
|
|
463
|
+
agent._publish({
|
|
464
|
+
topic: hook.config.topic,
|
|
465
|
+
receiver: hook.config.receiver || 'broadcast',
|
|
466
|
+
content: {
|
|
467
|
+
text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
|
|
468
|
+
data: {
|
|
469
|
+
approved: false, // REJECT!
|
|
470
|
+
crashedAfterRetries: true,
|
|
471
|
+
errors: JSON.stringify([
|
|
472
|
+
`VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
|
|
473
|
+
`Validation could not be performed - REJECTING to prevent broken code merge`,
|
|
474
|
+
`Investigation required before retry`,
|
|
475
|
+
]),
|
|
476
|
+
attempts: maxRetries,
|
|
477
|
+
requiresInvestigation: true,
|
|
478
|
+
},
|
|
479
|
+
},
|
|
364
480
|
});
|
|
481
|
+
}
|
|
365
482
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
// This happens when multiple validators try to run Claude CLI in the same workspace
|
|
371
|
-
const isLockError = error.message && error.message.includes('Lock file');
|
|
483
|
+
agent.state = 'error';
|
|
484
|
+
// Don't return - fall through to publish AGENT_ERROR and save failure info
|
|
485
|
+
// This allows the cluster to stop and be resumed after investigation
|
|
486
|
+
}
|
|
372
487
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
console.error(
|
|
376
|
-
`🔴 TASK EXECUTION FAILED - AGENT: ${agent.id} (Attempt ${attempt}/${maxRetries})`
|
|
377
|
-
);
|
|
378
|
-
console.error(`${'='.repeat(80)}`);
|
|
379
|
-
console.error(`Error: ${error.message}`);
|
|
380
|
-
|
|
381
|
-
if (isLockError) {
|
|
382
|
-
// Lock contention - add significant jittered delay
|
|
383
|
-
const lockDelay = 10000 + Math.floor(Math.random() * 20000); // 10-30 seconds
|
|
384
|
-
console.error(`⚠️ Lock contention detected - waiting ${Math.round(lockDelay / 1000)}s before retry`);
|
|
385
|
-
await new Promise(resolve => setTimeout(resolve, lockDelay));
|
|
386
|
-
} else if (attempt < maxRetries) {
|
|
387
|
-
console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
|
|
388
|
-
}
|
|
389
|
-
console.error(`${'='.repeat(80)}\n`);
|
|
488
|
+
// Non-validator agents: publish error and stop
|
|
489
|
+
agent.state = 'error';
|
|
390
490
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
491
|
+
// Save failure info to cluster for resume capability
|
|
492
|
+
agent.cluster.failureInfo = {
|
|
493
|
+
agentId: agent.id,
|
|
494
|
+
taskId: agent.currentTaskId,
|
|
495
|
+
iteration: agent.iteration,
|
|
496
|
+
error: error.message,
|
|
497
|
+
attempts: maxRetries,
|
|
498
|
+
timestamp: Date.now(),
|
|
499
|
+
};
|
|
400
500
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
]),
|
|
430
|
-
attempts: maxRetries,
|
|
431
|
-
requiresInvestigation: true,
|
|
432
|
-
},
|
|
433
|
-
},
|
|
434
|
-
});
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
agent.state = 'error';
|
|
438
|
-
// Don't return - fall through to publish AGENT_ERROR and save failure info
|
|
439
|
-
// This allows the cluster to stop and be resumed after investigation
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
// Non-validator agents: publish error and stop
|
|
443
|
-
agent.state = 'error';
|
|
444
|
-
|
|
445
|
-
// Save failure info to cluster for resume capability
|
|
446
|
-
agent.cluster.failureInfo = {
|
|
447
|
-
agentId: agent.id,
|
|
448
|
-
taskId: agent.currentTaskId,
|
|
449
|
-
iteration: agent.iteration,
|
|
450
|
-
error: error.message,
|
|
451
|
-
attempts: maxRetries,
|
|
452
|
-
timestamp: Date.now(),
|
|
453
|
-
};
|
|
454
|
-
|
|
455
|
-
// Publish error to message bus for visibility in logs
|
|
456
|
-
agent._publish({
|
|
457
|
-
topic: 'AGENT_ERROR',
|
|
458
|
-
receiver: 'broadcast',
|
|
459
|
-
content: {
|
|
460
|
-
text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
|
|
461
|
-
data: {
|
|
462
|
-
error: error.message,
|
|
463
|
-
stack: error.stack,
|
|
464
|
-
agent: agent.id,
|
|
465
|
-
role: agent.role,
|
|
466
|
-
iteration: agent.iteration,
|
|
467
|
-
taskId: agent.currentTaskId,
|
|
468
|
-
attempts: maxRetries,
|
|
469
|
-
hookFailureContext: error.message.includes('Hook uses result')
|
|
470
|
-
? {
|
|
471
|
-
taskId: agent.currentTaskId || 'UNKNOWN',
|
|
472
|
-
retrieveLogs: agent.currentTaskId
|
|
473
|
-
? `zeroshot task logs ${agent.currentTaskId}`
|
|
474
|
-
: 'N/A',
|
|
475
|
-
}
|
|
476
|
-
: undefined,
|
|
477
|
-
},
|
|
478
|
-
},
|
|
479
|
-
metadata: {
|
|
480
|
-
triggeringTopic: triggeringMessage.topic,
|
|
481
|
-
},
|
|
482
|
-
});
|
|
501
|
+
// Publish error to message bus for visibility in logs
|
|
502
|
+
agent._publish({
|
|
503
|
+
topic: 'AGENT_ERROR',
|
|
504
|
+
receiver: 'broadcast',
|
|
505
|
+
content: {
|
|
506
|
+
text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
|
|
507
|
+
data: {
|
|
508
|
+
error: error.message,
|
|
509
|
+
stack: error.stack,
|
|
510
|
+
agent: agent.id,
|
|
511
|
+
role: agent.role,
|
|
512
|
+
iteration: agent.iteration,
|
|
513
|
+
taskId: agent.currentTaskId,
|
|
514
|
+
attempts: maxRetries,
|
|
515
|
+
hookFailureContext: error.message.includes('Hook uses result')
|
|
516
|
+
? {
|
|
517
|
+
taskId: agent.currentTaskId || 'UNKNOWN',
|
|
518
|
+
retrieveLogs: agent.currentTaskId
|
|
519
|
+
? `zeroshot task logs ${agent.currentTaskId}`
|
|
520
|
+
: 'N/A',
|
|
521
|
+
}
|
|
522
|
+
: undefined,
|
|
523
|
+
},
|
|
524
|
+
},
|
|
525
|
+
metadata: {
|
|
526
|
+
triggeringTopic: triggeringMessage.topic,
|
|
527
|
+
},
|
|
528
|
+
});
|
|
483
529
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
530
|
+
// Execute onError hook
|
|
531
|
+
await executeHook({
|
|
532
|
+
hook: agent.config.hooks?.onError,
|
|
533
|
+
agent: agent,
|
|
534
|
+
message: triggeringMessage,
|
|
535
|
+
result: { error },
|
|
536
|
+
messageBus: agent.messageBus,
|
|
537
|
+
cluster: agent.cluster,
|
|
538
|
+
orchestrator: agent.orchestrator,
|
|
539
|
+
});
|
|
494
540
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
}
|
|
541
|
+
agent.state = 'idle';
|
|
542
|
+
}
|
|
498
543
|
|
|
499
|
-
|
|
500
|
-
|
|
544
|
+
async function scheduleRetry(agent, error, attempt, maxRetries, baseDelay) {
|
|
545
|
+
const delay = baseDelay * Math.pow(2, attempt - 1); // 2s, 4s, 8s
|
|
501
546
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
547
|
+
agent._publishLifecycle('RETRY_SCHEDULED', {
|
|
548
|
+
attempt,
|
|
549
|
+
maxRetries,
|
|
550
|
+
delayMs: delay,
|
|
551
|
+
error: error.message,
|
|
552
|
+
});
|
|
508
553
|
|
|
509
|
-
|
|
554
|
+
agent._log(`[${agent.id}] ⚠️ Retrying in ${delay}ms... (${attempt + 1}/${maxRetries})`);
|
|
510
555
|
|
|
511
|
-
|
|
512
|
-
|
|
556
|
+
// Exponential backoff
|
|
557
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
513
558
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
559
|
+
agent._log(`[${agent.id}] 🔄 Starting retry attempt ${attempt + 1}/${maxRetries}`);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
async function handleTaskAttemptFailure({
|
|
563
|
+
agent,
|
|
564
|
+
triggeringMessage,
|
|
565
|
+
error,
|
|
566
|
+
attempt,
|
|
567
|
+
maxRetries,
|
|
568
|
+
baseDelay,
|
|
569
|
+
}) {
|
|
570
|
+
// LOCK CONTENTION: Add extra jittered delay for lock file errors
|
|
571
|
+
// This happens when multiple validators try to run Claude CLI in the same workspace
|
|
572
|
+
const isLockError = error.message && error.message.includes('Lock file');
|
|
573
|
+
|
|
574
|
+
logTaskAttemptFailure(agent, attempt, maxRetries, error);
|
|
575
|
+
|
|
576
|
+
if (isLockError) {
|
|
577
|
+
await handleLockContention();
|
|
578
|
+
} else if (attempt < maxRetries) {
|
|
579
|
+
console.error(`Will retry in ${baseDelay * Math.pow(2, attempt - 1)}ms...`);
|
|
517
580
|
}
|
|
581
|
+
console.error(`${'='.repeat(80)}
|
|
582
|
+
`);
|
|
583
|
+
|
|
584
|
+
if (attempt >= maxRetries) {
|
|
585
|
+
await handleFinalFailure(agent, triggeringMessage, error, maxRetries);
|
|
586
|
+
return true;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
await scheduleRetry(agent, error, attempt, maxRetries, baseDelay);
|
|
590
|
+
return false;
|
|
518
591
|
}
|
|
519
592
|
|
|
520
593
|
/**
|
|
521
|
-
*
|
|
522
|
-
*
|
|
523
|
-
* SAFE DETECTION: Only flags as stuck when MULTIPLE indicators agree:
|
|
524
|
-
* - Process sleeping (state=S)
|
|
525
|
-
* - Blocked on epoll/poll wait
|
|
526
|
-
* - Low CPU usage (<1%)
|
|
527
|
-
* - Low context switches (<10)
|
|
528
|
-
* - No network data in flight
|
|
529
|
-
*
|
|
530
|
-
* Single-indicator detection (just output freshness) has HIGH false positive risk.
|
|
531
|
-
* This multi-indicator approach eliminates false positives.
|
|
532
|
-
*
|
|
594
|
+
* Execute claude-zeroshots with built context
|
|
595
|
+
* Retries disabled by default. Set agent config `maxRetries` to enable (e.g., 3).
|
|
533
596
|
* @param {AgentWrapper} agent - Agent instance
|
|
597
|
+
* @param {Object} triggeringMessage - Message that triggered execution
|
|
534
598
|
*/
|
|
599
|
+
async function executeTask(agent, triggeringMessage) {
|
|
600
|
+
// Early exit if agent was stopped
|
|
601
|
+
if (!agent.running) {
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Default: no retries (maxRetries=1 means 1 attempt only)
|
|
606
|
+
// Set agent config `maxRetries: 3` to enable exponential backoff retries
|
|
607
|
+
const maxRetries = agent.config.maxRetries ?? 1;
|
|
608
|
+
const baseDelay = 2000; // 2 seconds
|
|
609
|
+
|
|
610
|
+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
|
|
611
|
+
// Check if agent was stopped between retries
|
|
612
|
+
if (!agent.running) {
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
try {
|
|
617
|
+
await runTaskAttempt(agent, triggeringMessage);
|
|
618
|
+
return;
|
|
619
|
+
} catch (error) {
|
|
620
|
+
const shouldStop = await handleTaskAttemptFailure({
|
|
621
|
+
agent,
|
|
622
|
+
triggeringMessage,
|
|
623
|
+
error,
|
|
624
|
+
attempt,
|
|
625
|
+
maxRetries,
|
|
626
|
+
baseDelay,
|
|
627
|
+
});
|
|
628
|
+
if (shouldStop) {
|
|
629
|
+
return;
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
535
634
|
function startLivenessCheck(agent) {
|
|
536
635
|
if (agent.livenessCheckInterval) {
|
|
537
636
|
clearInterval(agent.livenessCheckInterval);
|