@covibes/zeroshot 5.3.0 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +94 -14
  2. package/cli/commands/providers.js +8 -9
  3. package/cli/index.js +3032 -2409
  4. package/cli/message-formatters-normal.js +28 -6
  5. package/cluster-templates/base-templates/debug-workflow.json +1 -1
  6. package/cluster-templates/base-templates/full-workflow.json +72 -188
  7. package/cluster-templates/base-templates/worker-validator.json +59 -3
  8. package/cluster-templates/conductor-bootstrap.json +4 -4
  9. package/lib/docker-config.js +8 -0
  10. package/lib/git-remote-utils.js +165 -0
  11. package/lib/id-detector.js +10 -7
  12. package/lib/provider-defaults.js +62 -0
  13. package/lib/provider-names.js +2 -1
  14. package/lib/settings/claude-auth.js +78 -0
  15. package/lib/settings.js +161 -63
  16. package/package.json +7 -2
  17. package/scripts/setup-merge-queue.sh +170 -0
  18. package/src/agent/agent-config.js +135 -82
  19. package/src/agent/agent-context-builder.js +297 -188
  20. package/src/agent/agent-hook-executor.js +310 -113
  21. package/src/agent/agent-lifecycle.js +385 -325
  22. package/src/agent/agent-stuck-detector.js +7 -7
  23. package/src/agent/agent-task-executor.js +824 -565
  24. package/src/agent/output-extraction.js +41 -24
  25. package/src/agent/output-reformatter.js +1 -1
  26. package/src/agent/schema-utils.js +108 -73
  27. package/src/agent-wrapper.js +10 -1
  28. package/src/agents/git-pusher-template.js +285 -0
  29. package/src/claude-task-runner.js +85 -34
  30. package/src/config-validator.js +922 -657
  31. package/src/input-helpers.js +65 -0
  32. package/src/isolation-manager.js +289 -199
  33. package/src/issue-providers/README.md +305 -0
  34. package/src/issue-providers/azure-devops-provider.js +273 -0
  35. package/src/issue-providers/base-provider.js +232 -0
  36. package/src/issue-providers/github-provider.js +179 -0
  37. package/src/issue-providers/gitlab-provider.js +241 -0
  38. package/src/issue-providers/index.js +196 -0
  39. package/src/issue-providers/jira-provider.js +239 -0
  40. package/src/ledger.js +22 -5
  41. package/src/lib/safe-exec.js +88 -0
  42. package/src/orchestrator.js +1107 -811
  43. package/src/preflight.js +313 -159
  44. package/src/process-metrics.js +98 -56
  45. package/src/providers/anthropic/cli-builder.js +53 -25
  46. package/src/providers/anthropic/index.js +72 -2
  47. package/src/providers/anthropic/output-parser.js +32 -14
  48. package/src/providers/base-provider.js +70 -0
  49. package/src/providers/capabilities.js +9 -0
  50. package/src/providers/google/output-parser.js +47 -38
  51. package/src/providers/index.js +19 -3
  52. package/src/providers/openai/cli-builder.js +14 -3
  53. package/src/providers/openai/index.js +1 -0
  54. package/src/providers/openai/output-parser.js +44 -30
  55. package/src/providers/opencode/cli-builder.js +42 -0
  56. package/src/providers/opencode/index.js +103 -0
  57. package/src/providers/opencode/models.js +55 -0
  58. package/src/providers/opencode/output-parser.js +122 -0
  59. package/src/schemas/sub-cluster.js +68 -39
  60. package/src/status-footer.js +94 -48
  61. package/src/sub-cluster-wrapper.js +76 -35
  62. package/src/template-resolver.js +12 -9
  63. package/src/tui/data-poller.js +123 -99
  64. package/src/tui/formatters.js +4 -3
  65. package/src/tui/keybindings.js +259 -318
  66. package/src/tui/renderer.js +11 -21
  67. package/task-lib/attachable-watcher.js +118 -81
  68. package/task-lib/claude-recovery.js +61 -24
  69. package/task-lib/commands/episodes.js +105 -0
  70. package/task-lib/commands/list.js +2 -2
  71. package/task-lib/commands/logs.js +231 -189
  72. package/task-lib/commands/schedules.js +111 -61
  73. package/task-lib/config.js +0 -2
  74. package/task-lib/runner.js +84 -27
  75. package/task-lib/scheduler.js +1 -1
  76. package/task-lib/store.js +467 -168
  77. package/task-lib/tui/formatters.js +94 -45
  78. package/task-lib/tui/renderer.js +13 -3
  79. package/task-lib/tui.js +73 -32
  80. package/task-lib/watcher.js +128 -90
  81. package/src/agents/git-pusher-agent.json +0 -20
  82. package/src/github.js +0 -139
@@ -14,11 +14,43 @@ const { spawn } = require('child_process');
14
14
  const path = require('path');
15
15
  const fs = require('fs');
16
16
  const os = require('os');
17
+ const { exec, execSync } = require('../lib/safe-exec'); // Enforces timeouts - prevents infinite hangs
17
18
  const { getProvider, parseChunkWithProvider } = require('../providers');
19
+ const { getTask } = require('../../task-lib/store.js');
20
+ const { loadSettings } = require('../../lib/settings.js');
21
+ const { resolveClaudeAuth } = require('../../lib/settings/claude-auth.js');
18
22
 
19
23
  // Schema utilities for normalizing LLM output
20
24
  const { normalizeEnumValues } = require('./schema-utils');
21
25
 
26
+ /**
27
+ * Build Claude-specific environment variables for task spawning
28
+ * Consolidates auth resolution and model mapping logic used by both isolated and non-isolated modes
29
+ * @param {Object} modelSpec - Model specification from agent
30
+ * @param {Object} [options] - Options
31
+ * @param {boolean} [options.includeAuth=true] - Include auth env vars (false for isolated mode where IsolationManager handles auth)
32
+ * @returns {Object} Environment variables to merge into spawn env
33
+ */
34
+ function buildClaudeEnv(modelSpec, options = {}) {
35
+ const { includeAuth = true } = options;
36
+ const env = {};
37
+
38
+ if (includeAuth) {
39
+ const settings = loadSettings();
40
+ const authEnv = resolveClaudeAuth(settings);
41
+ Object.assign(env, authEnv);
42
+ }
43
+
44
+ if (modelSpec?.model) {
45
+ env.ANTHROPIC_MODEL = modelSpec.model;
46
+ }
47
+
48
+ // Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
49
+ env.ZEROSHOT_BLOCK_ASK_USER = '1';
50
+
51
+ return env;
52
+ }
53
+
22
54
  /**
23
55
  * Validate and sanitize error messages.
24
56
  * Detects TypeScript type annotations that may have leaked into error storage.
@@ -38,12 +70,17 @@ function sanitizeErrorMessage(error) {
38
70
  /^unknown$/i,
39
71
  /^void$/i,
40
72
  /^never$/i,
41
- /^[A-Z][a-zA-Z]*\s*\|\s*(null|undefined)$/, // e.g., "Error | null"
42
- /^[a-z]+(\s*\|\s*[a-z]+)+$/i, // e.g., "string | number | boolean"
73
+ /^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
43
74
  ];
44
75
 
76
+ const trimmedError = error.trim();
77
+
78
+ // Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
79
+ const unionParts = trimmedError.split(/\s*\|\s*/);
80
+ const isUnionType = unionParts.length > 1 && unionParts.every((p) => /^[a-z]+$/i.test(p));
81
+
45
82
  for (const pattern of typeAnnotationPatterns) {
46
- if (pattern.test(error.trim())) {
83
+ if (pattern.test(trimmedError) || isUnionType) {
47
84
  console.warn(
48
85
  `[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${error}". ` +
49
86
  `This indicates corrupted data. Replacing with generic error.`
@@ -101,16 +138,31 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
101
138
  );
102
139
  }
103
140
 
104
- // Fall back to extracting from output (last 500 chars)
105
- const lastOutput = fullOutput.slice(-500).trim();
106
- if (!lastOutput) {
141
+ // NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
142
+ // If output is too verbose, that's a prompt problem - fix the prompts, not the data
143
+ const trimmedOutput = (output || '').trim();
144
+ if (!trimmedOutput) {
107
145
  return sanitizeErrorMessage(
108
146
  'Task failed with no output (check if task was interrupted or timed out)'
109
147
  );
110
148
  }
111
149
 
150
+ // Try to extract structured JSON from output first - it may contain the actual result
151
+ // even if the task was marked as "failed" due to timeout/stale status
152
+ try {
153
+ const { extractJsonFromOutput } = require('./output-extraction');
154
+ const extracted = extractJsonFromOutput(trimmedOutput);
155
+ if (extracted && typeof extracted === 'object') {
156
+ // If we found valid JSON, return it as the error context
157
+ // This preserves the actual agent output for downstream processing
158
+ return JSON.stringify(extracted);
159
+ }
160
+ } catch {
161
+ // Extraction failed, fall through to error pattern matching
162
+ }
163
+
112
164
  // Extract non-JSON lines only (JSON lines contain "is_error": true which falsely matches)
113
- const nonJsonLines = lastOutput
165
+ const nonJsonLines = trimmedOutput
114
166
  .split('\n')
115
167
  .filter((line) => {
116
168
  const trimmed = line.trim();
@@ -120,7 +172,7 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
120
172
  .join('\n');
121
173
 
122
174
  // Common error patterns - match against non-JSON content
123
- const textToSearch = nonJsonLines || lastOutput;
175
+ const textToSearch = nonJsonLines || trimmedOutput;
124
176
  const errorPatterns = [
125
177
  /Error:\s*(.+)/i,
126
178
  /error:\s*(.+)/i,
@@ -132,12 +184,14 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
132
184
  for (const pattern of errorPatterns) {
133
185
  const match = textToSearch.match(pattern);
134
186
  if (match) {
135
- return sanitizeErrorMessage(match[1].slice(0, 200));
187
+ // Don't truncate - let the full error message through
188
+ return sanitizeErrorMessage(match[1]);
136
189
  }
137
190
  }
138
191
 
139
- // No pattern matched - include last portion of output
140
- return sanitizeErrorMessage(`Task failed. Last output: ${lastOutput.slice(-200)}`);
192
+ // No pattern matched - return full output (no truncation)
193
+ // If this is too long, the solution is to make agents output less, not to corrupt data
194
+ return sanitizeErrorMessage(`Task failed. Output: ${trimmedOutput}`);
141
195
  }
142
196
 
143
197
  // Track if we've already ensured the AskUserQuestion hook is installed
@@ -339,14 +393,109 @@ function ensureDangerousGitHook() {
339
393
  */
340
394
  async function spawnClaudeTask(agent, context) {
341
395
  const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
342
- const modelSpec = agent._resolveModelSpec
343
- ? agent._resolveModelSpec()
344
- : { model: agent._selectModel() };
396
+ const modelSpec = resolveAgentModelSpec(agent);
345
397
 
346
398
  const ctPath = getClaudeTasksPath();
347
399
  const cwd = agent.config.cwd || process.cwd();
348
400
 
349
401
  // Build zeroshot task run args.
402
+ // CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
403
+ // strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
404
+ // strictSchema=false uses stream-json with post-run validation (live logs but fragile)
405
+ const { desiredOutputFormat, runOutputFormat } = resolveOutputFormatConfig(agent);
406
+ const args = buildTaskRunArgs({
407
+ agent,
408
+ providerName,
409
+ modelSpec,
410
+ runOutputFormat,
411
+ });
412
+
413
+ // NOTE: maxRetries is handled by the agent wrapper's internal retry loop,
414
+ // not passed to the CLI. See _handleTrigger() for retry logic.
415
+
416
+ maybeLogStreamJsonNotice(agent, runOutputFormat);
417
+
418
+ // If schema enforcement is desired but we had to run stream-json for live logs,
419
+ // add explicit output instructions so the model still knows the required shape.
420
+ const finalContext = buildFinalContext({
421
+ agent,
422
+ context,
423
+ desiredOutputFormat,
424
+ runOutputFormat,
425
+ });
426
+
427
+ args.push(finalContext);
428
+
429
+ // MOCK SUPPORT: Use injected mock function if provided
430
+ if (agent.mockSpawnFn) {
431
+ return agent.mockSpawnFn(args, { context });
432
+ }
433
+
434
+ // SAFETY: Fail hard if testMode=true but no mock (should be caught in constructor)
435
+ if (agent.testMode) {
436
+ throw new Error(
437
+ `AgentWrapper: testMode=true but attempting real Claude API call for agent '${agent.id}'. ` +
438
+ `This is a bug - mock should be set in constructor.`
439
+ );
440
+ }
441
+
442
+ // ISOLATION MODE: Run inside Docker container
443
+ if (agent.isolation?.enabled) {
444
+ return spawnClaudeTaskIsolated(agent, context);
445
+ }
446
+
447
+ // NON-ISOLATION MODE: For Claude, use user's existing Claude config
448
+ // AskUserQuestion blocking handled via:
449
+ // 1. Prompt injection (see agent-context-builder)
450
+ // 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
451
+ ensureProviderHooks(agent, providerName);
452
+ const spawnEnv = buildSpawnEnv(agent, providerName, modelSpec);
453
+ const taskId = await spawnTaskProcess({
454
+ agent,
455
+ ctPath,
456
+ args,
457
+ cwd,
458
+ spawnEnv,
459
+ });
460
+
461
+ agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
462
+
463
+ // Wait for task to be registered in zeroshot storage (race condition fix)
464
+ await waitForTaskReady(agent, taskId);
465
+
466
+ // CRITICAL: Poll for REAL process PID from task store
467
+ // The watcher spawns the actual CLI and writes PID to SQLite asynchronously.
468
+ // We must poll because the watcher runs in a forked process.
469
+ const MAX_PID_POLLS = 30; // 3 seconds max
470
+ const PID_POLL_DELAY = 100;
471
+ let realPid = null;
472
+
473
+ for (let i = 0; i < MAX_PID_POLLS; i++) {
474
+ const taskInfo = getTask(taskId);
475
+ if (taskInfo?.pid) {
476
+ realPid = taskInfo.pid;
477
+ break;
478
+ }
479
+ await new Promise((r) => setTimeout(r, PID_POLL_DELAY));
480
+ }
481
+
482
+ if (realPid) {
483
+ agent.processPid = realPid;
484
+ agent._publishLifecycle('PROCESS_SPAWNED', { pid: realPid });
485
+ agent._log(`📋 Agent ${agent.id}: Process PID: ${realPid}`);
486
+ } else {
487
+ agent._log(`⚠️ Agent ${agent.id}: PID not available (task may use non-standard watcher)`);
488
+ }
489
+
490
+ // Now follow the logs and stream output
491
+ return followClaudeTaskLogs(agent, taskId);
492
+ }
493
+
494
+ function resolveAgentModelSpec(agent) {
495
+ return agent._resolveModelSpec ? agent._resolveModelSpec() : { model: agent._selectModel() };
496
+ }
497
+
498
+ function resolveOutputFormatConfig(agent) {
350
499
  // CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
351
500
  // strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
352
501
  // strictSchema=false uses stream-json with post-run validation (live logs but fragile)
@@ -356,6 +505,11 @@ async function spawnClaudeTask(agent, context) {
356
505
  agent.config.jsonSchema && desiredOutputFormat === 'json' && !strictSchema
357
506
  ? 'stream-json'
358
507
  : desiredOutputFormat;
508
+
509
+ return { desiredOutputFormat, strictSchema, runOutputFormat };
510
+ }
511
+
512
+ function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) {
359
513
  const args = ['task', 'run', '--output-format', runOutputFormat, '--provider', providerName];
360
514
 
361
515
  if (modelSpec?.model) {
@@ -371,82 +525,61 @@ async function spawnClaudeTask(agent, context) {
371
525
  args.push('-v');
372
526
  }
373
527
 
374
- // NOTE: maxRetries is handled by the agent wrapper's internal retry loop,
375
- // not passed to the CLI. See _handleTrigger() for retry logic.
376
-
377
528
  // Add JSON schema if specified in agent config.
378
529
  // If we are running stream-json for live logs (strictSchema=false), do NOT pass schema to CLI.
379
- if (agent.config.jsonSchema) {
380
- if (runOutputFormat === 'json') {
381
- // strictSchema=true OR no schema conflict: pass schema to CLI for native enforcement
382
- const schema = JSON.stringify(agent.config.jsonSchema);
383
- args.push('--json-schema', schema);
384
- } else if (!agent.quiet) {
385
- agent._log(
386
- `[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
387
- );
388
- }
530
+ if (agent.config.jsonSchema && runOutputFormat === 'json') {
531
+ const schema = JSON.stringify(agent.config.jsonSchema);
532
+ args.push('--json-schema', schema);
389
533
  }
390
534
 
391
- // If schema enforcement is desired but we had to run stream-json for live logs,
392
- // add explicit output instructions so the model still knows the required shape.
393
- let finalContext = context;
535
+ return args;
536
+ }
537
+
538
+ function maybeLogStreamJsonNotice(agent, runOutputFormat) {
539
+ if (agent.config.jsonSchema && runOutputFormat !== 'json' && !agent.quiet) {
540
+ agent._log(
541
+ `[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
542
+ );
543
+ }
544
+ }
545
+
546
+ function buildFinalContext({ agent, context, desiredOutputFormat, runOutputFormat }) {
394
547
  if (
395
548
  agent.config.jsonSchema &&
396
549
  desiredOutputFormat === 'json' &&
397
550
  runOutputFormat === 'stream-json'
398
551
  ) {
399
- finalContext += `\n\n## Output Format (REQUIRED)\n\nReturn a JSON object that matches this schema exactly.\n\nSchema:\n\`\`\`json\n${JSON.stringify(
400
- agent.config.jsonSchema,
401
- null,
402
- 2
403
- )}\n\`\`\`\n`;
404
- }
405
-
406
- args.push(finalContext);
407
-
408
- // MOCK SUPPORT: Use injected mock function if provided
409
- if (agent.mockSpawnFn) {
410
- return agent.mockSpawnFn(args, { context });
411
- }
412
-
413
- // SAFETY: Fail hard if testMode=true but no mock (should be caught in constructor)
414
- if (agent.testMode) {
415
- throw new Error(
416
- `AgentWrapper: testMode=true but attempting real Claude API call for agent '${agent.id}'. ` +
417
- `This is a bug - mock should be set in constructor.`
552
+ return (
553
+ context +
554
+ `\n\n## Output Format (REQUIRED)\n\nReturn a JSON object that matches this schema exactly.\n\nSchema:\n\`\`\`json\n${JSON.stringify(
555
+ agent.config.jsonSchema,
556
+ null,
557
+ 2
558
+ )}\n\`\`\`\n`
418
559
  );
419
560
  }
420
561
 
421
- // ISOLATION MODE: Run inside Docker container
422
- if (agent.isolation?.enabled) {
423
- return spawnClaudeTaskIsolated(agent, context);
562
+ return context;
563
+ }
564
+
565
+ function ensureProviderHooks(agent, providerName) {
566
+ if (providerName !== 'claude') {
567
+ return;
424
568
  }
425
569
 
426
- // NON-ISOLATION MODE: For Claude, use user's existing Claude config
427
- // AskUserQuestion blocking handled via:
428
- // 1. Prompt injection (see agent-context-builder)
429
- // 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
430
- if (providerName === 'claude') {
431
- ensureAskUserQuestionHook();
570
+ ensureAskUserQuestionHook();
432
571
 
433
- // WORKTREE MODE: Install git safety hook (blocks dangerous git commands)
434
- if (agent.worktree?.enabled) {
435
- ensureDangerousGitHook();
436
- }
572
+ // WORKTREE MODE: Install git safety hook (blocks dangerous git commands)
573
+ if (agent.worktree?.enabled) {
574
+ ensureDangerousGitHook();
437
575
  }
576
+ }
438
577
 
439
- // Build environment for spawn
440
- const spawnEnv = {
441
- ...process.env,
442
- };
578
+ function buildSpawnEnv(agent, providerName, modelSpec) {
579
+ const spawnEnv = { ...process.env };
443
580
 
444
581
  if (providerName === 'claude') {
445
- if (modelSpec?.model) {
446
- spawnEnv.ANTHROPIC_MODEL = modelSpec.model;
447
- }
448
- // Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
449
- spawnEnv.ZEROSHOT_BLOCK_ASK_USER = '1';
582
+ Object.assign(spawnEnv, buildClaudeEnv(modelSpec));
450
583
 
451
584
  // WORKTREE MODE: Activate git safety hook via environment variable
452
585
  if (agent.worktree?.enabled) {
@@ -454,18 +587,45 @@ async function spawnClaudeTask(agent, context) {
454
587
  }
455
588
  }
456
589
 
457
- const taskId = await new Promise((resolve, reject) => {
590
+ return spawnEnv;
591
+ }
592
+
593
+ function parseTaskIdFromOutput(stdout) {
594
+ const match = stdout.match(/Task spawned: ((?:task-)?[a-z]+-[a-z]+-[a-z0-9]+)/);
595
+ return match ? match[1] : null;
596
+ }
597
+
598
+ function spawnTaskProcess({ agent, ctPath, args, cwd, spawnEnv }) {
599
+ // Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
600
+ const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
601
+
602
+ return new Promise((resolve, reject) => {
458
603
  const proc = spawn(ctPath, args, {
459
604
  cwd,
460
605
  stdio: ['ignore', 'pipe', 'pipe'],
461
606
  env: spawnEnv,
462
607
  });
463
- // Track PID for resource monitoring
464
- agent.processPid = proc.pid;
465
- agent._publishLifecycle('PROCESS_SPAWNED', { pid: proc.pid });
608
+
609
+ // NOTE: Don't emit PROCESS_SPAWNED here - proc.pid is a wrapper that exits immediately.
610
+ // Real PID comes from task store after watcher spawns the actual CLI process.
611
+ // PROCESS_SPAWNED is emitted in spawnClaudeTask after waitForTaskReady + PID polling.
466
612
 
467
613
  let stdout = '';
468
614
  let stderr = '';
615
+ let resolved = false;
616
+
617
+ // CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
618
+ const spawnTimeout = setTimeout(() => {
619
+ if (resolved) return;
620
+ resolved = true;
621
+ proc.kill('SIGKILL');
622
+ reject(
623
+ new Error(
624
+ `Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
625
+ `stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
626
+ )
627
+ );
628
+ }, SPAWN_TIMEOUT_MS);
469
629
 
470
630
  proc.stdout.on('data', (data) => {
471
631
  stdout += data.toString();
@@ -476,6 +636,9 @@ async function spawnClaudeTask(agent, context) {
476
636
  });
477
637
 
478
638
  proc.on('close', (code, signal) => {
639
+ clearTimeout(spawnTimeout);
640
+ if (resolved) return;
641
+ resolved = true;
479
642
  // Handle process killed by signal (e.g., SIGTERM, SIGKILL, SIGSTOP)
480
643
  if (signal) {
481
644
  reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
@@ -485,9 +648,8 @@ async function spawnClaudeTask(agent, context) {
485
648
  if (code === 0) {
486
649
  // Parse task ID from output: "✓ Task spawned: xxx-yyy-nn"
487
650
  // Format: <adjective>-<noun>-<digits> (may or may not have task- prefix)
488
- const match = stdout.match(/Task spawned: ((?:task-)?[a-z]+-[a-z]+-[a-z0-9]+)/);
489
- if (match) {
490
- const spawnedTaskId = match[1];
651
+ const spawnedTaskId = parseTaskIdFromOutput(stdout);
652
+ if (spawnedTaskId) {
491
653
  agent.currentTaskId = spawnedTaskId; // Track for resume capability
492
654
  agent._publishLifecycle('TASK_ID_ASSIGNED', {
493
655
  pid: agent.processPid,
@@ -510,17 +672,12 @@ async function spawnClaudeTask(agent, context) {
510
672
  });
511
673
 
512
674
  proc.on('error', (error) => {
675
+ clearTimeout(spawnTimeout);
676
+ if (resolved) return;
677
+ resolved = true;
513
678
  reject(error);
514
679
  });
515
680
  });
516
-
517
- agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
518
-
519
- // Wait for task to be registered in zeroshot storage (race condition fix)
520
- await waitForTaskReady(agent, taskId);
521
-
522
- // Now follow the logs and stream output
523
- return followClaudeTaskLogs(agent, taskId);
524
681
  }
525
682
 
526
683
  /**
@@ -532,16 +689,16 @@ async function spawnClaudeTask(agent, context) {
532
689
  * @returns {Promise<void>}
533
690
  */
534
691
  async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
535
- const { exec } = require('child_process');
536
692
  const ctPath = getClaudeTasksPath();
537
693
 
538
694
  for (let i = 0; i < maxRetries; i++) {
539
- const exists = await new Promise((resolve) => {
540
- exec(`${ctPath} status ${taskId}`, (error, stdout) => {
541
- // Task exists if status doesn't return "Task not found"
542
- resolve(!error && !stdout.includes('Task not found'));
543
- });
544
- });
695
+ let exists = false;
696
+ try {
697
+ const { stdout } = await exec(`${ctPath} status ${taskId}`, { timeout: 5000 });
698
+ exists = !stdout.includes('Task not found');
699
+ } catch {
700
+ // Timeout or error - task not ready yet
701
+ }
545
702
 
546
703
  if (exists) return;
547
704
 
@@ -549,332 +706,372 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
549
706
  await new Promise((r) => setTimeout(r, delayMs));
550
707
  }
551
708
 
552
- // Continue anyway after max retries - the task may still work
553
- console.warn(`⚠️ Task ${taskId} not yet visible after ${maxRetries} retries, continuing anyway`);
709
+ // FAIL FAST: Task not found after retries = unrecoverable error
710
+ // Continuing with a non-existent task causes 30s of pointless polling then crash
711
+ throw new Error(
712
+ `Task ${taskId} not found after ${maxRetries} retries (${maxRetries * delayMs}ms). ` +
713
+ `Task spawn may have failed silently. Check zeroshot task run output.`
714
+ );
554
715
  }
555
716
 
556
- /**
557
- * Follow claude-zeroshots logs until completion, streaming to message bus
558
- * Reads log file directly for reliable streaming
559
- * @param {Object} agent - Agent instance
560
- * @param {String} taskId - Task ID to follow
561
- * @returns {Promise<Object>} Result object { success, output, error }
562
- */
563
- function followClaudeTaskLogs(agent, taskId) {
564
- const fsModule = require('fs');
565
- const { execSync, exec } = require('child_process');
566
- const ctPath = getClaudeTasksPath();
567
- const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
717
+ const MAX_STATUS_FAILURES = 30;
568
718
 
569
- return new Promise((resolve, _reject) => {
570
- let output = '';
571
- let logFilePath = null;
572
- let lastSize = 0;
573
- let pollInterval = null;
574
- let statusCheckInterval = null;
575
- let resolved = false;
719
+ function createLogFollowState() {
720
+ return {
721
+ output: '',
722
+ logFilePath: null,
723
+ lastSize: 0,
724
+ pollInterval: null,
725
+ statusCheckInterval: null,
726
+ resolved: false,
727
+ lineBuffer: '',
728
+ consecutiveExecFailures: 0,
729
+ };
730
+ }
576
731
 
577
- // Get log file path from ct
578
- try {
579
- logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
580
- encoding: 'utf-8',
581
- }).trim();
582
- agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
583
- } catch {
584
- // Task might not have log file yet, wait and retry
585
- agent._log(`⏳ Agent ${agent.id}: Waiting for log file...`);
732
+ function lookupLogFilePath(ctPath, taskId) {
733
+ try {
734
+ return execSync(`${ctPath} get-log-path ${taskId}`, {
735
+ encoding: 'utf-8',
736
+ timeout: 5000,
737
+ }).trim();
738
+ } catch {
739
+ return null;
740
+ }
741
+ }
742
+
743
+ function parseTimestampedLine(line) {
744
+ let timestamp = Date.now();
745
+ let content = line.replace(/\r$/, '');
746
+
747
+ const timestampMatch = content.match(/^\[(\d{13})\](.*)$/);
748
+ if (timestampMatch) {
749
+ timestamp = parseInt(timestampMatch[1], 10);
750
+ content = timestampMatch[2];
751
+ }
752
+
753
+ return { timestamp, content };
754
+ }
755
+
756
+ function shouldSkipLogLine(content) {
757
+ return (
758
+ content.startsWith('===') ||
759
+ content.startsWith('Finished:') ||
760
+ content.startsWith('Exit code:') ||
761
+ (content.includes('"type":"system"') && content.includes('"subtype":"init"'))
762
+ );
763
+ }
764
+
765
+ function isValidJsonLine(content) {
766
+ if (!content.trim().startsWith('{')) {
767
+ return false;
768
+ }
769
+
770
+ try {
771
+ JSON.parse(content);
772
+ return true;
773
+ } catch {
774
+ return false;
775
+ }
776
+ }
777
+
778
+ function broadcastAgentLine({ agent, providerName, state, line }) {
779
+ if (!line.trim()) return;
780
+
781
+ const { timestamp, content } = parseTimestampedLine(line);
782
+ if (shouldSkipLogLine(content)) {
783
+ return;
784
+ }
785
+
786
+ const isValidJson = isValidJsonLine(content);
787
+ state.output += content + '\n';
788
+
789
+ agent.lastOutputTime = Date.now();
790
+
791
+ agent._publish({
792
+ topic: 'AGENT_OUTPUT',
793
+ receiver: 'broadcast',
794
+ timestamp,
795
+ content: {
796
+ text: content,
797
+ data: {
798
+ type: isValidJson ? 'json' : 'text',
799
+ line: content,
800
+ agent: agent.id,
801
+ role: agent.role,
802
+ iteration: agent.iteration,
803
+ provider: providerName,
804
+ },
805
+ },
806
+ });
807
+ }
808
+
809
+ function appendContentToBuffer(state, content, onLine) {
810
+ state.lineBuffer += content;
811
+ const lines = state.lineBuffer.split('\n');
812
+
813
+ for (let i = 0; i < lines.length - 1; i++) {
814
+ onLine(lines[i]);
815
+ }
816
+
817
+ state.lineBuffer = lines[lines.length - 1];
818
+ }
819
+
820
+ function pollLogFileForUpdates({ agent, fsModule, ctPath, taskId, state, onNewContent }) {
821
+ if (!state.logFilePath) {
822
+ const logFilePath = lookupLogFilePath(ctPath, taskId);
823
+ if (!logFilePath) {
824
+ return;
586
825
  }
826
+ state.logFilePath = logFilePath;
827
+ agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
828
+ }
587
829
 
588
- // Buffer for incomplete lines across polls
589
- let lineBuffer = '';
830
+ if (!fsModule.existsSync(state.logFilePath)) {
831
+ return;
832
+ }
590
833
 
591
- // Broadcast a complete JSON line as one message
592
- // Lines are now prefixed with timestamps: [1733301234567]{json...}
593
- const broadcastLine = (line) => {
594
- if (!line.trim()) return;
834
+ try {
835
+ const stats = fsModule.statSync(state.logFilePath);
836
+ const currentSize = stats.size;
595
837
 
596
- // Parse timestamp prefix if present: [epochMs]content
597
- // IMPORTANT: Trim \r from CRLF line endings before matching
598
- let timestamp = Date.now();
599
- let content = line.replace(/\r$/, '');
838
+ if (currentSize > state.lastSize) {
839
+ const fd = fsModule.openSync(state.logFilePath, 'r');
840
+ const buffer = Buffer.alloc(currentSize - state.lastSize);
841
+ fsModule.readSync(fd, buffer, 0, buffer.length, state.lastSize);
842
+ fsModule.closeSync(fd);
600
843
 
601
- const timestampMatch = content.match(/^\[(\d{13})\](.*)$/);
602
- if (timestampMatch) {
603
- timestamp = parseInt(timestampMatch[1], 10);
604
- content = timestampMatch[2];
605
- }
844
+ onNewContent(buffer.toString('utf-8'));
845
+ state.lastSize = currentSize;
846
+ }
847
+ } catch (err) {
848
+ const error = /** @type {Error} */ (err);
849
+ console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${error.message}`);
850
+ }
851
+ }
606
852
 
607
- // Skip known noise patterns (footer, separators, metadata)
608
- if (
609
- content.startsWith('===') ||
610
- content.startsWith('Finished:') ||
611
- content.startsWith('Exit code:') ||
612
- (content.includes('"type":"system"') && content.includes('"subtype":"init"'))
613
- ) {
614
- return;
615
- }
853
+ function stripAnsiCodes(value) {
854
+ const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
855
+ return value.replace(ansiPattern, '');
856
+ }
616
857
 
617
- // Determine if this is JSON or plain text output
618
- const isJson = content.trim().startsWith('{');
619
- let isValidJson = false;
858
+ function parseStatusFlags(cleanStdout) {
859
+ return {
860
+ isCompleted: /Status:\s+completed/i.test(cleanStdout),
861
+ isFailed: /Status:\s+failed/i.test(cleanStdout),
862
+ isStale: /Status:\s+stale/i.test(cleanStdout),
863
+ };
864
+ }
620
865
 
621
- if (isJson) {
622
- try {
623
- JSON.parse(content);
624
- isValidJson = true;
625
- } catch {
626
- // Looks like JSON but isn't valid - treat as text
627
- }
628
- }
866
+ function determineStaleSuccess({ agent, output, providerName, taskId }) {
867
+ if (!output) {
868
+ return false;
869
+ }
629
870
 
630
- // Accumulate output (JSON for parsing, text for human visibility)
631
- output += content + '\n';
871
+ const hasStructuredOutput = /"structured_output"\s*:/.test(output);
872
+ const hasSuccessResult = /"subtype"\s*:\s*"success"/.test(output);
873
+ let hasParsedOutput = false;
632
874
 
633
- // Update liveness timestamp
634
- agent.lastOutputTime = Date.now();
875
+ try {
876
+ const { extractJsonFromOutput } = require('./output-extraction');
877
+ hasParsedOutput = !!extractJsonFromOutput(output, providerName);
878
+ } catch {
879
+ // Ignore extraction errors - fallback to other signals
880
+ }
635
881
 
636
- agent._publish({
637
- topic: 'AGENT_OUTPUT',
638
- receiver: 'broadcast',
639
- timestamp, // Use the actual timestamp from when output was produced
640
- content: {
641
- text: content,
642
- data: {
643
- type: isValidJson ? 'json' : 'text',
644
- line: content,
645
- agent: agent.id,
646
- role: agent.role,
647
- iteration: agent.iteration,
648
- provider: providerName,
649
- },
882
+ const success = hasStructuredOutput || hasSuccessResult || hasParsedOutput;
883
+ if (!agent.quiet) {
884
+ agent._log(
885
+ `[Agent ${agent.id}] Task ${taskId} is stale - recovered as ${success ? 'SUCCESS' : 'FAILURE'} based on output analysis`
886
+ );
887
+ }
888
+
889
+ return success;
890
+ }
891
+
892
+ function finalizeLogFollow(agent, state) {
893
+ if (state.pollInterval) {
894
+ clearInterval(state.pollInterval);
895
+ }
896
+ if (state.statusCheckInterval) {
897
+ clearInterval(state.statusCheckInterval);
898
+ }
899
+ agent.currentTask = null;
900
+ }
901
+
902
+ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve }) {
903
+ if (!error) {
904
+ return false;
905
+ }
906
+
907
+ state.consecutiveExecFailures++;
908
+ if (state.consecutiveExecFailures < MAX_STATUS_FAILURES) {
909
+ return true;
910
+ }
911
+
912
+ console.error(
913
+ `[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_STATUS_FAILURES} times consecutively! STOPPING.`
914
+ );
915
+ console.error(` Command: ${ctPath} status ${taskId}`);
916
+ console.error(` Error: ${error.message}`);
917
+ console.error(` Stderr: ${stderr || 'none'}`);
918
+ console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
919
+
920
+ if (!state.resolved) {
921
+ state.resolved = true;
922
+ finalizeLogFollow(agent, state);
923
+
924
+ agent._publish({
925
+ topic: 'AGENT_ERROR',
926
+ receiver: 'broadcast',
927
+ content: {
928
+ text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
929
+ data: {
930
+ taskId,
931
+ error: 'polling_timeout',
932
+ attempts: state.consecutiveExecFailures,
933
+ role: agent.role,
934
+ iteration: agent.iteration,
650
935
  },
651
- });
652
- };
936
+ },
937
+ });
653
938
 
654
- // Process new content by splitting into complete lines
655
- const processNewContent = (content) => {
656
- // Add to buffer
657
- lineBuffer += content;
939
+ resolve({
940
+ success: false,
941
+ output: state.output,
942
+ error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
943
+ });
944
+ }
658
945
 
659
- // Split by newlines
660
- const lines = lineBuffer.split('\n');
946
+ return true;
947
+ }
661
948
 
662
- // Process all complete lines (all except last, which might be incomplete)
663
- for (let i = 0; i < lines.length - 1; i++) {
664
- broadcastLine(lines[i]);
665
- }
949
+ function handleStatusCompletion({
950
+ agent,
951
+ taskId,
952
+ providerName,
953
+ state,
954
+ stdout,
955
+ pollLogFile,
956
+ resolve,
957
+ }) {
958
+ const cleanStdout = stripAnsiCodes(stdout);
959
+ const { isCompleted, isFailed, isStale } = parseStatusFlags(cleanStdout);
960
+
961
+ if (!isCompleted && !isFailed && !isStale) {
962
+ return false;
963
+ }
666
964
 
667
- // Keep last line in buffer (might be incomplete)
668
- lineBuffer = lines[lines.length - 1];
669
- };
670
-
671
- // Poll the log file for new content
672
- const pollLogFile = () => {
673
- // If we don't have log path yet, try to get it
674
- if (!logFilePath) {
675
- try {
676
- logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
677
- encoding: 'utf-8',
678
- }).trim();
679
- agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
680
- } catch {
681
- return; // Not ready yet
682
- }
683
- }
965
+ pollLogFile();
684
966
 
685
- // Check if file exists
686
- if (!fsModule.existsSync(logFilePath)) {
687
- return; // File not created yet
688
- }
967
+ let success = isCompleted;
968
+ if (isStale) {
969
+ success = determineStaleSuccess({ agent, output: state.output, providerName, taskId });
970
+ }
689
971
 
690
- try {
691
- const stats = fsModule.statSync(logFilePath);
692
- const currentSize = stats.size;
693
-
694
- if (currentSize > lastSize) {
695
- // Read new content
696
- const fd = fsModule.openSync(logFilePath, 'r');
697
- const buffer = Buffer.alloc(currentSize - lastSize);
698
- fsModule.readSync(fd, buffer, 0, buffer.length, lastSize);
699
- fsModule.closeSync(fd);
700
-
701
- const newContent = buffer.toString('utf-8');
702
- // Process new content line-by-line
703
- processNewContent(newContent);
704
- lastSize = currentSize;
705
- }
706
- } catch (err) {
707
- // File might have been deleted or locked
708
- console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${err.message}`);
709
- }
710
- };
711
-
712
- // Start polling log file (every 300ms for responsive streaming)
713
- pollInterval = setInterval(pollLogFile, 300);
714
-
715
- // Poll ct status to know when task is complete
716
- // Track consecutive failures for debugging stuck clusters
717
- let consecutiveExecFailures = 0;
718
- const MAX_CONSECUTIVE_FAILURES = 30; // 30 seconds of failures = log warning
719
-
720
- statusCheckInterval = setInterval(() => {
721
- exec(`${ctPath} status ${taskId}`, (error, stdout, stderr) => {
722
- if (resolved) return;
723
-
724
- // Track exec failures - if status command keeps failing, something is wrong
725
- if (error) {
726
- consecutiveExecFailures++;
727
- if (consecutiveExecFailures >= MAX_CONSECUTIVE_FAILURES) {
728
- console.error(
729
- `[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_CONSECUTIVE_FAILURES} times consecutively! STOPPING.`
730
- );
731
- console.error(` Command: ${ctPath} status ${taskId}`);
732
- console.error(` Error: ${error.message}`);
733
- console.error(` Stderr: ${stderr || 'none'}`);
734
- console.error(
735
- ` This may indicate zeroshot is not in PATH or task storage is corrupted.`
736
- );
972
+ setTimeout(() => {
973
+ if (state.resolved) return;
974
+ state.resolved = true;
737
975
 
738
- // Stop polling and resolve with failure
739
- if (!resolved) {
740
- resolved = true;
741
- clearInterval(pollInterval);
742
- clearInterval(statusCheckInterval);
743
- agent.currentTask = null;
744
-
745
- // Publish error for orchestrator/resume
746
- agent._publish({
747
- topic: 'AGENT_ERROR',
748
- receiver: 'broadcast',
749
- content: {
750
- text: `Task ${taskId} polling failed after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`,
751
- data: {
752
- taskId,
753
- error: 'polling_timeout',
754
- attempts: consecutiveExecFailures,
755
- role: agent.role,
756
- iteration: agent.iteration,
757
- },
758
- },
759
- });
760
-
761
- resolve({
762
- success: false,
763
- output,
764
- error: `Status polling failed ${MAX_CONSECUTIVE_FAILURES} times - task may not exist`,
765
- });
766
- }
767
- return;
768
- }
769
- return; // Keep polling - might be transient
770
- }
976
+ finalizeLogFollow(agent, state);
771
977
 
772
- // Reset failure counter on success
773
- consecutiveExecFailures = 0;
774
-
775
- // Check for completion/failure status
776
- // Strip ANSI codes in case chalk outputs them (shouldn't in non-TTY, but be safe)
777
- // Use RegExp constructor to avoid ESLint no-control-regex false positive
778
- const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
779
- const cleanStdout = stdout.replace(ansiPattern, '');
780
- // Use flexible whitespace matching in case spacing changes
781
- const isCompleted = /Status:\s+completed/i.test(cleanStdout);
782
- const isFailed = /Status:\s+failed/i.test(cleanStdout);
783
- // BUGFIX: Handle "stale (process died)" status - watcher died before updating status
784
- // Check if task produced a successful result (structured_output in log file)
785
- const isStale = /Status:\s+stale/i.test(cleanStdout);
786
-
787
- if (isCompleted || isFailed || isStale) {
788
- // CRITICAL: Read final log content BEFORE checking output
789
- // Fixes race where status flips to stale before log polling catches up
790
- pollLogFile();
791
-
792
- // For stale tasks, check log file for successful result
793
- let success = isCompleted;
794
- if (isStale && output) {
795
- // Look for structured_output in accumulated output - indicates success
796
- const hasStructuredOutput = /"structured_output"\s*:/.test(output);
797
- const hasSuccessResult = /"subtype"\s*:\s*"success"/.test(output);
798
- let hasParsedOutput = false;
799
- try {
800
- const { extractJsonFromOutput } = require('./output-extraction');
801
- hasParsedOutput = !!extractJsonFromOutput(output, providerName);
802
- } catch {
803
- // Ignore extraction errors - fallback to other signals
804
- }
805
- success = hasStructuredOutput || hasSuccessResult || hasParsedOutput;
806
- if (!agent.quiet) {
807
- agent._log(
808
- `[Agent ${agent.id}] Task ${taskId} is stale - recovered as ${success ? 'SUCCESS' : 'FAILURE'} based on output analysis`
809
- );
810
- }
811
- }
978
+ const errorContext = !success
979
+ ? extractErrorContext({ output: state.output, statusOutput: stdout, taskId })
980
+ : null;
812
981
 
813
- // Clean up and resolve
814
- setTimeout(() => {
815
- if (resolved) return;
816
- resolved = true;
817
-
818
- clearInterval(pollInterval);
819
- clearInterval(statusCheckInterval);
820
- agent.currentTask = null;
821
-
822
- // Extract error context using shared helper
823
- const errorContext = !success
824
- ? extractErrorContext({ output, statusOutput: stdout, taskId })
825
- : null;
826
-
827
- resolve({
828
- success,
829
- output,
830
- error: errorContext,
831
- tokenUsage: extractTokenUsage(output, providerName),
832
- });
833
- }, 500);
982
+ resolve({
983
+ success,
984
+ output: state.output,
985
+ error: errorContext,
986
+ tokenUsage: extractTokenUsage(state.output, providerName),
987
+ });
988
+ }, 500);
989
+
990
+ return true;
991
+ }
992
+
993
+ function buildKillHandler({ agent, state, providerName, resolve }) {
994
+ return {
995
+ kill: (reason = 'Task killed') => {
996
+ if (state.resolved) return;
997
+ state.resolved = true;
998
+ finalizeLogFollow(agent, state);
999
+ agent._stopLivenessCheck();
1000
+ resolve({
1001
+ success: false,
1002
+ output: state.output,
1003
+ error: reason,
1004
+ tokenUsage: extractTokenUsage(state.output, providerName),
1005
+ });
1006
+ },
1007
+ };
1008
+ }
1009
+
1010
+ function createLogFollower({ agent, taskId, fsModule, ctPath, providerName }) {
1011
+ return new Promise((resolve) => {
1012
+ const state = createLogFollowState();
1013
+
1014
+ state.logFilePath = lookupLogFilePath(ctPath, taskId);
1015
+ if (state.logFilePath) {
1016
+ agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
1017
+ } else {
1018
+ agent._log(`⏳ Agent ${agent.id}: Waiting for log file...`);
1019
+ }
1020
+
1021
+ const broadcastLine = (line) => broadcastAgentLine({ agent, providerName, state, line });
1022
+ const processNewContent = (content) => appendContentToBuffer(state, content, broadcastLine);
1023
+ const pollLogFile = () =>
1024
+ pollLogFileForUpdates({
1025
+ agent,
1026
+ fsModule,
1027
+ ctPath,
1028
+ taskId,
1029
+ state,
1030
+ onNewContent: processNewContent,
1031
+ });
1032
+
1033
+ state.pollInterval = setInterval(pollLogFile, 300);
1034
+
1035
+ state.statusCheckInterval = setInterval(() => {
1036
+ exec(`${ctPath} status ${taskId}`, { timeout: 5000 }, (error, stdout, stderr) => {
1037
+ if (state.resolved) return;
1038
+
1039
+ if (handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve })) {
1040
+ return;
834
1041
  }
1042
+
1043
+ state.consecutiveExecFailures = 0;
1044
+ handleStatusCompletion({
1045
+ agent,
1046
+ taskId,
1047
+ providerName,
1048
+ state,
1049
+ stdout,
1050
+ pollLogFile,
1051
+ resolve,
1052
+ });
835
1053
  });
836
1054
  }, 1000);
837
1055
 
838
- // Store cleanup function for kill
839
- // CRITICAL: Must reject promise to avoid orphaned promise that hangs forever
840
- agent.currentTask = {
841
- kill: (reason = 'Task killed') => {
842
- if (resolved) return;
843
- resolved = true;
844
- clearInterval(pollInterval);
845
- clearInterval(statusCheckInterval);
846
- agent._stopLivenessCheck();
847
- // BUGFIX: Resolve with failure instead of orphaning the promise
848
- // This allows the caller to handle the kill gracefully
849
- resolve({
850
- success: false,
851
- output,
852
- error: reason,
853
- tokenUsage: extractTokenUsage(output, providerName),
854
- });
855
- },
856
- };
857
-
858
- // REMOVED: Task timeout disabled - tasks run until completion or explicit kill
859
- // Tasks should run until:
860
- // - Completion
861
- // - Explicit kill
862
- // - External error (rate limit, API failure)
863
- //
864
- // setTimeout(() => {
865
- // if (resolved) return;
866
- // resolved = true;
867
- //
868
- // clearInterval(pollInterval);
869
- // clearInterval(statusCheckInterval);
870
- // agent._stopLivenessCheck();
871
- // agent.currentTask = null;
872
- // const timeoutMinutes = Math.round(agent.timeout / 60000);
873
- // reject(new Error(`Task timed out after ${timeoutMinutes} minutes`));
874
- // }, agent.timeout);
1056
+ agent.currentTask = buildKillHandler({ agent, state, providerName, resolve });
875
1057
  });
876
1058
  }
877
1059
 
1060
+ /**
1061
+ * Follow claude-zeroshots logs until completion, streaming to message bus
1062
+ * Reads log file directly for reliable streaming
1063
+ * @param {Object} agent - Agent instance
1064
+ * @param {String} taskId - Task ID to follow
1065
+ * @returns {Promise<Object>} Result object { success, output, error }
1066
+ */
1067
+ function followClaudeTaskLogs(agent, taskId) {
1068
+ const fsModule = require('fs');
1069
+ const ctPath = getClaudeTasksPath();
1070
+ const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
1071
+
1072
+ return createLogFollower({ agent, taskId, fsModule, ctPath, providerName });
1073
+ }
1074
+
878
1075
  /**
879
1076
  * Get path to claude-zeroshots executable
880
1077
  * @returns {String} Path to zeroshot command
@@ -963,16 +1160,15 @@ async function spawnClaudeTaskIsolated(agent, context) {
963
1160
  command.push(finalContext);
964
1161
 
965
1162
  // STEP 1: Spawn task and extract task ID (same as non-isolated mode)
1163
+ // Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
1164
+ const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
1165
+ // Note: Auth env vars are injected by IsolationManager, we only need model mapping here
1166
+ const isolatedEnv =
1167
+ providerName === 'claude' ? buildClaudeEnv(modelSpec, { includeAuth: false }) : {};
1168
+
966
1169
  const taskId = await new Promise((resolve, reject) => {
967
1170
  const proc = manager.spawnInContainer(clusterId, command, {
968
- env:
969
- providerName === 'claude'
970
- ? {
971
- ANTHROPIC_MODEL: modelSpec?.model,
972
- // Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
973
- ZEROSHOT_BLOCK_ASK_USER: '1',
974
- }
975
- : {},
1171
+ env: isolatedEnv,
976
1172
  });
977
1173
 
978
1174
  // Track PID for resource monitoring
@@ -981,6 +1177,20 @@ async function spawnClaudeTaskIsolated(agent, context) {
981
1177
 
982
1178
  let stdout = '';
983
1179
  let stderr = '';
1180
+ let resolved = false;
1181
+
1182
+ // CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
1183
+ const spawnTimeout = setTimeout(() => {
1184
+ if (resolved) return;
1185
+ resolved = true;
1186
+ proc.kill('SIGKILL');
1187
+ reject(
1188
+ new Error(
1189
+ `Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
1190
+ `stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
1191
+ )
1192
+ );
1193
+ }, SPAWN_TIMEOUT_MS);
984
1194
 
985
1195
  proc.stdout.on('data', (data) => {
986
1196
  stdout += data.toString();
@@ -991,6 +1201,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
991
1201
  });
992
1202
 
993
1203
  proc.on('close', (code, signal) => {
1204
+ clearTimeout(spawnTimeout);
1205
+ if (resolved) return;
1206
+ resolved = true;
994
1207
  // Handle process killed by signal
995
1208
  if (signal) {
996
1209
  reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
@@ -1024,6 +1237,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
1024
1237
  });
1025
1238
 
1026
1239
  proc.on('error', (error) => {
1240
+ clearTimeout(spawnTimeout);
1241
+ if (resolved) return;
1242
+ resolved = true;
1027
1243
  reject(error);
1028
1244
  });
1029
1245
  });
@@ -1057,6 +1273,196 @@ async function spawnClaudeTaskIsolated(agent, context) {
1057
1273
  * - Status checks reduced to every 2 seconds (not every poll)
1058
1274
  * - Result: 10-20% overall latency reduction
1059
1275
  */
1276
+ function createIsolatedLogState() {
1277
+ return {
1278
+ taskExited: false,
1279
+ fullOutput: '',
1280
+ tailProcess: null,
1281
+ statusCheckInterval: null,
1282
+ lineBuffer: '',
1283
+ };
1284
+ }
1285
+
1286
+ function buildIsolatedCleanup(state) {
1287
+ return () => {
1288
+ if (state.tailProcess) {
1289
+ try {
1290
+ state.tailProcess.kill('SIGTERM');
1291
+ } catch {
1292
+ // Ignore - process may already be dead
1293
+ }
1294
+ state.tailProcess = null;
1295
+ }
1296
+ if (state.statusCheckInterval) {
1297
+ clearInterval(state.statusCheckInterval);
1298
+ state.statusCheckInterval = null;
1299
+ }
1300
+ };
1301
+ }
1302
+
1303
+ function broadcastIsolatedLine({ agent, providerName, taskId, line }) {
1304
+ const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
1305
+ const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
1306
+ const content = timestampMatch ? timestampMatch[2] : line;
1307
+
1308
+ agent.messageBus.publish({
1309
+ cluster_id: agent.cluster.id,
1310
+ topic: 'AGENT_OUTPUT',
1311
+ sender: agent.id,
1312
+ content: {
1313
+ data: {
1314
+ line: content,
1315
+ taskId,
1316
+ iteration: agent.iteration,
1317
+ provider: providerName,
1318
+ },
1319
+ },
1320
+ timestamp,
1321
+ });
1322
+
1323
+ agent.lastOutputTime = Date.now();
1324
+ }
1325
+
1326
+ function appendIsolatedContent(state, content, onLine) {
1327
+ state.lineBuffer += content;
1328
+ const lines = state.lineBuffer.split('\n');
1329
+
1330
+ for (let i = 0; i < lines.length - 1; i++) {
1331
+ if (lines[i].trim()) {
1332
+ onLine(lines[i]);
1333
+ }
1334
+ }
1335
+
1336
+ state.lineBuffer = lines[lines.length - 1];
1337
+ }
1338
+
1339
+ function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
1340
+ state.tailProcess = manager.spawnInContainer(clusterId, [
1341
+ 'sh',
1342
+ '-c',
1343
+ `while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
1344
+ ]);
1345
+
1346
+ state.tailProcess.stdout.on('data', (data) => {
1347
+ const chunk = data.toString();
1348
+ state.fullOutput += chunk;
1349
+ appendIsolatedContent(state, chunk, onLine);
1350
+ });
1351
+
1352
+ state.tailProcess.stderr.on('data', (data) => {
1353
+ const msg = data.toString().trim();
1354
+ if (msg && !msg.includes('file truncated')) {
1355
+ agent._log(`[${agent.id}] tail stderr: ${msg}`);
1356
+ }
1357
+ });
1358
+
1359
+ state.tailProcess.on('close', (exitCode) => {
1360
+ if (!state.taskExited) {
1361
+ agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
1362
+ }
1363
+ });
1364
+
1365
+ state.tailProcess.on('error', (err) => {
1366
+ agent._log(`[${agent.id}] tail process error: ${err.message}`);
1367
+ });
1368
+ }
1369
+
1370
+ async function checkIsolatedStatus({
1371
+ agent,
1372
+ manager,
1373
+ clusterId,
1374
+ logFilePath,
1375
+ taskId,
1376
+ providerName,
1377
+ state,
1378
+ cleanup,
1379
+ resolve,
1380
+ onLine,
1381
+ }) {
1382
+ if (state.taskExited) return;
1383
+
1384
+ const statusResult = await manager.execInContainer(clusterId, [
1385
+ 'sh',
1386
+ '-c',
1387
+ `zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
1388
+ ]);
1389
+
1390
+ const statusOutput = statusResult.stdout;
1391
+ const isSuccess = /Status:\s+completed/i.test(statusOutput);
1392
+ const isError = /Status:\s+failed/i.test(statusOutput);
1393
+ const isNotFound = statusOutput.includes('not_found');
1394
+
1395
+ if (!isSuccess && !isError && !isNotFound) {
1396
+ return;
1397
+ }
1398
+
1399
+ state.taskExited = true;
1400
+ await new Promise((r) => setTimeout(r, 200));
1401
+
1402
+ const finalReadResult = await manager.execInContainer(clusterId, [
1403
+ 'sh',
1404
+ '-c',
1405
+ `cat "${logFilePath}" 2>/dev/null || echo ""`,
1406
+ ]);
1407
+
1408
+ if (finalReadResult.code === 0 && finalReadResult.stdout) {
1409
+ state.fullOutput = finalReadResult.stdout;
1410
+ const remainingLines = state.fullOutput.split('\n');
1411
+ for (const line of remainingLines) {
1412
+ if (line.trim()) {
1413
+ onLine(line);
1414
+ }
1415
+ }
1416
+ }
1417
+
1418
+ cleanup();
1419
+
1420
+ const success = isSuccess && !isError;
1421
+ const errorContext = !success
1422
+ ? extractErrorContext({ output: state.fullOutput, taskId, isNotFound })
1423
+ : null;
1424
+ const parsedResult = await agent._parseResultOutput(state.fullOutput);
1425
+
1426
+ resolve({
1427
+ success,
1428
+ output: state.fullOutput,
1429
+ taskId,
1430
+ result: parsedResult,
1431
+ error: errorContext,
1432
+ tokenUsage: extractTokenUsage(state.fullOutput, providerName),
1433
+ });
1434
+ }
1435
+
1436
+ function startIsolatedStatusChecks({
1437
+ agent,
1438
+ manager,
1439
+ clusterId,
1440
+ logFilePath,
1441
+ taskId,
1442
+ providerName,
1443
+ state,
1444
+ cleanup,
1445
+ resolve,
1446
+ onLine,
1447
+ }) {
1448
+ state.statusCheckInterval = setInterval(() => {
1449
+ checkIsolatedStatus({
1450
+ agent,
1451
+ manager,
1452
+ clusterId,
1453
+ logFilePath,
1454
+ taskId,
1455
+ providerName,
1456
+ state,
1457
+ cleanup,
1458
+ resolve,
1459
+ onLine,
1460
+ }).catch((statusErr) => {
1461
+ agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
1462
+ });
1463
+ }, 2000);
1464
+ }
1465
+
1060
1466
  function followClaudeTaskLogsIsolated(agent, taskId) {
1061
1467
  const { isolation } = agent;
1062
1468
  if (!isolation?.manager) {
@@ -1068,70 +1474,10 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
1068
1474
  const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
1069
1475
 
1070
1476
  return new Promise((resolve, reject) => {
1071
- let taskExited = false;
1072
- let fullOutput = '';
1073
- let tailProcess = null;
1074
- let statusCheckInterval = null;
1075
- let lineBuffer = '';
1076
-
1077
- // Cleanup function - kill tail process and clear intervals
1078
- const cleanup = () => {
1079
- if (tailProcess) {
1080
- try {
1081
- tailProcess.kill('SIGTERM');
1082
- } catch {
1083
- // Ignore - process may already be dead
1084
- }
1085
- tailProcess = null;
1086
- }
1087
- if (statusCheckInterval) {
1088
- clearInterval(statusCheckInterval);
1089
- statusCheckInterval = null;
1090
- }
1091
- };
1092
-
1093
- // Broadcast line helper (same as non-isolated mode)
1094
- const broadcastLine = (line) => {
1095
- const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
1096
- const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
1097
- const content = timestampMatch ? timestampMatch[2] : line;
1098
-
1099
- agent.messageBus.publish({
1100
- cluster_id: agent.cluster.id,
1101
- topic: 'AGENT_OUTPUT',
1102
- sender: agent.id,
1103
- content: {
1104
- data: {
1105
- line: content,
1106
- taskId,
1107
- iteration: agent.iteration,
1108
- provider: providerName,
1109
- },
1110
- },
1111
- timestamp,
1112
- });
1113
-
1114
- // Update last output time for liveness tracking
1115
- agent.lastOutputTime = Date.now();
1116
- };
1117
-
1118
- // Process new content by splitting into complete lines
1119
- const processNewContent = (content) => {
1120
- lineBuffer += content;
1121
- const lines = lineBuffer.split('\n');
1122
-
1123
- // Process all complete lines (all except last, which might be incomplete)
1124
- for (let i = 0; i < lines.length - 1; i++) {
1125
- if (lines[i].trim()) {
1126
- broadcastLine(lines[i]);
1127
- }
1128
- }
1129
-
1130
- // Keep last line in buffer (might be incomplete)
1131
- lineBuffer = lines[lines.length - 1];
1132
- };
1477
+ const state = createIsolatedLogState();
1478
+ const cleanup = buildIsolatedCleanup(state);
1479
+ const onLine = (line) => broadcastIsolatedLine({ agent, providerName, taskId, line });
1133
1480
 
1134
- // Get log file path from zeroshot CLI inside container
1135
1481
  manager
1136
1482
  .execInContainer(clusterId, ['sh', '-c', `zeroshot get-log-path ${taskId}`])
1137
1483
  .then(({ stdout, stderr, code }) => {
@@ -1150,117 +1496,31 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
1150
1496
 
1151
1497
  agent._log(`[${agent.id}] Following isolated task logs (streaming): ${logFilePath}`);
1152
1498
 
1153
- // Start persistent tail -f stream
1154
- // Uses spawnInContainer() which creates a single docker exec process
1155
- // that streams output in real-time (no polling overhead)
1156
- tailProcess = manager.spawnInContainer(clusterId, [
1157
- 'sh',
1158
- '-c',
1159
- // Wait for file to exist, then tail -f from beginning
1160
- // The -F flag handles file recreation (rotation)
1161
- `while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
1162
- ]);
1163
-
1164
- // Stream stdout directly - lines arrive as they're written
1165
- tailProcess.stdout.on('data', (data) => {
1166
- const chunk = data.toString();
1167
- fullOutput += chunk;
1168
- processNewContent(chunk);
1499
+ startIsolatedTail({
1500
+ agent,
1501
+ manager,
1502
+ clusterId,
1503
+ logFilePath,
1504
+ state,
1505
+ onLine,
1169
1506
  });
1170
1507
 
1171
- // Log stderr but don't fail (tail might emit warnings)
1172
- tailProcess.stderr.on('data', (data) => {
1173
- const msg = data.toString().trim();
1174
- if (msg && !msg.includes('file truncated')) {
1175
- agent._log(`[${agent.id}] tail stderr: ${msg}`);
1176
- }
1177
- });
1178
-
1179
- // Handle tail process exit (shouldn't happen unless killed)
1180
- tailProcess.on('close', (exitCode) => {
1181
- if (!taskExited) {
1182
- agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
1183
- }
1184
- });
1185
-
1186
- tailProcess.on('error', (err) => {
1187
- agent._log(`[${agent.id}] tail process error: ${err.message}`);
1508
+ startIsolatedStatusChecks({
1509
+ agent,
1510
+ manager,
1511
+ clusterId,
1512
+ logFilePath,
1513
+ taskId,
1514
+ providerName,
1515
+ state,
1516
+ cleanup,
1517
+ resolve,
1518
+ onLine,
1188
1519
  });
1189
1520
 
1190
- // Check task status periodically (every 2 seconds - much less frequent than polling)
1191
- // This is the only remaining docker exec - but now at 2s intervals instead of 500ms
1192
- statusCheckInterval = setInterval(async () => {
1193
- if (taskExited) return;
1194
-
1195
- try {
1196
- const statusResult = await manager.execInContainer(clusterId, [
1197
- 'sh',
1198
- '-c',
1199
- `zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
1200
- ]);
1201
-
1202
- const statusOutput = statusResult.stdout;
1203
- const isSuccess = /Status:\s+completed/i.test(statusOutput);
1204
- const isError = /Status:\s+failed/i.test(statusOutput);
1205
- const isNotFound = statusOutput.includes('not_found');
1206
-
1207
- if (isSuccess || isError || isNotFound) {
1208
- taskExited = true;
1209
-
1210
- // Give tail a moment to flush remaining output
1211
- await new Promise((r) => setTimeout(r, 200));
1212
-
1213
- // Read final output to ensure we have everything
1214
- const finalReadResult = await manager.execInContainer(clusterId, [
1215
- 'sh',
1216
- '-c',
1217
- `cat "${logFilePath}" 2>/dev/null || echo ""`,
1218
- ]);
1219
-
1220
- if (finalReadResult.code === 0 && finalReadResult.stdout) {
1221
- fullOutput = finalReadResult.stdout;
1222
-
1223
- // Process any remaining content
1224
- const remainingLines = fullOutput.split('\n');
1225
- for (const line of remainingLines) {
1226
- if (line.trim()) {
1227
- broadcastLine(line);
1228
- }
1229
- }
1230
- }
1231
-
1232
- cleanup();
1233
-
1234
- // Determine success status
1235
- const success = isSuccess && !isError;
1236
-
1237
- // Extract error context using shared helper
1238
- const errorContext = !success
1239
- ? extractErrorContext({ output: fullOutput, taskId, isNotFound })
1240
- : null;
1241
-
1242
- // Parse result from output (async - may trigger reformatting)
1243
- const parsedResult = await agent._parseResultOutput(fullOutput);
1244
-
1245
- resolve({
1246
- success,
1247
- output: fullOutput,
1248
- taskId,
1249
- result: parsedResult,
1250
- error: errorContext,
1251
- tokenUsage: extractTokenUsage(fullOutput, providerName),
1252
- });
1253
- }
1254
- } catch (statusErr) {
1255
- // Log error but continue checking (transient failures are common)
1256
- agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
1257
- }
1258
- }, 2000); // Check every 2 seconds (was 500ms in polling mode)
1259
-
1260
- // Safety timeout (0 = no timeout, task runs until completion)
1261
1521
  if (agent.timeout > 0) {
1262
1522
  setTimeout(() => {
1263
- if (!taskExited) {
1523
+ if (!state.taskExited) {
1264
1524
  cleanup();
1265
1525
  reject(new Error(`Task ${taskId} timeout after ${agent.timeout}ms (isolated mode)`));
1266
1526
  }
@@ -1405,9 +1665,8 @@ function killTask(agent) {
1405
1665
  // Also kill the underlying zeroshot task if we have a task ID
1406
1666
  // This ensures the task process is stopped, not just our polling intervals
1407
1667
  if (agent.currentTaskId) {
1408
- const { exec } = require('child_process');
1409
1668
  const ctPath = getClaudeTasksPath();
1410
- exec(`${ctPath} task kill ${agent.currentTaskId}`, (error) => {
1669
+ exec(`${ctPath} task kill ${agent.currentTaskId}`, { timeout: 10000 }, (error) => {
1411
1670
  if (error) {
1412
1671
  // Task may have already completed or been killed, ignore errors
1413
1672
  agent._log(`Note: Could not kill task ${agent.currentTaskId}: ${error.message}`);