@covibes/zeroshot 5.3.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -14
- package/cli/commands/providers.js +8 -9
- package/cli/index.js +3032 -2409
- package/cli/message-formatters-normal.js +28 -6
- package/cluster-templates/base-templates/debug-workflow.json +1 -1
- package/cluster-templates/base-templates/full-workflow.json +72 -188
- package/cluster-templates/base-templates/worker-validator.json +59 -3
- package/cluster-templates/conductor-bootstrap.json +4 -4
- package/lib/docker-config.js +8 -0
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +10 -7
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-names.js +2 -1
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +161 -63
- package/package.json +7 -2
- package/scripts/setup-merge-queue.sh +170 -0
- package/src/agent/agent-config.js +135 -82
- package/src/agent/agent-context-builder.js +297 -188
- package/src/agent/agent-hook-executor.js +310 -113
- package/src/agent/agent-lifecycle.js +385 -325
- package/src/agent/agent-stuck-detector.js +7 -7
- package/src/agent/agent-task-executor.js +824 -565
- package/src/agent/output-extraction.js +41 -24
- package/src/agent/output-reformatter.js +1 -1
- package/src/agent/schema-utils.js +108 -73
- package/src/agent-wrapper.js +10 -1
- package/src/agents/git-pusher-template.js +285 -0
- package/src/claude-task-runner.js +85 -34
- package/src/config-validator.js +922 -657
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +289 -199
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +273 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +179 -0
- package/src/issue-providers/gitlab-provider.js +241 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +239 -0
- package/src/ledger.js +22 -5
- package/src/lib/safe-exec.js +88 -0
- package/src/orchestrator.js +1107 -811
- package/src/preflight.js +313 -159
- package/src/process-metrics.js +98 -56
- package/src/providers/anthropic/cli-builder.js +53 -25
- package/src/providers/anthropic/index.js +72 -2
- package/src/providers/anthropic/output-parser.js +32 -14
- package/src/providers/base-provider.js +70 -0
- package/src/providers/capabilities.js +9 -0
- package/src/providers/google/output-parser.js +47 -38
- package/src/providers/index.js +19 -3
- package/src/providers/openai/cli-builder.js +14 -3
- package/src/providers/openai/index.js +1 -0
- package/src/providers/openai/output-parser.js +44 -30
- package/src/providers/opencode/cli-builder.js +42 -0
- package/src/providers/opencode/index.js +103 -0
- package/src/providers/opencode/models.js +55 -0
- package/src/providers/opencode/output-parser.js +122 -0
- package/src/schemas/sub-cluster.js +68 -39
- package/src/status-footer.js +94 -48
- package/src/sub-cluster-wrapper.js +76 -35
- package/src/template-resolver.js +12 -9
- package/src/tui/data-poller.js +123 -99
- package/src/tui/formatters.js +4 -3
- package/src/tui/keybindings.js +259 -318
- package/src/tui/renderer.js +11 -21
- package/task-lib/attachable-watcher.js +118 -81
- package/task-lib/claude-recovery.js +61 -24
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/list.js +2 -2
- package/task-lib/commands/logs.js +231 -189
- package/task-lib/commands/schedules.js +111 -61
- package/task-lib/config.js +0 -2
- package/task-lib/runner.js +84 -27
- package/task-lib/scheduler.js +1 -1
- package/task-lib/store.js +467 -168
- package/task-lib/tui/formatters.js +94 -45
- package/task-lib/tui/renderer.js +13 -3
- package/task-lib/tui.js +73 -32
- package/task-lib/watcher.js +128 -90
- package/src/agents/git-pusher-agent.json +0 -20
- package/src/github.js +0 -139
|
@@ -14,11 +14,43 @@ const { spawn } = require('child_process');
|
|
|
14
14
|
const path = require('path');
|
|
15
15
|
const fs = require('fs');
|
|
16
16
|
const os = require('os');
|
|
17
|
+
const { exec, execSync } = require('../lib/safe-exec'); // Enforces timeouts - prevents infinite hangs
|
|
17
18
|
const { getProvider, parseChunkWithProvider } = require('../providers');
|
|
19
|
+
const { getTask } = require('../../task-lib/store.js');
|
|
20
|
+
const { loadSettings } = require('../../lib/settings.js');
|
|
21
|
+
const { resolveClaudeAuth } = require('../../lib/settings/claude-auth.js');
|
|
18
22
|
|
|
19
23
|
// Schema utilities for normalizing LLM output
|
|
20
24
|
const { normalizeEnumValues } = require('./schema-utils');
|
|
21
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Build Claude-specific environment variables for task spawning
|
|
28
|
+
* Consolidates auth resolution and model mapping logic used by both isolated and non-isolated modes
|
|
29
|
+
* @param {Object} modelSpec - Model specification from agent
|
|
30
|
+
* @param {Object} [options] - Options
|
|
31
|
+
* @param {boolean} [options.includeAuth=true] - Include auth env vars (false for isolated mode where IsolationManager handles auth)
|
|
32
|
+
* @returns {Object} Environment variables to merge into spawn env
|
|
33
|
+
*/
|
|
34
|
+
function buildClaudeEnv(modelSpec, options = {}) {
|
|
35
|
+
const { includeAuth = true } = options;
|
|
36
|
+
const env = {};
|
|
37
|
+
|
|
38
|
+
if (includeAuth) {
|
|
39
|
+
const settings = loadSettings();
|
|
40
|
+
const authEnv = resolveClaudeAuth(settings);
|
|
41
|
+
Object.assign(env, authEnv);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (modelSpec?.model) {
|
|
45
|
+
env.ANTHROPIC_MODEL = modelSpec.model;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
49
|
+
env.ZEROSHOT_BLOCK_ASK_USER = '1';
|
|
50
|
+
|
|
51
|
+
return env;
|
|
52
|
+
}
|
|
53
|
+
|
|
22
54
|
/**
|
|
23
55
|
* Validate and sanitize error messages.
|
|
24
56
|
* Detects TypeScript type annotations that may have leaked into error storage.
|
|
@@ -38,12 +70,17 @@ function sanitizeErrorMessage(error) {
|
|
|
38
70
|
/^unknown$/i,
|
|
39
71
|
/^void$/i,
|
|
40
72
|
/^never$/i,
|
|
41
|
-
/^[A-Z][a-zA-Z]*\s*\|\s*(null|undefined)$/, // e.g., "Error | null"
|
|
42
|
-
/^[a-z]+(\s*\|\s*[a-z]+)+$/i, // e.g., "string | number | boolean"
|
|
73
|
+
/^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
|
|
43
74
|
];
|
|
44
75
|
|
|
76
|
+
const trimmedError = error.trim();
|
|
77
|
+
|
|
78
|
+
// Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
|
|
79
|
+
const unionParts = trimmedError.split(/\s*\|\s*/);
|
|
80
|
+
const isUnionType = unionParts.length > 1 && unionParts.every((p) => /^[a-z]+$/i.test(p));
|
|
81
|
+
|
|
45
82
|
for (const pattern of typeAnnotationPatterns) {
|
|
46
|
-
if (pattern.test(
|
|
83
|
+
if (pattern.test(trimmedError) || isUnionType) {
|
|
47
84
|
console.warn(
|
|
48
85
|
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${error}". ` +
|
|
49
86
|
`This indicates corrupted data. Replacing with generic error.`
|
|
@@ -101,16 +138,31 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
|
|
|
101
138
|
);
|
|
102
139
|
}
|
|
103
140
|
|
|
104
|
-
//
|
|
105
|
-
|
|
106
|
-
|
|
141
|
+
// NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
|
|
142
|
+
// If output is too verbose, that's a prompt problem - fix the prompts, not the data
|
|
143
|
+
const trimmedOutput = (output || '').trim();
|
|
144
|
+
if (!trimmedOutput) {
|
|
107
145
|
return sanitizeErrorMessage(
|
|
108
146
|
'Task failed with no output (check if task was interrupted or timed out)'
|
|
109
147
|
);
|
|
110
148
|
}
|
|
111
149
|
|
|
150
|
+
// Try to extract structured JSON from output first - it may contain the actual result
|
|
151
|
+
// even if the task was marked as "failed" due to timeout/stale status
|
|
152
|
+
try {
|
|
153
|
+
const { extractJsonFromOutput } = require('./output-extraction');
|
|
154
|
+
const extracted = extractJsonFromOutput(trimmedOutput);
|
|
155
|
+
if (extracted && typeof extracted === 'object') {
|
|
156
|
+
// If we found valid JSON, return it as the error context
|
|
157
|
+
// This preserves the actual agent output for downstream processing
|
|
158
|
+
return JSON.stringify(extracted);
|
|
159
|
+
}
|
|
160
|
+
} catch {
|
|
161
|
+
// Extraction failed, fall through to error pattern matching
|
|
162
|
+
}
|
|
163
|
+
|
|
112
164
|
// Extract non-JSON lines only (JSON lines contain "is_error": true which falsely matches)
|
|
113
|
-
const nonJsonLines =
|
|
165
|
+
const nonJsonLines = trimmedOutput
|
|
114
166
|
.split('\n')
|
|
115
167
|
.filter((line) => {
|
|
116
168
|
const trimmed = line.trim();
|
|
@@ -120,7 +172,7 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
|
|
|
120
172
|
.join('\n');
|
|
121
173
|
|
|
122
174
|
// Common error patterns - match against non-JSON content
|
|
123
|
-
const textToSearch = nonJsonLines ||
|
|
175
|
+
const textToSearch = nonJsonLines || trimmedOutput;
|
|
124
176
|
const errorPatterns = [
|
|
125
177
|
/Error:\s*(.+)/i,
|
|
126
178
|
/error:\s*(.+)/i,
|
|
@@ -132,12 +184,14 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
|
|
|
132
184
|
for (const pattern of errorPatterns) {
|
|
133
185
|
const match = textToSearch.match(pattern);
|
|
134
186
|
if (match) {
|
|
135
|
-
|
|
187
|
+
// Don't truncate - let the full error message through
|
|
188
|
+
return sanitizeErrorMessage(match[1]);
|
|
136
189
|
}
|
|
137
190
|
}
|
|
138
191
|
|
|
139
|
-
// No pattern matched -
|
|
140
|
-
|
|
192
|
+
// No pattern matched - return full output (no truncation)
|
|
193
|
+
// If this is too long, the solution is to make agents output less, not to corrupt data
|
|
194
|
+
return sanitizeErrorMessage(`Task failed. Output: ${trimmedOutput}`);
|
|
141
195
|
}
|
|
142
196
|
|
|
143
197
|
// Track if we've already ensured the AskUserQuestion hook is installed
|
|
@@ -339,14 +393,109 @@ function ensureDangerousGitHook() {
|
|
|
339
393
|
*/
|
|
340
394
|
async function spawnClaudeTask(agent, context) {
|
|
341
395
|
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
342
|
-
const modelSpec = agent
|
|
343
|
-
? agent._resolveModelSpec()
|
|
344
|
-
: { model: agent._selectModel() };
|
|
396
|
+
const modelSpec = resolveAgentModelSpec(agent);
|
|
345
397
|
|
|
346
398
|
const ctPath = getClaudeTasksPath();
|
|
347
399
|
const cwd = agent.config.cwd || process.cwd();
|
|
348
400
|
|
|
349
401
|
// Build zeroshot task run args.
|
|
402
|
+
// CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
|
|
403
|
+
// strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
|
|
404
|
+
// strictSchema=false uses stream-json with post-run validation (live logs but fragile)
|
|
405
|
+
const { desiredOutputFormat, runOutputFormat } = resolveOutputFormatConfig(agent);
|
|
406
|
+
const args = buildTaskRunArgs({
|
|
407
|
+
agent,
|
|
408
|
+
providerName,
|
|
409
|
+
modelSpec,
|
|
410
|
+
runOutputFormat,
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
// NOTE: maxRetries is handled by the agent wrapper's internal retry loop,
|
|
414
|
+
// not passed to the CLI. See _handleTrigger() for retry logic.
|
|
415
|
+
|
|
416
|
+
maybeLogStreamJsonNotice(agent, runOutputFormat);
|
|
417
|
+
|
|
418
|
+
// If schema enforcement is desired but we had to run stream-json for live logs,
|
|
419
|
+
// add explicit output instructions so the model still knows the required shape.
|
|
420
|
+
const finalContext = buildFinalContext({
|
|
421
|
+
agent,
|
|
422
|
+
context,
|
|
423
|
+
desiredOutputFormat,
|
|
424
|
+
runOutputFormat,
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
args.push(finalContext);
|
|
428
|
+
|
|
429
|
+
// MOCK SUPPORT: Use injected mock function if provided
|
|
430
|
+
if (agent.mockSpawnFn) {
|
|
431
|
+
return agent.mockSpawnFn(args, { context });
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// SAFETY: Fail hard if testMode=true but no mock (should be caught in constructor)
|
|
435
|
+
if (agent.testMode) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`AgentWrapper: testMode=true but attempting real Claude API call for agent '${agent.id}'. ` +
|
|
438
|
+
`This is a bug - mock should be set in constructor.`
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
// ISOLATION MODE: Run inside Docker container
|
|
443
|
+
if (agent.isolation?.enabled) {
|
|
444
|
+
return spawnClaudeTaskIsolated(agent, context);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// NON-ISOLATION MODE: For Claude, use user's existing Claude config
|
|
448
|
+
// AskUserQuestion blocking handled via:
|
|
449
|
+
// 1. Prompt injection (see agent-context-builder)
|
|
450
|
+
// 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
|
|
451
|
+
ensureProviderHooks(agent, providerName);
|
|
452
|
+
const spawnEnv = buildSpawnEnv(agent, providerName, modelSpec);
|
|
453
|
+
const taskId = await spawnTaskProcess({
|
|
454
|
+
agent,
|
|
455
|
+
ctPath,
|
|
456
|
+
args,
|
|
457
|
+
cwd,
|
|
458
|
+
spawnEnv,
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
|
|
462
|
+
|
|
463
|
+
// Wait for task to be registered in zeroshot storage (race condition fix)
|
|
464
|
+
await waitForTaskReady(agent, taskId);
|
|
465
|
+
|
|
466
|
+
// CRITICAL: Poll for REAL process PID from task store
|
|
467
|
+
// The watcher spawns the actual CLI and writes PID to SQLite asynchronously.
|
|
468
|
+
// We must poll because the watcher runs in a forked process.
|
|
469
|
+
const MAX_PID_POLLS = 30; // 3 seconds max
|
|
470
|
+
const PID_POLL_DELAY = 100;
|
|
471
|
+
let realPid = null;
|
|
472
|
+
|
|
473
|
+
for (let i = 0; i < MAX_PID_POLLS; i++) {
|
|
474
|
+
const taskInfo = getTask(taskId);
|
|
475
|
+
if (taskInfo?.pid) {
|
|
476
|
+
realPid = taskInfo.pid;
|
|
477
|
+
break;
|
|
478
|
+
}
|
|
479
|
+
await new Promise((r) => setTimeout(r, PID_POLL_DELAY));
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if (realPid) {
|
|
483
|
+
agent.processPid = realPid;
|
|
484
|
+
agent._publishLifecycle('PROCESS_SPAWNED', { pid: realPid });
|
|
485
|
+
agent._log(`📋 Agent ${agent.id}: Process PID: ${realPid}`);
|
|
486
|
+
} else {
|
|
487
|
+
agent._log(`⚠️ Agent ${agent.id}: PID not available (task may use non-standard watcher)`);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// Now follow the logs and stream output
|
|
491
|
+
return followClaudeTaskLogs(agent, taskId);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function resolveAgentModelSpec(agent) {
|
|
495
|
+
return agent._resolveModelSpec ? agent._resolveModelSpec() : { model: agent._selectModel() };
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
function resolveOutputFormatConfig(agent) {
|
|
350
499
|
// CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
|
|
351
500
|
// strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
|
|
352
501
|
// strictSchema=false uses stream-json with post-run validation (live logs but fragile)
|
|
@@ -356,6 +505,11 @@ async function spawnClaudeTask(agent, context) {
|
|
|
356
505
|
agent.config.jsonSchema && desiredOutputFormat === 'json' && !strictSchema
|
|
357
506
|
? 'stream-json'
|
|
358
507
|
: desiredOutputFormat;
|
|
508
|
+
|
|
509
|
+
return { desiredOutputFormat, strictSchema, runOutputFormat };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) {
|
|
359
513
|
const args = ['task', 'run', '--output-format', runOutputFormat, '--provider', providerName];
|
|
360
514
|
|
|
361
515
|
if (modelSpec?.model) {
|
|
@@ -371,82 +525,61 @@ async function spawnClaudeTask(agent, context) {
|
|
|
371
525
|
args.push('-v');
|
|
372
526
|
}
|
|
373
527
|
|
|
374
|
-
// NOTE: maxRetries is handled by the agent wrapper's internal retry loop,
|
|
375
|
-
// not passed to the CLI. See _handleTrigger() for retry logic.
|
|
376
|
-
|
|
377
528
|
// Add JSON schema if specified in agent config.
|
|
378
529
|
// If we are running stream-json for live logs (strictSchema=false), do NOT pass schema to CLI.
|
|
379
|
-
if (agent.config.jsonSchema) {
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
const schema = JSON.stringify(agent.config.jsonSchema);
|
|
383
|
-
args.push('--json-schema', schema);
|
|
384
|
-
} else if (!agent.quiet) {
|
|
385
|
-
agent._log(
|
|
386
|
-
`[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
|
|
387
|
-
);
|
|
388
|
-
}
|
|
530
|
+
if (agent.config.jsonSchema && runOutputFormat === 'json') {
|
|
531
|
+
const schema = JSON.stringify(agent.config.jsonSchema);
|
|
532
|
+
args.push('--json-schema', schema);
|
|
389
533
|
}
|
|
390
534
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
535
|
+
return args;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function maybeLogStreamJsonNotice(agent, runOutputFormat) {
|
|
539
|
+
if (agent.config.jsonSchema && runOutputFormat !== 'json' && !agent.quiet) {
|
|
540
|
+
agent._log(
|
|
541
|
+
`[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function buildFinalContext({ agent, context, desiredOutputFormat, runOutputFormat }) {
|
|
394
547
|
if (
|
|
395
548
|
agent.config.jsonSchema &&
|
|
396
549
|
desiredOutputFormat === 'json' &&
|
|
397
550
|
runOutputFormat === 'stream-json'
|
|
398
551
|
) {
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
args.push(finalContext);
|
|
407
|
-
|
|
408
|
-
// MOCK SUPPORT: Use injected mock function if provided
|
|
409
|
-
if (agent.mockSpawnFn) {
|
|
410
|
-
return agent.mockSpawnFn(args, { context });
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
// SAFETY: Fail hard if testMode=true but no mock (should be caught in constructor)
|
|
414
|
-
if (agent.testMode) {
|
|
415
|
-
throw new Error(
|
|
416
|
-
`AgentWrapper: testMode=true but attempting real Claude API call for agent '${agent.id}'. ` +
|
|
417
|
-
`This is a bug - mock should be set in constructor.`
|
|
552
|
+
return (
|
|
553
|
+
context +
|
|
554
|
+
`\n\n## Output Format (REQUIRED)\n\nReturn a JSON object that matches this schema exactly.\n\nSchema:\n\`\`\`json\n${JSON.stringify(
|
|
555
|
+
agent.config.jsonSchema,
|
|
556
|
+
null,
|
|
557
|
+
2
|
|
558
|
+
)}\n\`\`\`\n`
|
|
418
559
|
);
|
|
419
560
|
}
|
|
420
561
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
562
|
+
return context;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function ensureProviderHooks(agent, providerName) {
|
|
566
|
+
if (providerName !== 'claude') {
|
|
567
|
+
return;
|
|
424
568
|
}
|
|
425
569
|
|
|
426
|
-
|
|
427
|
-
// AskUserQuestion blocking handled via:
|
|
428
|
-
// 1. Prompt injection (see agent-context-builder)
|
|
429
|
-
// 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
|
|
430
|
-
if (providerName === 'claude') {
|
|
431
|
-
ensureAskUserQuestionHook();
|
|
570
|
+
ensureAskUserQuestionHook();
|
|
432
571
|
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
}
|
|
572
|
+
// WORKTREE MODE: Install git safety hook (blocks dangerous git commands)
|
|
573
|
+
if (agent.worktree?.enabled) {
|
|
574
|
+
ensureDangerousGitHook();
|
|
437
575
|
}
|
|
576
|
+
}
|
|
438
577
|
|
|
439
|
-
|
|
440
|
-
const spawnEnv = {
|
|
441
|
-
...process.env,
|
|
442
|
-
};
|
|
578
|
+
function buildSpawnEnv(agent, providerName, modelSpec) {
|
|
579
|
+
const spawnEnv = { ...process.env };
|
|
443
580
|
|
|
444
581
|
if (providerName === 'claude') {
|
|
445
|
-
|
|
446
|
-
spawnEnv.ANTHROPIC_MODEL = modelSpec.model;
|
|
447
|
-
}
|
|
448
|
-
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
449
|
-
spawnEnv.ZEROSHOT_BLOCK_ASK_USER = '1';
|
|
582
|
+
Object.assign(spawnEnv, buildClaudeEnv(modelSpec));
|
|
450
583
|
|
|
451
584
|
// WORKTREE MODE: Activate git safety hook via environment variable
|
|
452
585
|
if (agent.worktree?.enabled) {
|
|
@@ -454,18 +587,45 @@ async function spawnClaudeTask(agent, context) {
|
|
|
454
587
|
}
|
|
455
588
|
}
|
|
456
589
|
|
|
457
|
-
|
|
590
|
+
return spawnEnv;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function parseTaskIdFromOutput(stdout) {
|
|
594
|
+
const match = stdout.match(/Task spawned: ((?:task-)?[a-z]+-[a-z]+-[a-z0-9]+)/);
|
|
595
|
+
return match ? match[1] : null;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function spawnTaskProcess({ agent, ctPath, args, cwd, spawnEnv }) {
|
|
599
|
+
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
600
|
+
const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
|
|
601
|
+
|
|
602
|
+
return new Promise((resolve, reject) => {
|
|
458
603
|
const proc = spawn(ctPath, args, {
|
|
459
604
|
cwd,
|
|
460
605
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
461
606
|
env: spawnEnv,
|
|
462
607
|
});
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
608
|
+
|
|
609
|
+
// NOTE: Don't emit PROCESS_SPAWNED here - proc.pid is a wrapper that exits immediately.
|
|
610
|
+
// Real PID comes from task store after watcher spawns the actual CLI process.
|
|
611
|
+
// PROCESS_SPAWNED is emitted in spawnClaudeTask after waitForTaskReady + PID polling.
|
|
466
612
|
|
|
467
613
|
let stdout = '';
|
|
468
614
|
let stderr = '';
|
|
615
|
+
let resolved = false;
|
|
616
|
+
|
|
617
|
+
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
|
|
618
|
+
const spawnTimeout = setTimeout(() => {
|
|
619
|
+
if (resolved) return;
|
|
620
|
+
resolved = true;
|
|
621
|
+
proc.kill('SIGKILL');
|
|
622
|
+
reject(
|
|
623
|
+
new Error(
|
|
624
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
625
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
626
|
+
)
|
|
627
|
+
);
|
|
628
|
+
}, SPAWN_TIMEOUT_MS);
|
|
469
629
|
|
|
470
630
|
proc.stdout.on('data', (data) => {
|
|
471
631
|
stdout += data.toString();
|
|
@@ -476,6 +636,9 @@ async function spawnClaudeTask(agent, context) {
|
|
|
476
636
|
});
|
|
477
637
|
|
|
478
638
|
proc.on('close', (code, signal) => {
|
|
639
|
+
clearTimeout(spawnTimeout);
|
|
640
|
+
if (resolved) return;
|
|
641
|
+
resolved = true;
|
|
479
642
|
// Handle process killed by signal (e.g., SIGTERM, SIGKILL, SIGSTOP)
|
|
480
643
|
if (signal) {
|
|
481
644
|
reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
|
|
@@ -485,9 +648,8 @@ async function spawnClaudeTask(agent, context) {
|
|
|
485
648
|
if (code === 0) {
|
|
486
649
|
// Parse task ID from output: "✓ Task spawned: xxx-yyy-nn"
|
|
487
650
|
// Format: <adjective>-<noun>-<digits> (may or may not have task- prefix)
|
|
488
|
-
const
|
|
489
|
-
if (
|
|
490
|
-
const spawnedTaskId = match[1];
|
|
651
|
+
const spawnedTaskId = parseTaskIdFromOutput(stdout);
|
|
652
|
+
if (spawnedTaskId) {
|
|
491
653
|
agent.currentTaskId = spawnedTaskId; // Track for resume capability
|
|
492
654
|
agent._publishLifecycle('TASK_ID_ASSIGNED', {
|
|
493
655
|
pid: agent.processPid,
|
|
@@ -510,17 +672,12 @@ async function spawnClaudeTask(agent, context) {
|
|
|
510
672
|
});
|
|
511
673
|
|
|
512
674
|
proc.on('error', (error) => {
|
|
675
|
+
clearTimeout(spawnTimeout);
|
|
676
|
+
if (resolved) return;
|
|
677
|
+
resolved = true;
|
|
513
678
|
reject(error);
|
|
514
679
|
});
|
|
515
680
|
});
|
|
516
|
-
|
|
517
|
-
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
|
|
518
|
-
|
|
519
|
-
// Wait for task to be registered in zeroshot storage (race condition fix)
|
|
520
|
-
await waitForTaskReady(agent, taskId);
|
|
521
|
-
|
|
522
|
-
// Now follow the logs and stream output
|
|
523
|
-
return followClaudeTaskLogs(agent, taskId);
|
|
524
681
|
}
|
|
525
682
|
|
|
526
683
|
/**
|
|
@@ -532,16 +689,16 @@ async function spawnClaudeTask(agent, context) {
|
|
|
532
689
|
* @returns {Promise<void>}
|
|
533
690
|
*/
|
|
534
691
|
async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
535
|
-
const { exec } = require('child_process');
|
|
536
692
|
const ctPath = getClaudeTasksPath();
|
|
537
693
|
|
|
538
694
|
for (let i = 0; i < maxRetries; i++) {
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
695
|
+
let exists = false;
|
|
696
|
+
try {
|
|
697
|
+
const { stdout } = await exec(`${ctPath} status ${taskId}`, { timeout: 5000 });
|
|
698
|
+
exists = !stdout.includes('Task not found');
|
|
699
|
+
} catch {
|
|
700
|
+
// Timeout or error - task not ready yet
|
|
701
|
+
}
|
|
545
702
|
|
|
546
703
|
if (exists) return;
|
|
547
704
|
|
|
@@ -549,332 +706,372 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
|
549
706
|
await new Promise((r) => setTimeout(r, delayMs));
|
|
550
707
|
}
|
|
551
708
|
|
|
552
|
-
//
|
|
553
|
-
|
|
709
|
+
// FAIL FAST: Task not found after retries = unrecoverable error
|
|
710
|
+
// Continuing with a non-existent task causes 30s of pointless polling then crash
|
|
711
|
+
throw new Error(
|
|
712
|
+
`Task ${taskId} not found after ${maxRetries} retries (${maxRetries * delayMs}ms). ` +
|
|
713
|
+
`Task spawn may have failed silently. Check zeroshot task run output.`
|
|
714
|
+
);
|
|
554
715
|
}
|
|
555
716
|
|
|
556
|
-
|
|
557
|
-
* Follow claude-zeroshots logs until completion, streaming to message bus
|
|
558
|
-
* Reads log file directly for reliable streaming
|
|
559
|
-
* @param {Object} agent - Agent instance
|
|
560
|
-
* @param {String} taskId - Task ID to follow
|
|
561
|
-
* @returns {Promise<Object>} Result object { success, output, error }
|
|
562
|
-
*/
|
|
563
|
-
function followClaudeTaskLogs(agent, taskId) {
|
|
564
|
-
const fsModule = require('fs');
|
|
565
|
-
const { execSync, exec } = require('child_process');
|
|
566
|
-
const ctPath = getClaudeTasksPath();
|
|
567
|
-
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
717
|
+
const MAX_STATUS_FAILURES = 30;
|
|
568
718
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
719
|
+
function createLogFollowState() {
|
|
720
|
+
return {
|
|
721
|
+
output: '',
|
|
722
|
+
logFilePath: null,
|
|
723
|
+
lastSize: 0,
|
|
724
|
+
pollInterval: null,
|
|
725
|
+
statusCheckInterval: null,
|
|
726
|
+
resolved: false,
|
|
727
|
+
lineBuffer: '',
|
|
728
|
+
consecutiveExecFailures: 0,
|
|
729
|
+
};
|
|
730
|
+
}
|
|
576
731
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
732
|
+
function lookupLogFilePath(ctPath, taskId) {
|
|
733
|
+
try {
|
|
734
|
+
return execSync(`${ctPath} get-log-path ${taskId}`, {
|
|
735
|
+
encoding: 'utf-8',
|
|
736
|
+
timeout: 5000,
|
|
737
|
+
}).trim();
|
|
738
|
+
} catch {
|
|
739
|
+
return null;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function parseTimestampedLine(line) {
|
|
744
|
+
let timestamp = Date.now();
|
|
745
|
+
let content = line.replace(/\r$/, '');
|
|
746
|
+
|
|
747
|
+
const timestampMatch = content.match(/^\[(\d{13})\](.*)$/);
|
|
748
|
+
if (timestampMatch) {
|
|
749
|
+
timestamp = parseInt(timestampMatch[1], 10);
|
|
750
|
+
content = timestampMatch[2];
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
return { timestamp, content };
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function shouldSkipLogLine(content) {
|
|
757
|
+
return (
|
|
758
|
+
content.startsWith('===') ||
|
|
759
|
+
content.startsWith('Finished:') ||
|
|
760
|
+
content.startsWith('Exit code:') ||
|
|
761
|
+
(content.includes('"type":"system"') && content.includes('"subtype":"init"'))
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
function isValidJsonLine(content) {
|
|
766
|
+
if (!content.trim().startsWith('{')) {
|
|
767
|
+
return false;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
try {
|
|
771
|
+
JSON.parse(content);
|
|
772
|
+
return true;
|
|
773
|
+
} catch {
|
|
774
|
+
return false;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
779
|
+
if (!line.trim()) return;
|
|
780
|
+
|
|
781
|
+
const { timestamp, content } = parseTimestampedLine(line);
|
|
782
|
+
if (shouldSkipLogLine(content)) {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const isValidJson = isValidJsonLine(content);
|
|
787
|
+
state.output += content + '\n';
|
|
788
|
+
|
|
789
|
+
agent.lastOutputTime = Date.now();
|
|
790
|
+
|
|
791
|
+
agent._publish({
|
|
792
|
+
topic: 'AGENT_OUTPUT',
|
|
793
|
+
receiver: 'broadcast',
|
|
794
|
+
timestamp,
|
|
795
|
+
content: {
|
|
796
|
+
text: content,
|
|
797
|
+
data: {
|
|
798
|
+
type: isValidJson ? 'json' : 'text',
|
|
799
|
+
line: content,
|
|
800
|
+
agent: agent.id,
|
|
801
|
+
role: agent.role,
|
|
802
|
+
iteration: agent.iteration,
|
|
803
|
+
provider: providerName,
|
|
804
|
+
},
|
|
805
|
+
},
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function appendContentToBuffer(state, content, onLine) {
|
|
810
|
+
state.lineBuffer += content;
|
|
811
|
+
const lines = state.lineBuffer.split('\n');
|
|
812
|
+
|
|
813
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
814
|
+
onLine(lines[i]);
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
state.lineBuffer = lines[lines.length - 1];
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function pollLogFileForUpdates({ agent, fsModule, ctPath, taskId, state, onNewContent }) {
|
|
821
|
+
if (!state.logFilePath) {
|
|
822
|
+
const logFilePath = lookupLogFilePath(ctPath, taskId);
|
|
823
|
+
if (!logFilePath) {
|
|
824
|
+
return;
|
|
586
825
|
}
|
|
826
|
+
state.logFilePath = logFilePath;
|
|
827
|
+
agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
|
|
828
|
+
}
|
|
587
829
|
|
|
588
|
-
|
|
589
|
-
|
|
830
|
+
if (!fsModule.existsSync(state.logFilePath)) {
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
590
833
|
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
const
|
|
594
|
-
if (!line.trim()) return;
|
|
834
|
+
try {
|
|
835
|
+
const stats = fsModule.statSync(state.logFilePath);
|
|
836
|
+
const currentSize = stats.size;
|
|
595
837
|
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
838
|
+
if (currentSize > state.lastSize) {
|
|
839
|
+
const fd = fsModule.openSync(state.logFilePath, 'r');
|
|
840
|
+
const buffer = Buffer.alloc(currentSize - state.lastSize);
|
|
841
|
+
fsModule.readSync(fd, buffer, 0, buffer.length, state.lastSize);
|
|
842
|
+
fsModule.closeSync(fd);
|
|
600
843
|
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
844
|
+
onNewContent(buffer.toString('utf-8'));
|
|
845
|
+
state.lastSize = currentSize;
|
|
846
|
+
}
|
|
847
|
+
} catch (err) {
|
|
848
|
+
const error = /** @type {Error} */ (err);
|
|
849
|
+
console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${error.message}`);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
606
852
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
content.startsWith('Exit code:') ||
|
|
612
|
-
(content.includes('"type":"system"') && content.includes('"subtype":"init"'))
|
|
613
|
-
) {
|
|
614
|
-
return;
|
|
615
|
-
}
|
|
853
|
+
function stripAnsiCodes(value) {
|
|
854
|
+
const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
|
|
855
|
+
return value.replace(ansiPattern, '');
|
|
856
|
+
}
|
|
616
857
|
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
858
|
+
function parseStatusFlags(cleanStdout) {
|
|
859
|
+
return {
|
|
860
|
+
isCompleted: /Status:\s+completed/i.test(cleanStdout),
|
|
861
|
+
isFailed: /Status:\s+failed/i.test(cleanStdout),
|
|
862
|
+
isStale: /Status:\s+stale/i.test(cleanStdout),
|
|
863
|
+
};
|
|
864
|
+
}
|
|
620
865
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
} catch {
|
|
626
|
-
// Looks like JSON but isn't valid - treat as text
|
|
627
|
-
}
|
|
628
|
-
}
|
|
866
|
+
function determineStaleSuccess({ agent, output, providerName, taskId }) {
|
|
867
|
+
if (!output) {
|
|
868
|
+
return false;
|
|
869
|
+
}
|
|
629
870
|
|
|
630
|
-
|
|
631
|
-
|
|
871
|
+
const hasStructuredOutput = /"structured_output"\s*:/.test(output);
|
|
872
|
+
const hasSuccessResult = /"subtype"\s*:\s*"success"/.test(output);
|
|
873
|
+
let hasParsedOutput = false;
|
|
632
874
|
|
|
633
|
-
|
|
634
|
-
|
|
875
|
+
try {
|
|
876
|
+
const { extractJsonFromOutput } = require('./output-extraction');
|
|
877
|
+
hasParsedOutput = !!extractJsonFromOutput(output, providerName);
|
|
878
|
+
} catch {
|
|
879
|
+
// Ignore extraction errors - fallback to other signals
|
|
880
|
+
}
|
|
635
881
|
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
882
|
+
const success = hasStructuredOutput || hasSuccessResult || hasParsedOutput;
|
|
883
|
+
if (!agent.quiet) {
|
|
884
|
+
agent._log(
|
|
885
|
+
`[Agent ${agent.id}] Task ${taskId} is stale - recovered as ${success ? 'SUCCESS' : 'FAILURE'} based on output analysis`
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
return success;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function finalizeLogFollow(agent, state) {
|
|
893
|
+
if (state.pollInterval) {
|
|
894
|
+
clearInterval(state.pollInterval);
|
|
895
|
+
}
|
|
896
|
+
if (state.statusCheckInterval) {
|
|
897
|
+
clearInterval(state.statusCheckInterval);
|
|
898
|
+
}
|
|
899
|
+
agent.currentTask = null;
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve }) {
|
|
903
|
+
if (!error) {
|
|
904
|
+
return false;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
state.consecutiveExecFailures++;
|
|
908
|
+
if (state.consecutiveExecFailures < MAX_STATUS_FAILURES) {
|
|
909
|
+
return true;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
console.error(
|
|
913
|
+
`[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_STATUS_FAILURES} times consecutively! STOPPING.`
|
|
914
|
+
);
|
|
915
|
+
console.error(` Command: ${ctPath} status ${taskId}`);
|
|
916
|
+
console.error(` Error: ${error.message}`);
|
|
917
|
+
console.error(` Stderr: ${stderr || 'none'}`);
|
|
918
|
+
console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
|
|
919
|
+
|
|
920
|
+
if (!state.resolved) {
|
|
921
|
+
state.resolved = true;
|
|
922
|
+
finalizeLogFollow(agent, state);
|
|
923
|
+
|
|
924
|
+
agent._publish({
|
|
925
|
+
topic: 'AGENT_ERROR',
|
|
926
|
+
receiver: 'broadcast',
|
|
927
|
+
content: {
|
|
928
|
+
text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
|
|
929
|
+
data: {
|
|
930
|
+
taskId,
|
|
931
|
+
error: 'polling_timeout',
|
|
932
|
+
attempts: state.consecutiveExecFailures,
|
|
933
|
+
role: agent.role,
|
|
934
|
+
iteration: agent.iteration,
|
|
650
935
|
},
|
|
651
|
-
}
|
|
652
|
-
};
|
|
936
|
+
},
|
|
937
|
+
});
|
|
653
938
|
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
939
|
+
resolve({
|
|
940
|
+
success: false,
|
|
941
|
+
output: state.output,
|
|
942
|
+
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
658
945
|
|
|
659
|
-
|
|
660
|
-
|
|
946
|
+
return true;
|
|
947
|
+
}
|
|
661
948
|
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
949
|
+
function handleStatusCompletion({
|
|
950
|
+
agent,
|
|
951
|
+
taskId,
|
|
952
|
+
providerName,
|
|
953
|
+
state,
|
|
954
|
+
stdout,
|
|
955
|
+
pollLogFile,
|
|
956
|
+
resolve,
|
|
957
|
+
}) {
|
|
958
|
+
const cleanStdout = stripAnsiCodes(stdout);
|
|
959
|
+
const { isCompleted, isFailed, isStale } = parseStatusFlags(cleanStdout);
|
|
960
|
+
|
|
961
|
+
if (!isCompleted && !isFailed && !isStale) {
|
|
962
|
+
return false;
|
|
963
|
+
}
|
|
666
964
|
|
|
667
|
-
|
|
668
|
-
lineBuffer = lines[lines.length - 1];
|
|
669
|
-
};
|
|
670
|
-
|
|
671
|
-
// Poll the log file for new content
|
|
672
|
-
const pollLogFile = () => {
|
|
673
|
-
// If we don't have log path yet, try to get it
|
|
674
|
-
if (!logFilePath) {
|
|
675
|
-
try {
|
|
676
|
-
logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
|
|
677
|
-
encoding: 'utf-8',
|
|
678
|
-
}).trim();
|
|
679
|
-
agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
|
|
680
|
-
} catch {
|
|
681
|
-
return; // Not ready yet
|
|
682
|
-
}
|
|
683
|
-
}
|
|
965
|
+
pollLogFile();
|
|
684
966
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
967
|
+
let success = isCompleted;
|
|
968
|
+
if (isStale) {
|
|
969
|
+
success = determineStaleSuccess({ agent, output: state.output, providerName, taskId });
|
|
970
|
+
}
|
|
689
971
|
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
if (currentSize > lastSize) {
|
|
695
|
-
// Read new content
|
|
696
|
-
const fd = fsModule.openSync(logFilePath, 'r');
|
|
697
|
-
const buffer = Buffer.alloc(currentSize - lastSize);
|
|
698
|
-
fsModule.readSync(fd, buffer, 0, buffer.length, lastSize);
|
|
699
|
-
fsModule.closeSync(fd);
|
|
700
|
-
|
|
701
|
-
const newContent = buffer.toString('utf-8');
|
|
702
|
-
// Process new content line-by-line
|
|
703
|
-
processNewContent(newContent);
|
|
704
|
-
lastSize = currentSize;
|
|
705
|
-
}
|
|
706
|
-
} catch (err) {
|
|
707
|
-
// File might have been deleted or locked
|
|
708
|
-
console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${err.message}`);
|
|
709
|
-
}
|
|
710
|
-
};
|
|
711
|
-
|
|
712
|
-
// Start polling log file (every 300ms for responsive streaming)
|
|
713
|
-
pollInterval = setInterval(pollLogFile, 300);
|
|
714
|
-
|
|
715
|
-
// Poll ct status to know when task is complete
|
|
716
|
-
// Track consecutive failures for debugging stuck clusters
|
|
717
|
-
let consecutiveExecFailures = 0;
|
|
718
|
-
const MAX_CONSECUTIVE_FAILURES = 30; // 30 seconds of failures = log warning
|
|
719
|
-
|
|
720
|
-
statusCheckInterval = setInterval(() => {
|
|
721
|
-
exec(`${ctPath} status ${taskId}`, (error, stdout, stderr) => {
|
|
722
|
-
if (resolved) return;
|
|
723
|
-
|
|
724
|
-
// Track exec failures - if status command keeps failing, something is wrong
|
|
725
|
-
if (error) {
|
|
726
|
-
consecutiveExecFailures++;
|
|
727
|
-
if (consecutiveExecFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
728
|
-
console.error(
|
|
729
|
-
`[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_CONSECUTIVE_FAILURES} times consecutively! STOPPING.`
|
|
730
|
-
);
|
|
731
|
-
console.error(` Command: ${ctPath} status ${taskId}`);
|
|
732
|
-
console.error(` Error: ${error.message}`);
|
|
733
|
-
console.error(` Stderr: ${stderr || 'none'}`);
|
|
734
|
-
console.error(
|
|
735
|
-
` This may indicate zeroshot is not in PATH or task storage is corrupted.`
|
|
736
|
-
);
|
|
972
|
+
setTimeout(() => {
|
|
973
|
+
if (state.resolved) return;
|
|
974
|
+
state.resolved = true;
|
|
737
975
|
|
|
738
|
-
|
|
739
|
-
if (!resolved) {
|
|
740
|
-
resolved = true;
|
|
741
|
-
clearInterval(pollInterval);
|
|
742
|
-
clearInterval(statusCheckInterval);
|
|
743
|
-
agent.currentTask = null;
|
|
744
|
-
|
|
745
|
-
// Publish error for orchestrator/resume
|
|
746
|
-
agent._publish({
|
|
747
|
-
topic: 'AGENT_ERROR',
|
|
748
|
-
receiver: 'broadcast',
|
|
749
|
-
content: {
|
|
750
|
-
text: `Task ${taskId} polling failed after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`,
|
|
751
|
-
data: {
|
|
752
|
-
taskId,
|
|
753
|
-
error: 'polling_timeout',
|
|
754
|
-
attempts: consecutiveExecFailures,
|
|
755
|
-
role: agent.role,
|
|
756
|
-
iteration: agent.iteration,
|
|
757
|
-
},
|
|
758
|
-
},
|
|
759
|
-
});
|
|
760
|
-
|
|
761
|
-
resolve({
|
|
762
|
-
success: false,
|
|
763
|
-
output,
|
|
764
|
-
error: `Status polling failed ${MAX_CONSECUTIVE_FAILURES} times - task may not exist`,
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
return;
|
|
768
|
-
}
|
|
769
|
-
return; // Keep polling - might be transient
|
|
770
|
-
}
|
|
976
|
+
finalizeLogFollow(agent, state);
|
|
771
977
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
// Check for completion/failure status
|
|
776
|
-
// Strip ANSI codes in case chalk outputs them (shouldn't in non-TTY, but be safe)
|
|
777
|
-
// Use RegExp constructor to avoid ESLint no-control-regex false positive
|
|
778
|
-
const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
|
|
779
|
-
const cleanStdout = stdout.replace(ansiPattern, '');
|
|
780
|
-
// Use flexible whitespace matching in case spacing changes
|
|
781
|
-
const isCompleted = /Status:\s+completed/i.test(cleanStdout);
|
|
782
|
-
const isFailed = /Status:\s+failed/i.test(cleanStdout);
|
|
783
|
-
// BUGFIX: Handle "stale (process died)" status - watcher died before updating status
|
|
784
|
-
// Check if task produced a successful result (structured_output in log file)
|
|
785
|
-
const isStale = /Status:\s+stale/i.test(cleanStdout);
|
|
786
|
-
|
|
787
|
-
if (isCompleted || isFailed || isStale) {
|
|
788
|
-
// CRITICAL: Read final log content BEFORE checking output
|
|
789
|
-
// Fixes race where status flips to stale before log polling catches up
|
|
790
|
-
pollLogFile();
|
|
791
|
-
|
|
792
|
-
// For stale tasks, check log file for successful result
|
|
793
|
-
let success = isCompleted;
|
|
794
|
-
if (isStale && output) {
|
|
795
|
-
// Look for structured_output in accumulated output - indicates success
|
|
796
|
-
const hasStructuredOutput = /"structured_output"\s*:/.test(output);
|
|
797
|
-
const hasSuccessResult = /"subtype"\s*:\s*"success"/.test(output);
|
|
798
|
-
let hasParsedOutput = false;
|
|
799
|
-
try {
|
|
800
|
-
const { extractJsonFromOutput } = require('./output-extraction');
|
|
801
|
-
hasParsedOutput = !!extractJsonFromOutput(output, providerName);
|
|
802
|
-
} catch {
|
|
803
|
-
// Ignore extraction errors - fallback to other signals
|
|
804
|
-
}
|
|
805
|
-
success = hasStructuredOutput || hasSuccessResult || hasParsedOutput;
|
|
806
|
-
if (!agent.quiet) {
|
|
807
|
-
agent._log(
|
|
808
|
-
`[Agent ${agent.id}] Task ${taskId} is stale - recovered as ${success ? 'SUCCESS' : 'FAILURE'} based on output analysis`
|
|
809
|
-
);
|
|
810
|
-
}
|
|
811
|
-
}
|
|
978
|
+
const errorContext = !success
|
|
979
|
+
? extractErrorContext({ output: state.output, statusOutput: stdout, taskId })
|
|
980
|
+
: null;
|
|
812
981
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
982
|
+
resolve({
|
|
983
|
+
success,
|
|
984
|
+
output: state.output,
|
|
985
|
+
error: errorContext,
|
|
986
|
+
tokenUsage: extractTokenUsage(state.output, providerName),
|
|
987
|
+
});
|
|
988
|
+
}, 500);
|
|
989
|
+
|
|
990
|
+
return true;
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
function buildKillHandler({ agent, state, providerName, resolve }) {
|
|
994
|
+
return {
|
|
995
|
+
kill: (reason = 'Task killed') => {
|
|
996
|
+
if (state.resolved) return;
|
|
997
|
+
state.resolved = true;
|
|
998
|
+
finalizeLogFollow(agent, state);
|
|
999
|
+
agent._stopLivenessCheck();
|
|
1000
|
+
resolve({
|
|
1001
|
+
success: false,
|
|
1002
|
+
output: state.output,
|
|
1003
|
+
error: reason,
|
|
1004
|
+
tokenUsage: extractTokenUsage(state.output, providerName),
|
|
1005
|
+
});
|
|
1006
|
+
},
|
|
1007
|
+
};
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function createLogFollower({ agent, taskId, fsModule, ctPath, providerName }) {
|
|
1011
|
+
return new Promise((resolve) => {
|
|
1012
|
+
const state = createLogFollowState();
|
|
1013
|
+
|
|
1014
|
+
state.logFilePath = lookupLogFilePath(ctPath, taskId);
|
|
1015
|
+
if (state.logFilePath) {
|
|
1016
|
+
agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
|
|
1017
|
+
} else {
|
|
1018
|
+
agent._log(`⏳ Agent ${agent.id}: Waiting for log file...`);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
const broadcastLine = (line) => broadcastAgentLine({ agent, providerName, state, line });
|
|
1022
|
+
const processNewContent = (content) => appendContentToBuffer(state, content, broadcastLine);
|
|
1023
|
+
const pollLogFile = () =>
|
|
1024
|
+
pollLogFileForUpdates({
|
|
1025
|
+
agent,
|
|
1026
|
+
fsModule,
|
|
1027
|
+
ctPath,
|
|
1028
|
+
taskId,
|
|
1029
|
+
state,
|
|
1030
|
+
onNewContent: processNewContent,
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
state.pollInterval = setInterval(pollLogFile, 300);
|
|
1034
|
+
|
|
1035
|
+
state.statusCheckInterval = setInterval(() => {
|
|
1036
|
+
exec(`${ctPath} status ${taskId}`, { timeout: 5000 }, (error, stdout, stderr) => {
|
|
1037
|
+
if (state.resolved) return;
|
|
1038
|
+
|
|
1039
|
+
if (handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve })) {
|
|
1040
|
+
return;
|
|
834
1041
|
}
|
|
1042
|
+
|
|
1043
|
+
state.consecutiveExecFailures = 0;
|
|
1044
|
+
handleStatusCompletion({
|
|
1045
|
+
agent,
|
|
1046
|
+
taskId,
|
|
1047
|
+
providerName,
|
|
1048
|
+
state,
|
|
1049
|
+
stdout,
|
|
1050
|
+
pollLogFile,
|
|
1051
|
+
resolve,
|
|
1052
|
+
});
|
|
835
1053
|
});
|
|
836
1054
|
}, 1000);
|
|
837
1055
|
|
|
838
|
-
|
|
839
|
-
// CRITICAL: Must reject promise to avoid orphaned promise that hangs forever
|
|
840
|
-
agent.currentTask = {
|
|
841
|
-
kill: (reason = 'Task killed') => {
|
|
842
|
-
if (resolved) return;
|
|
843
|
-
resolved = true;
|
|
844
|
-
clearInterval(pollInterval);
|
|
845
|
-
clearInterval(statusCheckInterval);
|
|
846
|
-
agent._stopLivenessCheck();
|
|
847
|
-
// BUGFIX: Resolve with failure instead of orphaning the promise
|
|
848
|
-
// This allows the caller to handle the kill gracefully
|
|
849
|
-
resolve({
|
|
850
|
-
success: false,
|
|
851
|
-
output,
|
|
852
|
-
error: reason,
|
|
853
|
-
tokenUsage: extractTokenUsage(output, providerName),
|
|
854
|
-
});
|
|
855
|
-
},
|
|
856
|
-
};
|
|
857
|
-
|
|
858
|
-
// REMOVED: Task timeout disabled - tasks run until completion or explicit kill
|
|
859
|
-
// Tasks should run until:
|
|
860
|
-
// - Completion
|
|
861
|
-
// - Explicit kill
|
|
862
|
-
// - External error (rate limit, API failure)
|
|
863
|
-
//
|
|
864
|
-
// setTimeout(() => {
|
|
865
|
-
// if (resolved) return;
|
|
866
|
-
// resolved = true;
|
|
867
|
-
//
|
|
868
|
-
// clearInterval(pollInterval);
|
|
869
|
-
// clearInterval(statusCheckInterval);
|
|
870
|
-
// agent._stopLivenessCheck();
|
|
871
|
-
// agent.currentTask = null;
|
|
872
|
-
// const timeoutMinutes = Math.round(agent.timeout / 60000);
|
|
873
|
-
// reject(new Error(`Task timed out after ${timeoutMinutes} minutes`));
|
|
874
|
-
// }, agent.timeout);
|
|
1056
|
+
agent.currentTask = buildKillHandler({ agent, state, providerName, resolve });
|
|
875
1057
|
});
|
|
876
1058
|
}
|
|
877
1059
|
|
|
1060
|
+
/**
|
|
1061
|
+
* Follow claude-zeroshots logs until completion, streaming to message bus
|
|
1062
|
+
* Reads log file directly for reliable streaming
|
|
1063
|
+
* @param {Object} agent - Agent instance
|
|
1064
|
+
* @param {String} taskId - Task ID to follow
|
|
1065
|
+
* @returns {Promise<Object>} Result object { success, output, error }
|
|
1066
|
+
*/
|
|
1067
|
+
function followClaudeTaskLogs(agent, taskId) {
|
|
1068
|
+
const fsModule = require('fs');
|
|
1069
|
+
const ctPath = getClaudeTasksPath();
|
|
1070
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1071
|
+
|
|
1072
|
+
return createLogFollower({ agent, taskId, fsModule, ctPath, providerName });
|
|
1073
|
+
}
|
|
1074
|
+
|
|
878
1075
|
/**
|
|
879
1076
|
* Get path to claude-zeroshots executable
|
|
880
1077
|
* @returns {String} Path to zeroshot command
|
|
@@ -963,16 +1160,15 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
963
1160
|
command.push(finalContext);
|
|
964
1161
|
|
|
965
1162
|
// STEP 1: Spawn task and extract task ID (same as non-isolated mode)
|
|
1163
|
+
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
1164
|
+
const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
|
|
1165
|
+
// Note: Auth env vars are injected by IsolationManager, we only need model mapping here
|
|
1166
|
+
const isolatedEnv =
|
|
1167
|
+
providerName === 'claude' ? buildClaudeEnv(modelSpec, { includeAuth: false }) : {};
|
|
1168
|
+
|
|
966
1169
|
const taskId = await new Promise((resolve, reject) => {
|
|
967
1170
|
const proc = manager.spawnInContainer(clusterId, command, {
|
|
968
|
-
env:
|
|
969
|
-
providerName === 'claude'
|
|
970
|
-
? {
|
|
971
|
-
ANTHROPIC_MODEL: modelSpec?.model,
|
|
972
|
-
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
973
|
-
ZEROSHOT_BLOCK_ASK_USER: '1',
|
|
974
|
-
}
|
|
975
|
-
: {},
|
|
1171
|
+
env: isolatedEnv,
|
|
976
1172
|
});
|
|
977
1173
|
|
|
978
1174
|
// Track PID for resource monitoring
|
|
@@ -981,6 +1177,20 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
981
1177
|
|
|
982
1178
|
let stdout = '';
|
|
983
1179
|
let stderr = '';
|
|
1180
|
+
let resolved = false;
|
|
1181
|
+
|
|
1182
|
+
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
|
|
1183
|
+
const spawnTimeout = setTimeout(() => {
|
|
1184
|
+
if (resolved) return;
|
|
1185
|
+
resolved = true;
|
|
1186
|
+
proc.kill('SIGKILL');
|
|
1187
|
+
reject(
|
|
1188
|
+
new Error(
|
|
1189
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
1190
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
1191
|
+
)
|
|
1192
|
+
);
|
|
1193
|
+
}, SPAWN_TIMEOUT_MS);
|
|
984
1194
|
|
|
985
1195
|
proc.stdout.on('data', (data) => {
|
|
986
1196
|
stdout += data.toString();
|
|
@@ -991,6 +1201,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
991
1201
|
});
|
|
992
1202
|
|
|
993
1203
|
proc.on('close', (code, signal) => {
|
|
1204
|
+
clearTimeout(spawnTimeout);
|
|
1205
|
+
if (resolved) return;
|
|
1206
|
+
resolved = true;
|
|
994
1207
|
// Handle process killed by signal
|
|
995
1208
|
if (signal) {
|
|
996
1209
|
reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
|
|
@@ -1024,6 +1237,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
1024
1237
|
});
|
|
1025
1238
|
|
|
1026
1239
|
proc.on('error', (error) => {
|
|
1240
|
+
clearTimeout(spawnTimeout);
|
|
1241
|
+
if (resolved) return;
|
|
1242
|
+
resolved = true;
|
|
1027
1243
|
reject(error);
|
|
1028
1244
|
});
|
|
1029
1245
|
});
|
|
@@ -1057,6 +1273,196 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
1057
1273
|
* - Status checks reduced to every 2 seconds (not every poll)
|
|
1058
1274
|
* - Result: 10-20% overall latency reduction
|
|
1059
1275
|
*/
|
|
1276
|
+
function createIsolatedLogState() {
|
|
1277
|
+
return {
|
|
1278
|
+
taskExited: false,
|
|
1279
|
+
fullOutput: '',
|
|
1280
|
+
tailProcess: null,
|
|
1281
|
+
statusCheckInterval: null,
|
|
1282
|
+
lineBuffer: '',
|
|
1283
|
+
};
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
function buildIsolatedCleanup(state) {
|
|
1287
|
+
return () => {
|
|
1288
|
+
if (state.tailProcess) {
|
|
1289
|
+
try {
|
|
1290
|
+
state.tailProcess.kill('SIGTERM');
|
|
1291
|
+
} catch {
|
|
1292
|
+
// Ignore - process may already be dead
|
|
1293
|
+
}
|
|
1294
|
+
state.tailProcess = null;
|
|
1295
|
+
}
|
|
1296
|
+
if (state.statusCheckInterval) {
|
|
1297
|
+
clearInterval(state.statusCheckInterval);
|
|
1298
|
+
state.statusCheckInterval = null;
|
|
1299
|
+
}
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
function broadcastIsolatedLine({ agent, providerName, taskId, line }) {
|
|
1304
|
+
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
1305
|
+
const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
|
|
1306
|
+
const content = timestampMatch ? timestampMatch[2] : line;
|
|
1307
|
+
|
|
1308
|
+
agent.messageBus.publish({
|
|
1309
|
+
cluster_id: agent.cluster.id,
|
|
1310
|
+
topic: 'AGENT_OUTPUT',
|
|
1311
|
+
sender: agent.id,
|
|
1312
|
+
content: {
|
|
1313
|
+
data: {
|
|
1314
|
+
line: content,
|
|
1315
|
+
taskId,
|
|
1316
|
+
iteration: agent.iteration,
|
|
1317
|
+
provider: providerName,
|
|
1318
|
+
},
|
|
1319
|
+
},
|
|
1320
|
+
timestamp,
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
agent.lastOutputTime = Date.now();
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function appendIsolatedContent(state, content, onLine) {
|
|
1327
|
+
state.lineBuffer += content;
|
|
1328
|
+
const lines = state.lineBuffer.split('\n');
|
|
1329
|
+
|
|
1330
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
1331
|
+
if (lines[i].trim()) {
|
|
1332
|
+
onLine(lines[i]);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
state.lineBuffer = lines[lines.length - 1];
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
|
|
1340
|
+
state.tailProcess = manager.spawnInContainer(clusterId, [
|
|
1341
|
+
'sh',
|
|
1342
|
+
'-c',
|
|
1343
|
+
`while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
|
|
1344
|
+
]);
|
|
1345
|
+
|
|
1346
|
+
state.tailProcess.stdout.on('data', (data) => {
|
|
1347
|
+
const chunk = data.toString();
|
|
1348
|
+
state.fullOutput += chunk;
|
|
1349
|
+
appendIsolatedContent(state, chunk, onLine);
|
|
1350
|
+
});
|
|
1351
|
+
|
|
1352
|
+
state.tailProcess.stderr.on('data', (data) => {
|
|
1353
|
+
const msg = data.toString().trim();
|
|
1354
|
+
if (msg && !msg.includes('file truncated')) {
|
|
1355
|
+
agent._log(`[${agent.id}] tail stderr: ${msg}`);
|
|
1356
|
+
}
|
|
1357
|
+
});
|
|
1358
|
+
|
|
1359
|
+
state.tailProcess.on('close', (exitCode) => {
|
|
1360
|
+
if (!state.taskExited) {
|
|
1361
|
+
agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
|
|
1362
|
+
}
|
|
1363
|
+
});
|
|
1364
|
+
|
|
1365
|
+
state.tailProcess.on('error', (err) => {
|
|
1366
|
+
agent._log(`[${agent.id}] tail process error: ${err.message}`);
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
async function checkIsolatedStatus({
|
|
1371
|
+
agent,
|
|
1372
|
+
manager,
|
|
1373
|
+
clusterId,
|
|
1374
|
+
logFilePath,
|
|
1375
|
+
taskId,
|
|
1376
|
+
providerName,
|
|
1377
|
+
state,
|
|
1378
|
+
cleanup,
|
|
1379
|
+
resolve,
|
|
1380
|
+
onLine,
|
|
1381
|
+
}) {
|
|
1382
|
+
if (state.taskExited) return;
|
|
1383
|
+
|
|
1384
|
+
const statusResult = await manager.execInContainer(clusterId, [
|
|
1385
|
+
'sh',
|
|
1386
|
+
'-c',
|
|
1387
|
+
`zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
|
|
1388
|
+
]);
|
|
1389
|
+
|
|
1390
|
+
const statusOutput = statusResult.stdout;
|
|
1391
|
+
const isSuccess = /Status:\s+completed/i.test(statusOutput);
|
|
1392
|
+
const isError = /Status:\s+failed/i.test(statusOutput);
|
|
1393
|
+
const isNotFound = statusOutput.includes('not_found');
|
|
1394
|
+
|
|
1395
|
+
if (!isSuccess && !isError && !isNotFound) {
|
|
1396
|
+
return;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
state.taskExited = true;
|
|
1400
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
1401
|
+
|
|
1402
|
+
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
1403
|
+
'sh',
|
|
1404
|
+
'-c',
|
|
1405
|
+
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
1406
|
+
]);
|
|
1407
|
+
|
|
1408
|
+
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
1409
|
+
state.fullOutput = finalReadResult.stdout;
|
|
1410
|
+
const remainingLines = state.fullOutput.split('\n');
|
|
1411
|
+
for (const line of remainingLines) {
|
|
1412
|
+
if (line.trim()) {
|
|
1413
|
+
onLine(line);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
cleanup();
|
|
1419
|
+
|
|
1420
|
+
const success = isSuccess && !isError;
|
|
1421
|
+
const errorContext = !success
|
|
1422
|
+
? extractErrorContext({ output: state.fullOutput, taskId, isNotFound })
|
|
1423
|
+
: null;
|
|
1424
|
+
const parsedResult = await agent._parseResultOutput(state.fullOutput);
|
|
1425
|
+
|
|
1426
|
+
resolve({
|
|
1427
|
+
success,
|
|
1428
|
+
output: state.fullOutput,
|
|
1429
|
+
taskId,
|
|
1430
|
+
result: parsedResult,
|
|
1431
|
+
error: errorContext,
|
|
1432
|
+
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
function startIsolatedStatusChecks({
|
|
1437
|
+
agent,
|
|
1438
|
+
manager,
|
|
1439
|
+
clusterId,
|
|
1440
|
+
logFilePath,
|
|
1441
|
+
taskId,
|
|
1442
|
+
providerName,
|
|
1443
|
+
state,
|
|
1444
|
+
cleanup,
|
|
1445
|
+
resolve,
|
|
1446
|
+
onLine,
|
|
1447
|
+
}) {
|
|
1448
|
+
state.statusCheckInterval = setInterval(() => {
|
|
1449
|
+
checkIsolatedStatus({
|
|
1450
|
+
agent,
|
|
1451
|
+
manager,
|
|
1452
|
+
clusterId,
|
|
1453
|
+
logFilePath,
|
|
1454
|
+
taskId,
|
|
1455
|
+
providerName,
|
|
1456
|
+
state,
|
|
1457
|
+
cleanup,
|
|
1458
|
+
resolve,
|
|
1459
|
+
onLine,
|
|
1460
|
+
}).catch((statusErr) => {
|
|
1461
|
+
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
1462
|
+
});
|
|
1463
|
+
}, 2000);
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1060
1466
|
function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
1061
1467
|
const { isolation } = agent;
|
|
1062
1468
|
if (!isolation?.manager) {
|
|
@@ -1068,70 +1474,10 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
|
1068
1474
|
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1069
1475
|
|
|
1070
1476
|
return new Promise((resolve, reject) => {
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
let statusCheckInterval = null;
|
|
1075
|
-
let lineBuffer = '';
|
|
1076
|
-
|
|
1077
|
-
// Cleanup function - kill tail process and clear intervals
|
|
1078
|
-
const cleanup = () => {
|
|
1079
|
-
if (tailProcess) {
|
|
1080
|
-
try {
|
|
1081
|
-
tailProcess.kill('SIGTERM');
|
|
1082
|
-
} catch {
|
|
1083
|
-
// Ignore - process may already be dead
|
|
1084
|
-
}
|
|
1085
|
-
tailProcess = null;
|
|
1086
|
-
}
|
|
1087
|
-
if (statusCheckInterval) {
|
|
1088
|
-
clearInterval(statusCheckInterval);
|
|
1089
|
-
statusCheckInterval = null;
|
|
1090
|
-
}
|
|
1091
|
-
};
|
|
1092
|
-
|
|
1093
|
-
// Broadcast line helper (same as non-isolated mode)
|
|
1094
|
-
const broadcastLine = (line) => {
|
|
1095
|
-
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
1096
|
-
const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
|
|
1097
|
-
const content = timestampMatch ? timestampMatch[2] : line;
|
|
1098
|
-
|
|
1099
|
-
agent.messageBus.publish({
|
|
1100
|
-
cluster_id: agent.cluster.id,
|
|
1101
|
-
topic: 'AGENT_OUTPUT',
|
|
1102
|
-
sender: agent.id,
|
|
1103
|
-
content: {
|
|
1104
|
-
data: {
|
|
1105
|
-
line: content,
|
|
1106
|
-
taskId,
|
|
1107
|
-
iteration: agent.iteration,
|
|
1108
|
-
provider: providerName,
|
|
1109
|
-
},
|
|
1110
|
-
},
|
|
1111
|
-
timestamp,
|
|
1112
|
-
});
|
|
1113
|
-
|
|
1114
|
-
// Update last output time for liveness tracking
|
|
1115
|
-
agent.lastOutputTime = Date.now();
|
|
1116
|
-
};
|
|
1117
|
-
|
|
1118
|
-
// Process new content by splitting into complete lines
|
|
1119
|
-
const processNewContent = (content) => {
|
|
1120
|
-
lineBuffer += content;
|
|
1121
|
-
const lines = lineBuffer.split('\n');
|
|
1122
|
-
|
|
1123
|
-
// Process all complete lines (all except last, which might be incomplete)
|
|
1124
|
-
for (let i = 0; i < lines.length - 1; i++) {
|
|
1125
|
-
if (lines[i].trim()) {
|
|
1126
|
-
broadcastLine(lines[i]);
|
|
1127
|
-
}
|
|
1128
|
-
}
|
|
1129
|
-
|
|
1130
|
-
// Keep last line in buffer (might be incomplete)
|
|
1131
|
-
lineBuffer = lines[lines.length - 1];
|
|
1132
|
-
};
|
|
1477
|
+
const state = createIsolatedLogState();
|
|
1478
|
+
const cleanup = buildIsolatedCleanup(state);
|
|
1479
|
+
const onLine = (line) => broadcastIsolatedLine({ agent, providerName, taskId, line });
|
|
1133
1480
|
|
|
1134
|
-
// Get log file path from zeroshot CLI inside container
|
|
1135
1481
|
manager
|
|
1136
1482
|
.execInContainer(clusterId, ['sh', '-c', `zeroshot get-log-path ${taskId}`])
|
|
1137
1483
|
.then(({ stdout, stderr, code }) => {
|
|
@@ -1150,117 +1496,31 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
|
1150
1496
|
|
|
1151
1497
|
agent._log(`[${agent.id}] Following isolated task logs (streaming): ${logFilePath}`);
|
|
1152
1498
|
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
// The -F flag handles file recreation (rotation)
|
|
1161
|
-
`while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
|
|
1162
|
-
]);
|
|
1163
|
-
|
|
1164
|
-
// Stream stdout directly - lines arrive as they're written
|
|
1165
|
-
tailProcess.stdout.on('data', (data) => {
|
|
1166
|
-
const chunk = data.toString();
|
|
1167
|
-
fullOutput += chunk;
|
|
1168
|
-
processNewContent(chunk);
|
|
1499
|
+
startIsolatedTail({
|
|
1500
|
+
agent,
|
|
1501
|
+
manager,
|
|
1502
|
+
clusterId,
|
|
1503
|
+
logFilePath,
|
|
1504
|
+
state,
|
|
1505
|
+
onLine,
|
|
1169
1506
|
});
|
|
1170
1507
|
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
|
|
1183
|
-
}
|
|
1184
|
-
});
|
|
1185
|
-
|
|
1186
|
-
tailProcess.on('error', (err) => {
|
|
1187
|
-
agent._log(`[${agent.id}] tail process error: ${err.message}`);
|
|
1508
|
+
startIsolatedStatusChecks({
|
|
1509
|
+
agent,
|
|
1510
|
+
manager,
|
|
1511
|
+
clusterId,
|
|
1512
|
+
logFilePath,
|
|
1513
|
+
taskId,
|
|
1514
|
+
providerName,
|
|
1515
|
+
state,
|
|
1516
|
+
cleanup,
|
|
1517
|
+
resolve,
|
|
1518
|
+
onLine,
|
|
1188
1519
|
});
|
|
1189
1520
|
|
|
1190
|
-
// Check task status periodically (every 2 seconds - much less frequent than polling)
|
|
1191
|
-
// This is the only remaining docker exec - but now at 2s intervals instead of 500ms
|
|
1192
|
-
statusCheckInterval = setInterval(async () => {
|
|
1193
|
-
if (taskExited) return;
|
|
1194
|
-
|
|
1195
|
-
try {
|
|
1196
|
-
const statusResult = await manager.execInContainer(clusterId, [
|
|
1197
|
-
'sh',
|
|
1198
|
-
'-c',
|
|
1199
|
-
`zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
|
|
1200
|
-
]);
|
|
1201
|
-
|
|
1202
|
-
const statusOutput = statusResult.stdout;
|
|
1203
|
-
const isSuccess = /Status:\s+completed/i.test(statusOutput);
|
|
1204
|
-
const isError = /Status:\s+failed/i.test(statusOutput);
|
|
1205
|
-
const isNotFound = statusOutput.includes('not_found');
|
|
1206
|
-
|
|
1207
|
-
if (isSuccess || isError || isNotFound) {
|
|
1208
|
-
taskExited = true;
|
|
1209
|
-
|
|
1210
|
-
// Give tail a moment to flush remaining output
|
|
1211
|
-
await new Promise((r) => setTimeout(r, 200));
|
|
1212
|
-
|
|
1213
|
-
// Read final output to ensure we have everything
|
|
1214
|
-
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
1215
|
-
'sh',
|
|
1216
|
-
'-c',
|
|
1217
|
-
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
1218
|
-
]);
|
|
1219
|
-
|
|
1220
|
-
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
1221
|
-
fullOutput = finalReadResult.stdout;
|
|
1222
|
-
|
|
1223
|
-
// Process any remaining content
|
|
1224
|
-
const remainingLines = fullOutput.split('\n');
|
|
1225
|
-
for (const line of remainingLines) {
|
|
1226
|
-
if (line.trim()) {
|
|
1227
|
-
broadcastLine(line);
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
}
|
|
1231
|
-
|
|
1232
|
-
cleanup();
|
|
1233
|
-
|
|
1234
|
-
// Determine success status
|
|
1235
|
-
const success = isSuccess && !isError;
|
|
1236
|
-
|
|
1237
|
-
// Extract error context using shared helper
|
|
1238
|
-
const errorContext = !success
|
|
1239
|
-
? extractErrorContext({ output: fullOutput, taskId, isNotFound })
|
|
1240
|
-
: null;
|
|
1241
|
-
|
|
1242
|
-
// Parse result from output (async - may trigger reformatting)
|
|
1243
|
-
const parsedResult = await agent._parseResultOutput(fullOutput);
|
|
1244
|
-
|
|
1245
|
-
resolve({
|
|
1246
|
-
success,
|
|
1247
|
-
output: fullOutput,
|
|
1248
|
-
taskId,
|
|
1249
|
-
result: parsedResult,
|
|
1250
|
-
error: errorContext,
|
|
1251
|
-
tokenUsage: extractTokenUsage(fullOutput, providerName),
|
|
1252
|
-
});
|
|
1253
|
-
}
|
|
1254
|
-
} catch (statusErr) {
|
|
1255
|
-
// Log error but continue checking (transient failures are common)
|
|
1256
|
-
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
1257
|
-
}
|
|
1258
|
-
}, 2000); // Check every 2 seconds (was 500ms in polling mode)
|
|
1259
|
-
|
|
1260
|
-
// Safety timeout (0 = no timeout, task runs until completion)
|
|
1261
1521
|
if (agent.timeout > 0) {
|
|
1262
1522
|
setTimeout(() => {
|
|
1263
|
-
if (!taskExited) {
|
|
1523
|
+
if (!state.taskExited) {
|
|
1264
1524
|
cleanup();
|
|
1265
1525
|
reject(new Error(`Task ${taskId} timeout after ${agent.timeout}ms (isolated mode)`));
|
|
1266
1526
|
}
|
|
@@ -1405,9 +1665,8 @@ function killTask(agent) {
|
|
|
1405
1665
|
// Also kill the underlying zeroshot task if we have a task ID
|
|
1406
1666
|
// This ensures the task process is stopped, not just our polling intervals
|
|
1407
1667
|
if (agent.currentTaskId) {
|
|
1408
|
-
const { exec } = require('child_process');
|
|
1409
1668
|
const ctPath = getClaudeTasksPath();
|
|
1410
|
-
exec(`${ctPath} task kill ${agent.currentTaskId}`, (error) => {
|
|
1669
|
+
exec(`${ctPath} task kill ${agent.currentTaskId}`, { timeout: 10000 }, (error) => {
|
|
1411
1670
|
if (error) {
|
|
1412
1671
|
// Task may have already completed or been killed, ignore errors
|
|
1413
1672
|
agent._log(`Note: Could not kill task ${agent.currentTaskId}: ${error.message}`);
|