@covibes/zeroshot 5.2.1 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +174 -189
- package/README.md +226 -195
- package/cli/commands/providers.js +149 -0
- package/cli/index.js +3145 -2366
- package/cli/lib/first-run.js +40 -3
- package/cli/message-formatters-normal.js +28 -6
- package/cluster-templates/base-templates/debug-workflow.json +24 -78
- package/cluster-templates/base-templates/full-workflow.json +99 -316
- package/cluster-templates/base-templates/single-worker.json +23 -15
- package/cluster-templates/base-templates/worker-validator.json +105 -36
- package/cluster-templates/conductor-bootstrap.json +9 -7
- package/lib/docker-config.js +14 -1
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +10 -7
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-detection.js +59 -0
- package/lib/provider-names.js +57 -0
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +298 -15
- package/lib/stream-json-parser.js +4 -238
- package/package.json +27 -6
- package/scripts/setup-merge-queue.sh +170 -0
- package/scripts/validate-templates.js +100 -0
- package/src/agent/agent-config.js +140 -63
- package/src/agent/agent-context-builder.js +336 -165
- package/src/agent/agent-hook-executor.js +337 -67
- package/src/agent/agent-lifecycle.js +386 -287
- package/src/agent/agent-stuck-detector.js +7 -7
- package/src/agent/agent-task-executor.js +944 -683
- package/src/agent/output-extraction.js +217 -0
- package/src/agent/output-reformatter.js +175 -0
- package/src/agent/schema-utils.js +146 -0
- package/src/agent-wrapper.js +112 -31
- package/src/agents/git-pusher-template.js +285 -0
- package/src/claude-task-runner.js +145 -44
- package/src/config-router.js +13 -13
- package/src/config-validator.js +1049 -563
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +499 -320
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +273 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +179 -0
- package/src/issue-providers/gitlab-provider.js +241 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +239 -0
- package/src/ledger.js +50 -11
- package/src/lib/safe-exec.js +88 -0
- package/src/orchestrator.js +1348 -757
- package/src/preflight.js +306 -149
- package/src/process-metrics.js +98 -56
- package/src/providers/anthropic/cli-builder.js +73 -0
- package/src/providers/anthropic/index.js +204 -0
- package/src/providers/anthropic/models.js +23 -0
- package/src/providers/anthropic/output-parser.js +177 -0
- package/src/providers/base-provider.js +251 -0
- package/src/providers/capabilities.js +60 -0
- package/src/providers/google/cli-builder.js +55 -0
- package/src/providers/google/index.js +116 -0
- package/src/providers/google/models.js +24 -0
- package/src/providers/google/output-parser.js +101 -0
- package/src/providers/index.js +91 -0
- package/src/providers/openai/cli-builder.js +133 -0
- package/src/providers/openai/index.js +136 -0
- package/src/providers/openai/models.js +21 -0
- package/src/providers/openai/output-parser.js +143 -0
- package/src/providers/opencode/cli-builder.js +42 -0
- package/src/providers/opencode/index.js +103 -0
- package/src/providers/opencode/models.js +55 -0
- package/src/providers/opencode/output-parser.js +122 -0
- package/src/schemas/sub-cluster.js +68 -39
- package/src/status-footer.js +94 -48
- package/src/sub-cluster-wrapper.js +92 -36
- package/src/task-runner.js +8 -6
- package/src/template-resolver.js +12 -9
- package/src/tui/data-poller.js +123 -99
- package/src/tui/formatters.js +4 -3
- package/src/tui/keybindings.js +259 -318
- package/src/tui/layout.js +20 -3
- package/src/tui/renderer.js +11 -21
- package/task-lib/attachable-watcher.js +150 -111
- package/task-lib/claude-recovery.js +156 -0
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/list.js +3 -3
- package/task-lib/commands/logs.js +231 -189
- package/task-lib/commands/resume.js +3 -2
- package/task-lib/commands/run.js +12 -3
- package/task-lib/commands/schedules.js +111 -61
- package/task-lib/config.js +0 -2
- package/task-lib/runner.js +128 -50
- package/task-lib/scheduler.js +3 -3
- package/task-lib/store.js +464 -152
- package/task-lib/tui/formatters.js +94 -45
- package/task-lib/tui/renderer.js +13 -3
- package/task-lib/tui.js +73 -32
- package/task-lib/watcher.js +157 -100
- package/src/agents/git-pusher-agent.json +0 -20
- package/src/github.js +0 -103
|
@@ -14,6 +14,42 @@ const { spawn } = require('child_process');
|
|
|
14
14
|
const path = require('path');
|
|
15
15
|
const fs = require('fs');
|
|
16
16
|
const os = require('os');
|
|
17
|
+
const { exec, execSync } = require('../lib/safe-exec'); // Enforces timeouts - prevents infinite hangs
|
|
18
|
+
const { getProvider, parseChunkWithProvider } = require('../providers');
|
|
19
|
+
const { getTask } = require('../../task-lib/store.js');
|
|
20
|
+
const { loadSettings } = require('../../lib/settings.js');
|
|
21
|
+
const { resolveClaudeAuth } = require('../../lib/settings/claude-auth.js');
|
|
22
|
+
|
|
23
|
+
// Schema utilities for normalizing LLM output
|
|
24
|
+
const { normalizeEnumValues } = require('./schema-utils');
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Build Claude-specific environment variables for task spawning
|
|
28
|
+
* Consolidates auth resolution and model mapping logic used by both isolated and non-isolated modes
|
|
29
|
+
* @param {Object} modelSpec - Model specification from agent
|
|
30
|
+
* @param {Object} [options] - Options
|
|
31
|
+
* @param {boolean} [options.includeAuth=true] - Include auth env vars (false for isolated mode where IsolationManager handles auth)
|
|
32
|
+
* @returns {Object} Environment variables to merge into spawn env
|
|
33
|
+
*/
|
|
34
|
+
function buildClaudeEnv(modelSpec, options = {}) {
|
|
35
|
+
const { includeAuth = true } = options;
|
|
36
|
+
const env = {};
|
|
37
|
+
|
|
38
|
+
if (includeAuth) {
|
|
39
|
+
const settings = loadSettings();
|
|
40
|
+
const authEnv = resolveClaudeAuth(settings);
|
|
41
|
+
Object.assign(env, authEnv);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (modelSpec?.model) {
|
|
45
|
+
env.ANTHROPIC_MODEL = modelSpec.model;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
49
|
+
env.ZEROSHOT_BLOCK_ASK_USER = '1';
|
|
50
|
+
|
|
51
|
+
return env;
|
|
52
|
+
}
|
|
17
53
|
|
|
18
54
|
/**
|
|
19
55
|
* Validate and sanitize error messages.
|
|
@@ -34,12 +70,17 @@ function sanitizeErrorMessage(error) {
|
|
|
34
70
|
/^unknown$/i,
|
|
35
71
|
/^void$/i,
|
|
36
72
|
/^never$/i,
|
|
37
|
-
/^[A-Z][a-zA-Z]*\s*\|\s*(null|undefined)$/, // e.g., "Error | null"
|
|
38
|
-
/^[a-z]+(\s*\|\s*[a-z]+)+$/i, // e.g., "string | number | boolean"
|
|
73
|
+
/^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
|
|
39
74
|
];
|
|
40
75
|
|
|
76
|
+
const trimmedError = error.trim();
|
|
77
|
+
|
|
78
|
+
// Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
|
|
79
|
+
const unionParts = trimmedError.split(/\s*\|\s*/);
|
|
80
|
+
const isUnionType = unionParts.length > 1 && unionParts.every((p) => /^[a-z]+$/i.test(p));
|
|
81
|
+
|
|
41
82
|
for (const pattern of typeAnnotationPatterns) {
|
|
42
|
-
if (pattern.test(
|
|
83
|
+
if (pattern.test(trimmedError) || isUnionType) {
|
|
43
84
|
console.warn(
|
|
44
85
|
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${error}". ` +
|
|
45
86
|
`This indicates corrupted data. Replacing with generic error.`
|
|
@@ -51,21 +92,6 @@ function sanitizeErrorMessage(error) {
|
|
|
51
92
|
return error;
|
|
52
93
|
}
|
|
53
94
|
|
|
54
|
-
/**
|
|
55
|
-
* Strip timestamp prefix from log lines.
|
|
56
|
-
* Log lines may have format: [epochMs]{json...} or [epochMs]text
|
|
57
|
-
*
|
|
58
|
-
* @param {string} line - Raw log line
|
|
59
|
-
* @returns {string} Line content without timestamp prefix, empty string for invalid input
|
|
60
|
-
*/
|
|
61
|
-
function stripTimestampPrefix(line) {
|
|
62
|
-
if (!line || typeof line !== 'string') return '';
|
|
63
|
-
const trimmed = line.trim().replace(/\r$/, '');
|
|
64
|
-
if (!trimmed) return '';
|
|
65
|
-
const match = trimmed.match(/^\[(\d{13})\](.*)$/);
|
|
66
|
-
return match ? match[2] : trimmed;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
95
|
/**
|
|
70
96
|
* Extract error context from task output.
|
|
71
97
|
* Shared by both isolated and non-isolated modes.
|
|
@@ -91,13 +117,62 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
|
|
|
91
117
|
}
|
|
92
118
|
}
|
|
93
119
|
|
|
94
|
-
//
|
|
95
|
-
const
|
|
96
|
-
|
|
97
|
-
|
|
120
|
+
// KNOWN CLAUDE CODE LIMITATIONS - detect and provide actionable guidance
|
|
121
|
+
const fullOutput = output || '';
|
|
122
|
+
|
|
123
|
+
// 256KB file limit error
|
|
124
|
+
if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
|
|
125
|
+
return sanitizeErrorMessage(
|
|
126
|
+
`FILE TOO LARGE (Claude Code 256KB limit). ` +
|
|
127
|
+
`Use offset and limit parameters when reading large files. ` +
|
|
128
|
+
`Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
|
|
129
|
+
);
|
|
98
130
|
}
|
|
99
131
|
|
|
100
|
-
//
|
|
132
|
+
// Streaming mode error (interactive tools in non-interactive mode)
|
|
133
|
+
if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
|
|
134
|
+
return sanitizeErrorMessage(
|
|
135
|
+
`STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
|
|
136
|
+
`This usually happens with AskUserQuestion or interactive prompts. ` +
|
|
137
|
+
`Zeroshot agents must run non-interactively.`
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
|
|
142
|
+
// If output is too verbose, that's a prompt problem - fix the prompts, not the data
|
|
143
|
+
const trimmedOutput = (output || '').trim();
|
|
144
|
+
if (!trimmedOutput) {
|
|
145
|
+
return sanitizeErrorMessage(
|
|
146
|
+
'Task failed with no output (check if task was interrupted or timed out)'
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Try to extract structured JSON from output first - it may contain the actual result
|
|
151
|
+
// even if the task was marked as "failed" due to timeout/stale status
|
|
152
|
+
try {
|
|
153
|
+
const { extractJsonFromOutput } = require('./output-extraction');
|
|
154
|
+
const extracted = extractJsonFromOutput(trimmedOutput);
|
|
155
|
+
if (extracted && typeof extracted === 'object') {
|
|
156
|
+
// If we found valid JSON, return it as the error context
|
|
157
|
+
// This preserves the actual agent output for downstream processing
|
|
158
|
+
return JSON.stringify(extracted);
|
|
159
|
+
}
|
|
160
|
+
} catch {
|
|
161
|
+
// Extraction failed, fall through to error pattern matching
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Extract non-JSON lines only (JSON lines contain "is_error": true which falsely matches)
|
|
165
|
+
const nonJsonLines = trimmedOutput
|
|
166
|
+
.split('\n')
|
|
167
|
+
.filter((line) => {
|
|
168
|
+
const trimmed = line.trim();
|
|
169
|
+
// Skip JSON objects and JSON-like content
|
|
170
|
+
return trimmed && !trimmed.startsWith('{') && !trimmed.startsWith('"');
|
|
171
|
+
})
|
|
172
|
+
.join('\n');
|
|
173
|
+
|
|
174
|
+
// Common error patterns - match against non-JSON content
|
|
175
|
+
const textToSearch = nonJsonLines || trimmedOutput;
|
|
101
176
|
const errorPatterns = [
|
|
102
177
|
/Error:\s*(.+)/i,
|
|
103
178
|
/error:\s*(.+)/i,
|
|
@@ -107,14 +182,16 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false
|
|
|
107
182
|
];
|
|
108
183
|
|
|
109
184
|
for (const pattern of errorPatterns) {
|
|
110
|
-
const match =
|
|
185
|
+
const match = textToSearch.match(pattern);
|
|
111
186
|
if (match) {
|
|
112
|
-
|
|
187
|
+
// Don't truncate - let the full error message through
|
|
188
|
+
return sanitizeErrorMessage(match[1]);
|
|
113
189
|
}
|
|
114
190
|
}
|
|
115
191
|
|
|
116
|
-
// No pattern matched -
|
|
117
|
-
|
|
192
|
+
// No pattern matched - return full output (no truncation)
|
|
193
|
+
// If this is too long, the solution is to make agents output less, not to corrupt data
|
|
194
|
+
return sanitizeErrorMessage(`Task failed. Output: ${trimmedOutput}`);
|
|
118
195
|
}
|
|
119
196
|
|
|
120
197
|
// Track if we've already ensured the AskUserQuestion hook is installed
|
|
@@ -130,36 +207,26 @@ let dangerousGitHookInstalled = false;
|
|
|
130
207
|
* @param {string} output - Full NDJSON output from Claude CLI
|
|
131
208
|
* @returns {Object|null} Token usage data or null if not found
|
|
132
209
|
*/
|
|
133
|
-
function extractTokenUsage(output) {
|
|
210
|
+
function extractTokenUsage(output, providerName = 'claude') {
|
|
134
211
|
if (!output) return null;
|
|
135
212
|
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
for (const line of lines) {
|
|
140
|
-
const content = stripTimestampPrefix(line);
|
|
141
|
-
if (!content) continue;
|
|
213
|
+
const provider = getProvider(providerName);
|
|
214
|
+
const events = parseChunkWithProvider(provider, output);
|
|
215
|
+
const resultEvent = events.find((event) => event.type === 'result');
|
|
142
216
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
if (event.type === 'result') {
|
|
146
|
-
const usage = event.usage || {};
|
|
147
|
-
return {
|
|
148
|
-
inputTokens: usage.input_tokens || 0,
|
|
149
|
-
outputTokens: usage.output_tokens || 0,
|
|
150
|
-
cacheReadInputTokens: usage.cache_read_input_tokens || 0,
|
|
151
|
-
cacheCreationInputTokens: usage.cache_creation_input_tokens || 0,
|
|
152
|
-
totalCostUsd: event.total_cost_usd || null,
|
|
153
|
-
durationMs: event.duration_ms || null,
|
|
154
|
-
modelUsage: event.modelUsage || null,
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
} catch {
|
|
158
|
-
// Not valid JSON, continue
|
|
159
|
-
}
|
|
217
|
+
if (!resultEvent) {
|
|
218
|
+
return null;
|
|
160
219
|
}
|
|
161
220
|
|
|
162
|
-
return
|
|
221
|
+
return {
|
|
222
|
+
inputTokens: resultEvent.inputTokens || 0,
|
|
223
|
+
outputTokens: resultEvent.outputTokens || 0,
|
|
224
|
+
cacheReadInputTokens: resultEvent.cacheReadInputTokens || 0,
|
|
225
|
+
cacheCreationInputTokens: resultEvent.cacheCreationInputTokens || 0,
|
|
226
|
+
totalCostUsd: resultEvent.cost || null,
|
|
227
|
+
durationMs: resultEvent.duration || null,
|
|
228
|
+
modelUsage: resultEvent.modelUsage || null,
|
|
229
|
+
};
|
|
163
230
|
}
|
|
164
231
|
|
|
165
232
|
/**
|
|
@@ -325,6 +392,9 @@ function ensureDangerousGitHook() {
|
|
|
325
392
|
* @returns {Promise<Object>} Result object { success, output, error }
|
|
326
393
|
*/
|
|
327
394
|
async function spawnClaudeTask(agent, context) {
|
|
395
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
396
|
+
const modelSpec = resolveAgentModelSpec(agent);
|
|
397
|
+
|
|
328
398
|
const ctPath = getClaudeTasksPath();
|
|
329
399
|
const cwd = agent.config.cwd || process.cwd();
|
|
330
400
|
|
|
@@ -332,50 +402,27 @@ async function spawnClaudeTask(agent, context) {
|
|
|
332
402
|
// CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
|
|
333
403
|
// strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
|
|
334
404
|
// strictSchema=false uses stream-json with post-run validation (live logs but fragile)
|
|
335
|
-
const desiredOutputFormat = agent
|
|
336
|
-
const
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
// Add verification mode flag if configured
|
|
344
|
-
if (agent.config.verificationMode) {
|
|
345
|
-
args.push('-v');
|
|
346
|
-
}
|
|
405
|
+
const { desiredOutputFormat, runOutputFormat } = resolveOutputFormatConfig(agent);
|
|
406
|
+
const args = buildTaskRunArgs({
|
|
407
|
+
agent,
|
|
408
|
+
providerName,
|
|
409
|
+
modelSpec,
|
|
410
|
+
runOutputFormat,
|
|
411
|
+
});
|
|
347
412
|
|
|
348
413
|
// NOTE: maxRetries is handled by the agent wrapper's internal retry loop,
|
|
349
414
|
// not passed to the CLI. See _handleTrigger() for retry logic.
|
|
350
415
|
|
|
351
|
-
|
|
352
|
-
// If we are running stream-json for live logs (strictSchema=false), do NOT pass schema to CLI.
|
|
353
|
-
if (agent.config.jsonSchema) {
|
|
354
|
-
if (runOutputFormat === 'json') {
|
|
355
|
-
// strictSchema=true OR no schema conflict: pass schema to CLI for native enforcement
|
|
356
|
-
const schema = JSON.stringify(agent.config.jsonSchema);
|
|
357
|
-
args.push('--json-schema', schema);
|
|
358
|
-
} else if (!agent.quiet) {
|
|
359
|
-
agent._log(
|
|
360
|
-
`[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
|
|
361
|
-
);
|
|
362
|
-
}
|
|
363
|
-
}
|
|
416
|
+
maybeLogStreamJsonNotice(agent, runOutputFormat);
|
|
364
417
|
|
|
365
418
|
// If schema enforcement is desired but we had to run stream-json for live logs,
|
|
366
419
|
// add explicit output instructions so the model still knows the required shape.
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
desiredOutputFormat
|
|
371
|
-
runOutputFormat
|
|
372
|
-
)
|
|
373
|
-
finalContext += `\n\n## Output Format (REQUIRED)\n\nReturn a JSON object that matches this schema exactly.\n\nSchema:\n\`\`\`json\n${JSON.stringify(
|
|
374
|
-
agent.config.jsonSchema,
|
|
375
|
-
null,
|
|
376
|
-
2
|
|
377
|
-
)}\n\`\`\`\n`;
|
|
378
|
-
}
|
|
420
|
+
const finalContext = buildFinalContext({
|
|
421
|
+
agent,
|
|
422
|
+
context,
|
|
423
|
+
desiredOutputFormat,
|
|
424
|
+
runOutputFormat,
|
|
425
|
+
});
|
|
379
426
|
|
|
380
427
|
args.push(finalContext);
|
|
381
428
|
|
|
@@ -397,44 +444,188 @@ async function spawnClaudeTask(agent, context) {
|
|
|
397
444
|
return spawnClaudeTaskIsolated(agent, context);
|
|
398
445
|
}
|
|
399
446
|
|
|
400
|
-
// NON-ISOLATION MODE:
|
|
447
|
+
// NON-ISOLATION MODE: For Claude, use user's existing Claude config
|
|
401
448
|
// AskUserQuestion blocking handled via:
|
|
402
|
-
// 1. Prompt injection (see agent-context-builder)
|
|
449
|
+
// 1. Prompt injection (see agent-context-builder)
|
|
403
450
|
// 2. PreToolUse hook (defense-in-depth) - activated by ZEROSHOT_BLOCK_ASK_USER env var
|
|
404
|
-
|
|
451
|
+
ensureProviderHooks(agent, providerName);
|
|
452
|
+
const spawnEnv = buildSpawnEnv(agent, providerName, modelSpec);
|
|
453
|
+
const taskId = await spawnTaskProcess({
|
|
454
|
+
agent,
|
|
455
|
+
ctPath,
|
|
456
|
+
args,
|
|
457
|
+
cwd,
|
|
458
|
+
spawnEnv,
|
|
459
|
+
});
|
|
460
|
+
|
|
461
|
+
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
|
|
462
|
+
|
|
463
|
+
// Wait for task to be registered in zeroshot storage (race condition fix)
|
|
464
|
+
await waitForTaskReady(agent, taskId);
|
|
465
|
+
|
|
466
|
+
// CRITICAL: Poll for REAL process PID from task store
|
|
467
|
+
// The watcher spawns the actual CLI and writes PID to SQLite asynchronously.
|
|
468
|
+
// We must poll because the watcher runs in a forked process.
|
|
469
|
+
const MAX_PID_POLLS = 30; // 3 seconds max
|
|
470
|
+
const PID_POLL_DELAY = 100;
|
|
471
|
+
let realPid = null;
|
|
472
|
+
|
|
473
|
+
for (let i = 0; i < MAX_PID_POLLS; i++) {
|
|
474
|
+
const taskInfo = getTask(taskId);
|
|
475
|
+
if (taskInfo?.pid) {
|
|
476
|
+
realPid = taskInfo.pid;
|
|
477
|
+
break;
|
|
478
|
+
}
|
|
479
|
+
await new Promise((r) => setTimeout(r, PID_POLL_DELAY));
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if (realPid) {
|
|
483
|
+
agent.processPid = realPid;
|
|
484
|
+
agent._publishLifecycle('PROCESS_SPAWNED', { pid: realPid });
|
|
485
|
+
agent._log(`📋 Agent ${agent.id}: Process PID: ${realPid}`);
|
|
486
|
+
} else {
|
|
487
|
+
agent._log(`⚠️ Agent ${agent.id}: PID not available (task may use non-standard watcher)`);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
// Now follow the logs and stream output
|
|
491
|
+
return followClaudeTaskLogs(agent, taskId);
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function resolveAgentModelSpec(agent) {
|
|
495
|
+
return agent._resolveModelSpec ? agent._resolveModelSpec() : { model: agent._selectModel() };
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
function resolveOutputFormatConfig(agent) {
|
|
499
|
+
// CRITICAL: Default to strict schema validation to prevent cluster crashes from parse failures
|
|
500
|
+
// strictSchema=true uses Claude CLI's native --json-schema enforcement (no streaming but guaranteed structure)
|
|
501
|
+
// strictSchema=false uses stream-json with post-run validation (live logs but fragile)
|
|
502
|
+
const desiredOutputFormat = agent.config.outputFormat || 'json';
|
|
503
|
+
const strictSchema = agent.config.strictSchema !== false; // DEFAULT TO TRUE
|
|
504
|
+
const runOutputFormat =
|
|
505
|
+
agent.config.jsonSchema && desiredOutputFormat === 'json' && !strictSchema
|
|
506
|
+
? 'stream-json'
|
|
507
|
+
: desiredOutputFormat;
|
|
508
|
+
|
|
509
|
+
return { desiredOutputFormat, strictSchema, runOutputFormat };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) {
|
|
513
|
+
const args = ['task', 'run', '--output-format', runOutputFormat, '--provider', providerName];
|
|
514
|
+
|
|
515
|
+
if (modelSpec?.model) {
|
|
516
|
+
args.push('--model', modelSpec.model);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
if (modelSpec?.reasoningEffort) {
|
|
520
|
+
args.push('--reasoning-effort', modelSpec.reasoningEffort);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// Add verification mode flag if configured
|
|
524
|
+
if (agent.config.verificationMode) {
|
|
525
|
+
args.push('-v');
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// Add JSON schema if specified in agent config.
|
|
529
|
+
// If we are running stream-json for live logs (strictSchema=false), do NOT pass schema to CLI.
|
|
530
|
+
if (agent.config.jsonSchema && runOutputFormat === 'json') {
|
|
531
|
+
const schema = JSON.stringify(agent.config.jsonSchema);
|
|
532
|
+
args.push('--json-schema', schema);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
return args;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function maybeLogStreamJsonNotice(agent, runOutputFormat) {
|
|
539
|
+
if (agent.config.jsonSchema && runOutputFormat !== 'json' && !agent.quiet) {
|
|
540
|
+
agent._log(
|
|
541
|
+
`[Agent ${agent.id}] jsonSchema configured; running stream-json for live logs (strictSchema=false). Schema will be validated after completion.`
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
function buildFinalContext({ agent, context, desiredOutputFormat, runOutputFormat }) {
|
|
547
|
+
if (
|
|
548
|
+
agent.config.jsonSchema &&
|
|
549
|
+
desiredOutputFormat === 'json' &&
|
|
550
|
+
runOutputFormat === 'stream-json'
|
|
551
|
+
) {
|
|
552
|
+
return (
|
|
553
|
+
context +
|
|
554
|
+
`\n\n## Output Format (REQUIRED)\n\nReturn a JSON object that matches this schema exactly.\n\nSchema:\n\`\`\`json\n${JSON.stringify(
|
|
555
|
+
agent.config.jsonSchema,
|
|
556
|
+
null,
|
|
557
|
+
2
|
|
558
|
+
)}\n\`\`\`\n`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
return context;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function ensureProviderHooks(agent, providerName) {
|
|
566
|
+
if (providerName !== 'claude') {
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
569
|
+
|
|
405
570
|
ensureAskUserQuestionHook();
|
|
406
571
|
|
|
407
572
|
// WORKTREE MODE: Install git safety hook (blocks dangerous git commands)
|
|
408
573
|
if (agent.worktree?.enabled) {
|
|
409
574
|
ensureDangerousGitHook();
|
|
410
575
|
}
|
|
576
|
+
}
|
|
411
577
|
|
|
412
|
-
|
|
413
|
-
const spawnEnv = {
|
|
414
|
-
...process.env,
|
|
415
|
-
ANTHROPIC_MODEL: agent._selectModel(),
|
|
416
|
-
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
417
|
-
ZEROSHOT_BLOCK_ASK_USER: '1',
|
|
418
|
-
};
|
|
578
|
+
function buildSpawnEnv(agent, providerName, modelSpec) {
|
|
579
|
+
const spawnEnv = { ...process.env };
|
|
419
580
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
581
|
+
if (providerName === 'claude') {
|
|
582
|
+
Object.assign(spawnEnv, buildClaudeEnv(modelSpec));
|
|
583
|
+
|
|
584
|
+
// WORKTREE MODE: Activate git safety hook via environment variable
|
|
585
|
+
if (agent.worktree?.enabled) {
|
|
586
|
+
spawnEnv.ZEROSHOT_WORKTREE = '1';
|
|
587
|
+
}
|
|
424
588
|
}
|
|
425
589
|
|
|
426
|
-
|
|
590
|
+
return spawnEnv;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function parseTaskIdFromOutput(stdout) {
|
|
594
|
+
const match = stdout.match(/Task spawned: ((?:task-)?[a-z]+-[a-z]+-[a-z0-9]+)/);
|
|
595
|
+
return match ? match[1] : null;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function spawnTaskProcess({ agent, ctPath, args, cwd, spawnEnv }) {
|
|
599
|
+
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
600
|
+
const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
|
|
601
|
+
|
|
602
|
+
return new Promise((resolve, reject) => {
|
|
427
603
|
const proc = spawn(ctPath, args, {
|
|
428
604
|
cwd,
|
|
429
605
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
430
606
|
env: spawnEnv,
|
|
431
607
|
});
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
608
|
+
|
|
609
|
+
// NOTE: Don't emit PROCESS_SPAWNED here - proc.pid is a wrapper that exits immediately.
|
|
610
|
+
// Real PID comes from task store after watcher spawns the actual CLI process.
|
|
611
|
+
// PROCESS_SPAWNED is emitted in spawnClaudeTask after waitForTaskReady + PID polling.
|
|
435
612
|
|
|
436
613
|
let stdout = '';
|
|
437
614
|
let stderr = '';
|
|
615
|
+
let resolved = false;
|
|
616
|
+
|
|
617
|
+
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
|
|
618
|
+
const spawnTimeout = setTimeout(() => {
|
|
619
|
+
if (resolved) return;
|
|
620
|
+
resolved = true;
|
|
621
|
+
proc.kill('SIGKILL');
|
|
622
|
+
reject(
|
|
623
|
+
new Error(
|
|
624
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
625
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
626
|
+
)
|
|
627
|
+
);
|
|
628
|
+
}, SPAWN_TIMEOUT_MS);
|
|
438
629
|
|
|
439
630
|
proc.stdout.on('data', (data) => {
|
|
440
631
|
stdout += data.toString();
|
|
@@ -445,6 +636,9 @@ async function spawnClaudeTask(agent, context) {
|
|
|
445
636
|
});
|
|
446
637
|
|
|
447
638
|
proc.on('close', (code, signal) => {
|
|
639
|
+
clearTimeout(spawnTimeout);
|
|
640
|
+
if (resolved) return;
|
|
641
|
+
resolved = true;
|
|
448
642
|
// Handle process killed by signal (e.g., SIGTERM, SIGKILL, SIGSTOP)
|
|
449
643
|
if (signal) {
|
|
450
644
|
reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
|
|
@@ -454,9 +648,8 @@ async function spawnClaudeTask(agent, context) {
|
|
|
454
648
|
if (code === 0) {
|
|
455
649
|
// Parse task ID from output: "✓ Task spawned: xxx-yyy-nn"
|
|
456
650
|
// Format: <adjective>-<noun>-<digits> (may or may not have task- prefix)
|
|
457
|
-
const
|
|
458
|
-
if (
|
|
459
|
-
const spawnedTaskId = match[1];
|
|
651
|
+
const spawnedTaskId = parseTaskIdFromOutput(stdout);
|
|
652
|
+
if (spawnedTaskId) {
|
|
460
653
|
agent.currentTaskId = spawnedTaskId; // Track for resume capability
|
|
461
654
|
agent._publishLifecycle('TASK_ID_ASSIGNED', {
|
|
462
655
|
pid: agent.processPid,
|
|
@@ -479,17 +672,12 @@ async function spawnClaudeTask(agent, context) {
|
|
|
479
672
|
});
|
|
480
673
|
|
|
481
674
|
proc.on('error', (error) => {
|
|
675
|
+
clearTimeout(spawnTimeout);
|
|
676
|
+
if (resolved) return;
|
|
677
|
+
resolved = true;
|
|
482
678
|
reject(error);
|
|
483
679
|
});
|
|
484
680
|
});
|
|
485
|
-
|
|
486
|
-
agent._log(`📋 Agent ${agent.id}: Following zeroshot logs for ${taskId}`);
|
|
487
|
-
|
|
488
|
-
// Wait for task to be registered in zeroshot storage (race condition fix)
|
|
489
|
-
await waitForTaskReady(agent, taskId);
|
|
490
|
-
|
|
491
|
-
// Now follow the logs and stream output
|
|
492
|
-
return followClaudeTaskLogs(agent, taskId);
|
|
493
681
|
}
|
|
494
682
|
|
|
495
683
|
/**
|
|
@@ -501,16 +689,16 @@ async function spawnClaudeTask(agent, context) {
|
|
|
501
689
|
* @returns {Promise<void>}
|
|
502
690
|
*/
|
|
503
691
|
async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
504
|
-
const { exec } = require('child_process');
|
|
505
692
|
const ctPath = getClaudeTasksPath();
|
|
506
693
|
|
|
507
694
|
for (let i = 0; i < maxRetries; i++) {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
695
|
+
let exists = false;
|
|
696
|
+
try {
|
|
697
|
+
const { stdout } = await exec(`${ctPath} status ${taskId}`, { timeout: 5000 });
|
|
698
|
+
exists = !stdout.includes('Task not found');
|
|
699
|
+
} catch {
|
|
700
|
+
// Timeout or error - task not ready yet
|
|
701
|
+
}
|
|
514
702
|
|
|
515
703
|
if (exists) return;
|
|
516
704
|
|
|
@@ -518,305 +706,372 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
|
518
706
|
await new Promise((r) => setTimeout(r, delayMs));
|
|
519
707
|
}
|
|
520
708
|
|
|
521
|
-
//
|
|
522
|
-
|
|
709
|
+
// FAIL FAST: Task not found after retries = unrecoverable error
|
|
710
|
+
// Continuing with a non-existent task causes 30s of pointless polling then crash
|
|
711
|
+
throw new Error(
|
|
712
|
+
`Task ${taskId} not found after ${maxRetries} retries (${maxRetries * delayMs}ms). ` +
|
|
713
|
+
`Task spawn may have failed silently. Check zeroshot task run output.`
|
|
714
|
+
);
|
|
523
715
|
}
|
|
524
716
|
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
717
|
+
const MAX_STATUS_FAILURES = 30;
|
|
718
|
+
|
|
719
|
+
function createLogFollowState() {
|
|
720
|
+
return {
|
|
721
|
+
output: '',
|
|
722
|
+
logFilePath: null,
|
|
723
|
+
lastSize: 0,
|
|
724
|
+
pollInterval: null,
|
|
725
|
+
statusCheckInterval: null,
|
|
726
|
+
resolved: false,
|
|
727
|
+
lineBuffer: '',
|
|
728
|
+
consecutiveExecFailures: 0,
|
|
729
|
+
};
|
|
730
|
+
}
|
|
536
731
|
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
732
|
+
function lookupLogFilePath(ctPath, taskId) {
|
|
733
|
+
try {
|
|
734
|
+
return execSync(`${ctPath} get-log-path ${taskId}`, {
|
|
735
|
+
encoding: 'utf-8',
|
|
736
|
+
timeout: 5000,
|
|
737
|
+
}).trim();
|
|
738
|
+
} catch {
|
|
739
|
+
return null;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
544
742
|
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
743
|
+
function parseTimestampedLine(line) {
|
|
744
|
+
let timestamp = Date.now();
|
|
745
|
+
let content = line.replace(/\r$/, '');
|
|
746
|
+
|
|
747
|
+
const timestampMatch = content.match(/^\[(\d{13})\](.*)$/);
|
|
748
|
+
if (timestampMatch) {
|
|
749
|
+
timestamp = parseInt(timestampMatch[1], 10);
|
|
750
|
+
content = timestampMatch[2];
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
return { timestamp, content };
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function shouldSkipLogLine(content) {
|
|
757
|
+
return (
|
|
758
|
+
content.startsWith('===') ||
|
|
759
|
+
content.startsWith('Finished:') ||
|
|
760
|
+
content.startsWith('Exit code:') ||
|
|
761
|
+
(content.includes('"type":"system"') && content.includes('"subtype":"init"'))
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
function isValidJsonLine(content) {
|
|
766
|
+
if (!content.trim().startsWith('{')) {
|
|
767
|
+
return false;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
try {
|
|
771
|
+
JSON.parse(content);
|
|
772
|
+
return true;
|
|
773
|
+
} catch {
|
|
774
|
+
return false;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
779
|
+
if (!line.trim()) return;
|
|
780
|
+
|
|
781
|
+
const { timestamp, content } = parseTimestampedLine(line);
|
|
782
|
+
if (shouldSkipLogLine(content)) {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const isValidJson = isValidJsonLine(content);
|
|
787
|
+
state.output += content + '\n';
|
|
788
|
+
|
|
789
|
+
agent.lastOutputTime = Date.now();
|
|
790
|
+
|
|
791
|
+
agent._publish({
|
|
792
|
+
topic: 'AGENT_OUTPUT',
|
|
793
|
+
receiver: 'broadcast',
|
|
794
|
+
timestamp,
|
|
795
|
+
content: {
|
|
796
|
+
text: content,
|
|
797
|
+
data: {
|
|
798
|
+
type: isValidJson ? 'json' : 'text',
|
|
799
|
+
line: content,
|
|
800
|
+
agent: agent.id,
|
|
801
|
+
role: agent.role,
|
|
802
|
+
iteration: agent.iteration,
|
|
803
|
+
provider: providerName,
|
|
804
|
+
},
|
|
805
|
+
},
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function appendContentToBuffer(state, content, onLine) {
|
|
810
|
+
state.lineBuffer += content;
|
|
811
|
+
const lines = state.lineBuffer.split('\n');
|
|
812
|
+
|
|
813
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
814
|
+
onLine(lines[i]);
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
state.lineBuffer = lines[lines.length - 1];
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function pollLogFileForUpdates({ agent, fsModule, ctPath, taskId, state, onNewContent }) {
|
|
821
|
+
if (!state.logFilePath) {
|
|
822
|
+
const logFilePath = lookupLogFilePath(ctPath, taskId);
|
|
823
|
+
if (!logFilePath) {
|
|
824
|
+
return;
|
|
554
825
|
}
|
|
826
|
+
state.logFilePath = logFilePath;
|
|
827
|
+
agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
|
|
828
|
+
}
|
|
555
829
|
|
|
556
|
-
|
|
557
|
-
|
|
830
|
+
if (!fsModule.existsSync(state.logFilePath)) {
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
558
833
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
const
|
|
562
|
-
if (!line.trim()) return;
|
|
834
|
+
try {
|
|
835
|
+
const stats = fsModule.statSync(state.logFilePath);
|
|
836
|
+
const currentSize = stats.size;
|
|
563
837
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
838
|
+
if (currentSize > state.lastSize) {
|
|
839
|
+
const fd = fsModule.openSync(state.logFilePath, 'r');
|
|
840
|
+
const buffer = Buffer.alloc(currentSize - state.lastSize);
|
|
841
|
+
fsModule.readSync(fd, buffer, 0, buffer.length, state.lastSize);
|
|
842
|
+
fsModule.closeSync(fd);
|
|
568
843
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
844
|
+
onNewContent(buffer.toString('utf-8'));
|
|
845
|
+
state.lastSize = currentSize;
|
|
846
|
+
}
|
|
847
|
+
} catch (err) {
|
|
848
|
+
const error = /** @type {Error} */ (err);
|
|
849
|
+
console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${error.message}`);
|
|
850
|
+
}
|
|
851
|
+
}
|
|
574
852
|
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
content.startsWith('Exit code:') ||
|
|
580
|
-
(content.includes('"type":"system"') && content.includes('"subtype":"init"'))
|
|
581
|
-
) {
|
|
582
|
-
return;
|
|
583
|
-
}
|
|
853
|
+
function stripAnsiCodes(value) {
|
|
854
|
+
const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
|
|
855
|
+
return value.replace(ansiPattern, '');
|
|
856
|
+
}
|
|
584
857
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
858
|
+
function parseStatusFlags(cleanStdout) {
|
|
859
|
+
return {
|
|
860
|
+
isCompleted: /Status:\s+completed/i.test(cleanStdout),
|
|
861
|
+
isFailed: /Status:\s+failed/i.test(cleanStdout),
|
|
862
|
+
isStale: /Status:\s+stale/i.test(cleanStdout),
|
|
863
|
+
};
|
|
864
|
+
}
|
|
588
865
|
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
} catch {
|
|
594
|
-
// Looks like JSON but isn't valid - treat as text
|
|
595
|
-
}
|
|
596
|
-
}
|
|
866
|
+
function determineStaleSuccess({ agent, output, providerName, taskId }) {
|
|
867
|
+
if (!output) {
|
|
868
|
+
return false;
|
|
869
|
+
}
|
|
597
870
|
|
|
598
|
-
|
|
599
|
-
|
|
871
|
+
const hasStructuredOutput = /"structured_output"\s*:/.test(output);
|
|
872
|
+
const hasSuccessResult = /"subtype"\s*:\s*"success"/.test(output);
|
|
873
|
+
let hasParsedOutput = false;
|
|
600
874
|
|
|
601
|
-
|
|
602
|
-
|
|
875
|
+
try {
|
|
876
|
+
const { extractJsonFromOutput } = require('./output-extraction');
|
|
877
|
+
hasParsedOutput = !!extractJsonFromOutput(output, providerName);
|
|
878
|
+
} catch {
|
|
879
|
+
// Ignore extraction errors - fallback to other signals
|
|
880
|
+
}
|
|
603
881
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
882
|
+
const success = hasStructuredOutput || hasSuccessResult || hasParsedOutput;
|
|
883
|
+
if (!agent.quiet) {
|
|
884
|
+
agent._log(
|
|
885
|
+
`[Agent ${agent.id}] Task ${taskId} is stale - recovered as ${success ? 'SUCCESS' : 'FAILURE'} based on output analysis`
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
return success;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
function finalizeLogFollow(agent, state) {
|
|
893
|
+
if (state.pollInterval) {
|
|
894
|
+
clearInterval(state.pollInterval);
|
|
895
|
+
}
|
|
896
|
+
if (state.statusCheckInterval) {
|
|
897
|
+
clearInterval(state.statusCheckInterval);
|
|
898
|
+
}
|
|
899
|
+
agent.currentTask = null;
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve }) {
|
|
903
|
+
if (!error) {
|
|
904
|
+
return false;
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
state.consecutiveExecFailures++;
|
|
908
|
+
if (state.consecutiveExecFailures < MAX_STATUS_FAILURES) {
|
|
909
|
+
return true;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
console.error(
|
|
913
|
+
`[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_STATUS_FAILURES} times consecutively! STOPPING.`
|
|
914
|
+
);
|
|
915
|
+
console.error(` Command: ${ctPath} status ${taskId}`);
|
|
916
|
+
console.error(` Error: ${error.message}`);
|
|
917
|
+
console.error(` Stderr: ${stderr || 'none'}`);
|
|
918
|
+
console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
|
|
919
|
+
|
|
920
|
+
if (!state.resolved) {
|
|
921
|
+
state.resolved = true;
|
|
922
|
+
finalizeLogFollow(agent, state);
|
|
923
|
+
|
|
924
|
+
agent._publish({
|
|
925
|
+
topic: 'AGENT_ERROR',
|
|
926
|
+
receiver: 'broadcast',
|
|
927
|
+
content: {
|
|
928
|
+
text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
|
|
929
|
+
data: {
|
|
930
|
+
taskId,
|
|
931
|
+
error: 'polling_timeout',
|
|
932
|
+
attempts: state.consecutiveExecFailures,
|
|
933
|
+
role: agent.role,
|
|
934
|
+
iteration: agent.iteration,
|
|
617
935
|
},
|
|
618
|
-
}
|
|
619
|
-
};
|
|
936
|
+
},
|
|
937
|
+
});
|
|
620
938
|
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
939
|
+
resolve({
|
|
940
|
+
success: false,
|
|
941
|
+
output: state.output,
|
|
942
|
+
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
943
|
+
});
|
|
944
|
+
}
|
|
625
945
|
|
|
626
|
-
|
|
627
|
-
|
|
946
|
+
return true;
|
|
947
|
+
}
|
|
628
948
|
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
949
|
+
function handleStatusCompletion({
|
|
950
|
+
agent,
|
|
951
|
+
taskId,
|
|
952
|
+
providerName,
|
|
953
|
+
state,
|
|
954
|
+
stdout,
|
|
955
|
+
pollLogFile,
|
|
956
|
+
resolve,
|
|
957
|
+
}) {
|
|
958
|
+
const cleanStdout = stripAnsiCodes(stdout);
|
|
959
|
+
const { isCompleted, isFailed, isStale } = parseStatusFlags(cleanStdout);
|
|
960
|
+
|
|
961
|
+
if (!isCompleted && !isFailed && !isStale) {
|
|
962
|
+
return false;
|
|
963
|
+
}
|
|
633
964
|
|
|
634
|
-
|
|
635
|
-
lineBuffer = lines[lines.length - 1];
|
|
636
|
-
};
|
|
637
|
-
|
|
638
|
-
// Poll the log file for new content
|
|
639
|
-
const pollLogFile = () => {
|
|
640
|
-
// If we don't have log path yet, try to get it
|
|
641
|
-
if (!logFilePath) {
|
|
642
|
-
try {
|
|
643
|
-
logFilePath = execSync(`${ctPath} get-log-path ${taskId}`, {
|
|
644
|
-
encoding: 'utf-8',
|
|
645
|
-
}).trim();
|
|
646
|
-
agent._log(`📋 Agent ${agent.id}: Found log file: ${logFilePath}`);
|
|
647
|
-
} catch {
|
|
648
|
-
return; // Not ready yet
|
|
649
|
-
}
|
|
650
|
-
}
|
|
965
|
+
pollLogFile();
|
|
651
966
|
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
967
|
+
let success = isCompleted;
|
|
968
|
+
if (isStale) {
|
|
969
|
+
success = determineStaleSuccess({ agent, output: state.output, providerName, taskId });
|
|
970
|
+
}
|
|
656
971
|
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
if (currentSize > lastSize) {
|
|
662
|
-
// Read new content
|
|
663
|
-
const fd = fsModule.openSync(logFilePath, 'r');
|
|
664
|
-
const buffer = Buffer.alloc(currentSize - lastSize);
|
|
665
|
-
fsModule.readSync(fd, buffer, 0, buffer.length, lastSize);
|
|
666
|
-
fsModule.closeSync(fd);
|
|
667
|
-
|
|
668
|
-
const newContent = buffer.toString('utf-8');
|
|
669
|
-
// Process new content line-by-line
|
|
670
|
-
processNewContent(newContent);
|
|
671
|
-
lastSize = currentSize;
|
|
672
|
-
}
|
|
673
|
-
} catch (err) {
|
|
674
|
-
// File might have been deleted or locked
|
|
675
|
-
console.warn(`⚠️ Agent ${agent.id}: Error reading log: ${err.message}`);
|
|
676
|
-
}
|
|
677
|
-
};
|
|
678
|
-
|
|
679
|
-
// Start polling log file (every 300ms for responsive streaming)
|
|
680
|
-
pollInterval = setInterval(pollLogFile, 300);
|
|
681
|
-
|
|
682
|
-
// Poll ct status to know when task is complete
|
|
683
|
-
// Track consecutive failures for debugging stuck clusters
|
|
684
|
-
let consecutiveExecFailures = 0;
|
|
685
|
-
const MAX_CONSECUTIVE_FAILURES = 30; // 30 seconds of failures = log warning
|
|
686
|
-
|
|
687
|
-
statusCheckInterval = setInterval(() => {
|
|
688
|
-
exec(`${ctPath} status ${taskId}`, (error, stdout, stderr) => {
|
|
689
|
-
if (resolved) return;
|
|
690
|
-
|
|
691
|
-
// Track exec failures - if status command keeps failing, something is wrong
|
|
692
|
-
if (error) {
|
|
693
|
-
consecutiveExecFailures++;
|
|
694
|
-
if (consecutiveExecFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
695
|
-
console.error(
|
|
696
|
-
`[Agent ${agent.id}] ⚠️ Status polling failed ${MAX_CONSECUTIVE_FAILURES} times consecutively! STOPPING.`
|
|
697
|
-
);
|
|
698
|
-
console.error(` Command: ${ctPath} status ${taskId}`);
|
|
699
|
-
console.error(` Error: ${error.message}`);
|
|
700
|
-
console.error(` Stderr: ${stderr || 'none'}`);
|
|
701
|
-
console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
|
|
702
|
-
|
|
703
|
-
// Stop polling and resolve with failure
|
|
704
|
-
if (!resolved) {
|
|
705
|
-
resolved = true;
|
|
706
|
-
clearInterval(pollInterval);
|
|
707
|
-
clearInterval(statusCheckInterval);
|
|
708
|
-
agent.currentTask = null;
|
|
709
|
-
|
|
710
|
-
// Publish error for orchestrator/resume
|
|
711
|
-
agent._publish({
|
|
712
|
-
topic: 'AGENT_ERROR',
|
|
713
|
-
receiver: 'broadcast',
|
|
714
|
-
content: {
|
|
715
|
-
text: `Task ${taskId} polling failed after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`,
|
|
716
|
-
data: {
|
|
717
|
-
taskId,
|
|
718
|
-
error: 'polling_timeout',
|
|
719
|
-
attempts: consecutiveExecFailures,
|
|
720
|
-
role: agent.role,
|
|
721
|
-
iteration: agent.iteration,
|
|
722
|
-
},
|
|
723
|
-
},
|
|
724
|
-
});
|
|
725
|
-
|
|
726
|
-
resolve({
|
|
727
|
-
success: false,
|
|
728
|
-
output,
|
|
729
|
-
error: `Status polling failed ${MAX_CONSECUTIVE_FAILURES} times - task may not exist`,
|
|
730
|
-
});
|
|
731
|
-
}
|
|
732
|
-
return;
|
|
733
|
-
}
|
|
734
|
-
return; // Keep polling - might be transient
|
|
735
|
-
}
|
|
972
|
+
setTimeout(() => {
|
|
973
|
+
if (state.resolved) return;
|
|
974
|
+
state.resolved = true;
|
|
736
975
|
|
|
737
|
-
|
|
738
|
-
consecutiveExecFailures = 0;
|
|
976
|
+
finalizeLogFollow(agent, state);
|
|
739
977
|
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
const ansiPattern = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
|
|
744
|
-
const cleanStdout = stdout.replace(ansiPattern, '');
|
|
745
|
-
// Use flexible whitespace matching in case spacing changes
|
|
746
|
-
const isCompleted = /Status:\s+completed/i.test(cleanStdout);
|
|
747
|
-
const isFailed = /Status:\s+failed/i.test(cleanStdout);
|
|
978
|
+
const errorContext = !success
|
|
979
|
+
? extractErrorContext({ output: state.output, statusOutput: stdout, taskId })
|
|
980
|
+
: null;
|
|
748
981
|
|
|
749
|
-
|
|
750
|
-
|
|
982
|
+
resolve({
|
|
983
|
+
success,
|
|
984
|
+
output: state.output,
|
|
985
|
+
error: errorContext,
|
|
986
|
+
tokenUsage: extractTokenUsage(state.output, providerName),
|
|
987
|
+
});
|
|
988
|
+
}, 500);
|
|
751
989
|
|
|
752
|
-
|
|
753
|
-
|
|
990
|
+
return true;
|
|
991
|
+
}
|
|
754
992
|
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
993
|
+
function buildKillHandler({ agent, state, providerName, resolve }) {
|
|
994
|
+
return {
|
|
995
|
+
kill: (reason = 'Task killed') => {
|
|
996
|
+
if (state.resolved) return;
|
|
997
|
+
state.resolved = true;
|
|
998
|
+
finalizeLogFollow(agent, state);
|
|
999
|
+
agent._stopLivenessCheck();
|
|
1000
|
+
resolve({
|
|
1001
|
+
success: false,
|
|
1002
|
+
output: state.output,
|
|
1003
|
+
error: reason,
|
|
1004
|
+
tokenUsage: extractTokenUsage(state.output, providerName),
|
|
1005
|
+
});
|
|
1006
|
+
},
|
|
1007
|
+
};
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function createLogFollower({ agent, taskId, fsModule, ctPath, providerName }) {
|
|
1011
|
+
return new Promise((resolve) => {
|
|
1012
|
+
const state = createLogFollowState();
|
|
1013
|
+
|
|
1014
|
+
state.logFilePath = lookupLogFilePath(ctPath, taskId);
|
|
1015
|
+
if (state.logFilePath) {
|
|
1016
|
+
agent._log(`📋 Agent ${agent.id}: Following ct logs for ${taskId}`);
|
|
1017
|
+
} else {
|
|
1018
|
+
agent._log(`⏳ Agent ${agent.id}: Waiting for log file...`);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
const broadcastLine = (line) => broadcastAgentLine({ agent, providerName, state, line });
|
|
1022
|
+
const processNewContent = (content) => appendContentToBuffer(state, content, broadcastLine);
|
|
1023
|
+
const pollLogFile = () =>
|
|
1024
|
+
pollLogFileForUpdates({
|
|
1025
|
+
agent,
|
|
1026
|
+
fsModule,
|
|
1027
|
+
ctPath,
|
|
1028
|
+
taskId,
|
|
1029
|
+
state,
|
|
1030
|
+
onNewContent: processNewContent,
|
|
1031
|
+
});
|
|
1032
|
+
|
|
1033
|
+
state.pollInterval = setInterval(pollLogFile, 300);
|
|
1034
|
+
|
|
1035
|
+
state.statusCheckInterval = setInterval(() => {
|
|
1036
|
+
exec(`${ctPath} status ${taskId}`, { timeout: 5000 }, (error, stdout, stderr) => {
|
|
1037
|
+
if (state.resolved) return;
|
|
1038
|
+
|
|
1039
|
+
if (handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve })) {
|
|
1040
|
+
return;
|
|
776
1041
|
}
|
|
1042
|
+
|
|
1043
|
+
state.consecutiveExecFailures = 0;
|
|
1044
|
+
handleStatusCompletion({
|
|
1045
|
+
agent,
|
|
1046
|
+
taskId,
|
|
1047
|
+
providerName,
|
|
1048
|
+
state,
|
|
1049
|
+
stdout,
|
|
1050
|
+
pollLogFile,
|
|
1051
|
+
resolve,
|
|
1052
|
+
});
|
|
777
1053
|
});
|
|
778
1054
|
}, 1000);
|
|
779
1055
|
|
|
780
|
-
|
|
781
|
-
// CRITICAL: Must reject promise to avoid orphaned promise that hangs forever
|
|
782
|
-
agent.currentTask = {
|
|
783
|
-
kill: (reason = 'Task killed') => {
|
|
784
|
-
if (resolved) return;
|
|
785
|
-
resolved = true;
|
|
786
|
-
clearInterval(pollInterval);
|
|
787
|
-
clearInterval(statusCheckInterval);
|
|
788
|
-
agent._stopLivenessCheck();
|
|
789
|
-
// BUGFIX: Resolve with failure instead of orphaning the promise
|
|
790
|
-
// This allows the caller to handle the kill gracefully
|
|
791
|
-
resolve({
|
|
792
|
-
success: false,
|
|
793
|
-
output,
|
|
794
|
-
error: reason,
|
|
795
|
-
tokenUsage: extractTokenUsage(output),
|
|
796
|
-
});
|
|
797
|
-
},
|
|
798
|
-
};
|
|
799
|
-
|
|
800
|
-
// REMOVED: Task timeout disabled - tasks run until completion or explicit kill
|
|
801
|
-
// Tasks should run until:
|
|
802
|
-
// - Completion
|
|
803
|
-
// - Explicit kill
|
|
804
|
-
// - External error (rate limit, API failure)
|
|
805
|
-
//
|
|
806
|
-
// setTimeout(() => {
|
|
807
|
-
// if (resolved) return;
|
|
808
|
-
// resolved = true;
|
|
809
|
-
//
|
|
810
|
-
// clearInterval(pollInterval);
|
|
811
|
-
// clearInterval(statusCheckInterval);
|
|
812
|
-
// agent._stopLivenessCheck();
|
|
813
|
-
// agent.currentTask = null;
|
|
814
|
-
// const timeoutMinutes = Math.round(agent.timeout / 60000);
|
|
815
|
-
// reject(new Error(`Task timed out after ${timeoutMinutes} minutes`));
|
|
816
|
-
// }, agent.timeout);
|
|
1056
|
+
agent.currentTask = buildKillHandler({ agent, state, providerName, resolve });
|
|
817
1057
|
});
|
|
818
1058
|
}
|
|
819
1059
|
|
|
1060
|
+
/**
|
|
1061
|
+
* Follow claude-zeroshots logs until completion, streaming to message bus
|
|
1062
|
+
* Reads log file directly for reliable streaming
|
|
1063
|
+
* @param {Object} agent - Agent instance
|
|
1064
|
+
* @param {String} taskId - Task ID to follow
|
|
1065
|
+
* @returns {Promise<Object>} Result object { success, output, error }
|
|
1066
|
+
*/
|
|
1067
|
+
function followClaudeTaskLogs(agent, taskId) {
|
|
1068
|
+
const fsModule = require('fs');
|
|
1069
|
+
const ctPath = getClaudeTasksPath();
|
|
1070
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1071
|
+
|
|
1072
|
+
return createLogFollower({ agent, taskId, fsModule, ctPath, providerName });
|
|
1073
|
+
}
|
|
1074
|
+
|
|
820
1075
|
/**
|
|
821
1076
|
* Get path to claude-zeroshots executable
|
|
822
1077
|
* @returns {String} Path to zeroshot command
|
|
@@ -835,6 +1090,10 @@ function getClaudeTasksPath() {
|
|
|
835
1090
|
*/
|
|
836
1091
|
async function spawnClaudeTaskIsolated(agent, context) {
|
|
837
1092
|
const { manager, clusterId } = agent.isolation;
|
|
1093
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1094
|
+
const modelSpec = agent._resolveModelSpec
|
|
1095
|
+
? agent._resolveModelSpec()
|
|
1096
|
+
: { model: agent._selectModel() };
|
|
838
1097
|
|
|
839
1098
|
agent._log(`📦 Agent ${agent.id}: Running task in isolated container using zeroshot task run...`);
|
|
840
1099
|
|
|
@@ -847,7 +1106,23 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
847
1106
|
? 'stream-json'
|
|
848
1107
|
: desiredOutputFormat;
|
|
849
1108
|
|
|
850
|
-
const command = [
|
|
1109
|
+
const command = [
|
|
1110
|
+
'zeroshot',
|
|
1111
|
+
'task',
|
|
1112
|
+
'run',
|
|
1113
|
+
'--output-format',
|
|
1114
|
+
runOutputFormat,
|
|
1115
|
+
'--provider',
|
|
1116
|
+
providerName,
|
|
1117
|
+
];
|
|
1118
|
+
|
|
1119
|
+
if (modelSpec?.model) {
|
|
1120
|
+
command.push('--model', modelSpec.model);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
if (modelSpec?.reasoningEffort) {
|
|
1124
|
+
command.push('--reasoning-effort', modelSpec.reasoningEffort);
|
|
1125
|
+
}
|
|
851
1126
|
|
|
852
1127
|
// Add verification mode flag if configured
|
|
853
1128
|
if (agent.config.verificationMode) {
|
|
@@ -885,14 +1160,15 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
885
1160
|
command.push(finalContext);
|
|
886
1161
|
|
|
887
1162
|
// STEP 1: Spawn task and extract task ID (same as non-isolated mode)
|
|
1163
|
+
// Timeout for spawn phase - if CLI hangs during init (e.g., opencode 429 bug), kill it
|
|
1164
|
+
const SPAWN_TIMEOUT_MS = 30000; // 30 seconds to spawn task
|
|
1165
|
+
// Note: Auth env vars are injected by IsolationManager, we only need model mapping here
|
|
1166
|
+
const isolatedEnv =
|
|
1167
|
+
providerName === 'claude' ? buildClaudeEnv(modelSpec, { includeAuth: false }) : {};
|
|
1168
|
+
|
|
888
1169
|
const taskId = await new Promise((resolve, reject) => {
|
|
889
|
-
const selectedModel = agent._selectModel();
|
|
890
1170
|
const proc = manager.spawnInContainer(clusterId, command, {
|
|
891
|
-
env:
|
|
892
|
-
ANTHROPIC_MODEL: selectedModel,
|
|
893
|
-
// Activate AskUserQuestion blocking hook (see hooks/block-ask-user-question.py)
|
|
894
|
-
ZEROSHOT_BLOCK_ASK_USER: '1',
|
|
895
|
-
},
|
|
1171
|
+
env: isolatedEnv,
|
|
896
1172
|
});
|
|
897
1173
|
|
|
898
1174
|
// Track PID for resource monitoring
|
|
@@ -901,6 +1177,20 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
901
1177
|
|
|
902
1178
|
let stdout = '';
|
|
903
1179
|
let stderr = '';
|
|
1180
|
+
let resolved = false;
|
|
1181
|
+
|
|
1182
|
+
// CRITICAL: Timeout to prevent infinite hang if provider CLI hangs
|
|
1183
|
+
const spawnTimeout = setTimeout(() => {
|
|
1184
|
+
if (resolved) return;
|
|
1185
|
+
resolved = true;
|
|
1186
|
+
proc.kill('SIGKILL');
|
|
1187
|
+
reject(
|
|
1188
|
+
new Error(
|
|
1189
|
+
`Spawn timeout after ${SPAWN_TIMEOUT_MS / 1000}s - provider CLI hung. ` +
|
|
1190
|
+
`stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`
|
|
1191
|
+
)
|
|
1192
|
+
);
|
|
1193
|
+
}, SPAWN_TIMEOUT_MS);
|
|
904
1194
|
|
|
905
1195
|
proc.stdout.on('data', (data) => {
|
|
906
1196
|
stdout += data.toString();
|
|
@@ -911,6 +1201,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
911
1201
|
});
|
|
912
1202
|
|
|
913
1203
|
proc.on('close', (code, signal) => {
|
|
1204
|
+
clearTimeout(spawnTimeout);
|
|
1205
|
+
if (resolved) return;
|
|
1206
|
+
resolved = true;
|
|
914
1207
|
// Handle process killed by signal
|
|
915
1208
|
if (signal) {
|
|
916
1209
|
reject(new Error(`Process killed by signal ${signal}${stderr ? `: ${stderr}` : ''}`));
|
|
@@ -944,6 +1237,9 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
944
1237
|
});
|
|
945
1238
|
|
|
946
1239
|
proc.on('error', (error) => {
|
|
1240
|
+
clearTimeout(spawnTimeout);
|
|
1241
|
+
if (resolved) return;
|
|
1242
|
+
resolved = true;
|
|
947
1243
|
reject(error);
|
|
948
1244
|
});
|
|
949
1245
|
});
|
|
@@ -977,6 +1273,196 @@ async function spawnClaudeTaskIsolated(agent, context) {
|
|
|
977
1273
|
* - Status checks reduced to every 2 seconds (not every poll)
|
|
978
1274
|
* - Result: 10-20% overall latency reduction
|
|
979
1275
|
*/
|
|
1276
|
+
function createIsolatedLogState() {
|
|
1277
|
+
return {
|
|
1278
|
+
taskExited: false,
|
|
1279
|
+
fullOutput: '',
|
|
1280
|
+
tailProcess: null,
|
|
1281
|
+
statusCheckInterval: null,
|
|
1282
|
+
lineBuffer: '',
|
|
1283
|
+
};
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
function buildIsolatedCleanup(state) {
|
|
1287
|
+
return () => {
|
|
1288
|
+
if (state.tailProcess) {
|
|
1289
|
+
try {
|
|
1290
|
+
state.tailProcess.kill('SIGTERM');
|
|
1291
|
+
} catch {
|
|
1292
|
+
// Ignore - process may already be dead
|
|
1293
|
+
}
|
|
1294
|
+
state.tailProcess = null;
|
|
1295
|
+
}
|
|
1296
|
+
if (state.statusCheckInterval) {
|
|
1297
|
+
clearInterval(state.statusCheckInterval);
|
|
1298
|
+
state.statusCheckInterval = null;
|
|
1299
|
+
}
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
function broadcastIsolatedLine({ agent, providerName, taskId, line }) {
|
|
1304
|
+
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
1305
|
+
const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
|
|
1306
|
+
const content = timestampMatch ? timestampMatch[2] : line;
|
|
1307
|
+
|
|
1308
|
+
agent.messageBus.publish({
|
|
1309
|
+
cluster_id: agent.cluster.id,
|
|
1310
|
+
topic: 'AGENT_OUTPUT',
|
|
1311
|
+
sender: agent.id,
|
|
1312
|
+
content: {
|
|
1313
|
+
data: {
|
|
1314
|
+
line: content,
|
|
1315
|
+
taskId,
|
|
1316
|
+
iteration: agent.iteration,
|
|
1317
|
+
provider: providerName,
|
|
1318
|
+
},
|
|
1319
|
+
},
|
|
1320
|
+
timestamp,
|
|
1321
|
+
});
|
|
1322
|
+
|
|
1323
|
+
agent.lastOutputTime = Date.now();
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function appendIsolatedContent(state, content, onLine) {
|
|
1327
|
+
state.lineBuffer += content;
|
|
1328
|
+
const lines = state.lineBuffer.split('\n');
|
|
1329
|
+
|
|
1330
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
1331
|
+
if (lines[i].trim()) {
|
|
1332
|
+
onLine(lines[i]);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
state.lineBuffer = lines[lines.length - 1];
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
|
|
1340
|
+
state.tailProcess = manager.spawnInContainer(clusterId, [
|
|
1341
|
+
'sh',
|
|
1342
|
+
'-c',
|
|
1343
|
+
`while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
|
|
1344
|
+
]);
|
|
1345
|
+
|
|
1346
|
+
state.tailProcess.stdout.on('data', (data) => {
|
|
1347
|
+
const chunk = data.toString();
|
|
1348
|
+
state.fullOutput += chunk;
|
|
1349
|
+
appendIsolatedContent(state, chunk, onLine);
|
|
1350
|
+
});
|
|
1351
|
+
|
|
1352
|
+
state.tailProcess.stderr.on('data', (data) => {
|
|
1353
|
+
const msg = data.toString().trim();
|
|
1354
|
+
if (msg && !msg.includes('file truncated')) {
|
|
1355
|
+
agent._log(`[${agent.id}] tail stderr: ${msg}`);
|
|
1356
|
+
}
|
|
1357
|
+
});
|
|
1358
|
+
|
|
1359
|
+
state.tailProcess.on('close', (exitCode) => {
|
|
1360
|
+
if (!state.taskExited) {
|
|
1361
|
+
agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
|
|
1362
|
+
}
|
|
1363
|
+
});
|
|
1364
|
+
|
|
1365
|
+
state.tailProcess.on('error', (err) => {
|
|
1366
|
+
agent._log(`[${agent.id}] tail process error: ${err.message}`);
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
async function checkIsolatedStatus({
|
|
1371
|
+
agent,
|
|
1372
|
+
manager,
|
|
1373
|
+
clusterId,
|
|
1374
|
+
logFilePath,
|
|
1375
|
+
taskId,
|
|
1376
|
+
providerName,
|
|
1377
|
+
state,
|
|
1378
|
+
cleanup,
|
|
1379
|
+
resolve,
|
|
1380
|
+
onLine,
|
|
1381
|
+
}) {
|
|
1382
|
+
if (state.taskExited) return;
|
|
1383
|
+
|
|
1384
|
+
const statusResult = await manager.execInContainer(clusterId, [
|
|
1385
|
+
'sh',
|
|
1386
|
+
'-c',
|
|
1387
|
+
`zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
|
|
1388
|
+
]);
|
|
1389
|
+
|
|
1390
|
+
const statusOutput = statusResult.stdout;
|
|
1391
|
+
const isSuccess = /Status:\s+completed/i.test(statusOutput);
|
|
1392
|
+
const isError = /Status:\s+failed/i.test(statusOutput);
|
|
1393
|
+
const isNotFound = statusOutput.includes('not_found');
|
|
1394
|
+
|
|
1395
|
+
if (!isSuccess && !isError && !isNotFound) {
|
|
1396
|
+
return;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
state.taskExited = true;
|
|
1400
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
1401
|
+
|
|
1402
|
+
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
1403
|
+
'sh',
|
|
1404
|
+
'-c',
|
|
1405
|
+
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
1406
|
+
]);
|
|
1407
|
+
|
|
1408
|
+
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
1409
|
+
state.fullOutput = finalReadResult.stdout;
|
|
1410
|
+
const remainingLines = state.fullOutput.split('\n');
|
|
1411
|
+
for (const line of remainingLines) {
|
|
1412
|
+
if (line.trim()) {
|
|
1413
|
+
onLine(line);
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
cleanup();
|
|
1419
|
+
|
|
1420
|
+
const success = isSuccess && !isError;
|
|
1421
|
+
const errorContext = !success
|
|
1422
|
+
? extractErrorContext({ output: state.fullOutput, taskId, isNotFound })
|
|
1423
|
+
: null;
|
|
1424
|
+
const parsedResult = await agent._parseResultOutput(state.fullOutput);
|
|
1425
|
+
|
|
1426
|
+
resolve({
|
|
1427
|
+
success,
|
|
1428
|
+
output: state.fullOutput,
|
|
1429
|
+
taskId,
|
|
1430
|
+
result: parsedResult,
|
|
1431
|
+
error: errorContext,
|
|
1432
|
+
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
function startIsolatedStatusChecks({
|
|
1437
|
+
agent,
|
|
1438
|
+
manager,
|
|
1439
|
+
clusterId,
|
|
1440
|
+
logFilePath,
|
|
1441
|
+
taskId,
|
|
1442
|
+
providerName,
|
|
1443
|
+
state,
|
|
1444
|
+
cleanup,
|
|
1445
|
+
resolve,
|
|
1446
|
+
onLine,
|
|
1447
|
+
}) {
|
|
1448
|
+
state.statusCheckInterval = setInterval(() => {
|
|
1449
|
+
checkIsolatedStatus({
|
|
1450
|
+
agent,
|
|
1451
|
+
manager,
|
|
1452
|
+
clusterId,
|
|
1453
|
+
logFilePath,
|
|
1454
|
+
taskId,
|
|
1455
|
+
providerName,
|
|
1456
|
+
state,
|
|
1457
|
+
cleanup,
|
|
1458
|
+
resolve,
|
|
1459
|
+
onLine,
|
|
1460
|
+
}).catch((statusErr) => {
|
|
1461
|
+
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
1462
|
+
});
|
|
1463
|
+
}, 2000);
|
|
1464
|
+
}
|
|
1465
|
+
|
|
980
1466
|
function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
981
1467
|
const { isolation } = agent;
|
|
982
1468
|
if (!isolation?.manager) {
|
|
@@ -985,82 +1471,20 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
|
985
1471
|
|
|
986
1472
|
const manager = isolation.manager;
|
|
987
1473
|
const clusterId = isolation.clusterId;
|
|
1474
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
988
1475
|
|
|
989
1476
|
return new Promise((resolve, reject) => {
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
let statusCheckInterval = null;
|
|
994
|
-
let lineBuffer = '';
|
|
995
|
-
|
|
996
|
-
// Cleanup function - kill tail process and clear intervals
|
|
997
|
-
const cleanup = () => {
|
|
998
|
-
if (tailProcess) {
|
|
999
|
-
try {
|
|
1000
|
-
tailProcess.kill('SIGTERM');
|
|
1001
|
-
} catch {
|
|
1002
|
-
// Ignore - process may already be dead
|
|
1003
|
-
}
|
|
1004
|
-
tailProcess = null;
|
|
1005
|
-
}
|
|
1006
|
-
if (statusCheckInterval) {
|
|
1007
|
-
clearInterval(statusCheckInterval);
|
|
1008
|
-
statusCheckInterval = null;
|
|
1009
|
-
}
|
|
1010
|
-
};
|
|
1011
|
-
|
|
1012
|
-
// Broadcast line helper (same as non-isolated mode)
|
|
1013
|
-
const broadcastLine = (line) => {
|
|
1014
|
-
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
1015
|
-
const timestamp = timestampMatch
|
|
1016
|
-
? new Date(timestampMatch[1]).getTime()
|
|
1017
|
-
: Date.now();
|
|
1018
|
-
const content = timestampMatch ? timestampMatch[2] : line;
|
|
1019
|
-
|
|
1020
|
-
agent.messageBus.publish({
|
|
1021
|
-
cluster_id: agent.cluster.id,
|
|
1022
|
-
topic: 'AGENT_OUTPUT',
|
|
1023
|
-
sender: agent.id,
|
|
1024
|
-
content: {
|
|
1025
|
-
data: {
|
|
1026
|
-
line: content,
|
|
1027
|
-
taskId,
|
|
1028
|
-
iteration: agent.iteration,
|
|
1029
|
-
},
|
|
1030
|
-
},
|
|
1031
|
-
timestamp,
|
|
1032
|
-
});
|
|
1477
|
+
const state = createIsolatedLogState();
|
|
1478
|
+
const cleanup = buildIsolatedCleanup(state);
|
|
1479
|
+
const onLine = (line) => broadcastIsolatedLine({ agent, providerName, taskId, line });
|
|
1033
1480
|
|
|
1034
|
-
// Update last output time for liveness tracking
|
|
1035
|
-
agent.lastOutputTime = Date.now();
|
|
1036
|
-
};
|
|
1037
|
-
|
|
1038
|
-
// Process new content by splitting into complete lines
|
|
1039
|
-
const processNewContent = (content) => {
|
|
1040
|
-
lineBuffer += content;
|
|
1041
|
-
const lines = lineBuffer.split('\n');
|
|
1042
|
-
|
|
1043
|
-
// Process all complete lines (all except last, which might be incomplete)
|
|
1044
|
-
for (let i = 0; i < lines.length - 1; i++) {
|
|
1045
|
-
if (lines[i].trim()) {
|
|
1046
|
-
broadcastLine(lines[i]);
|
|
1047
|
-
}
|
|
1048
|
-
}
|
|
1049
|
-
|
|
1050
|
-
// Keep last line in buffer (might be incomplete)
|
|
1051
|
-
lineBuffer = lines[lines.length - 1];
|
|
1052
|
-
};
|
|
1053
|
-
|
|
1054
|
-
// Get log file path from zeroshot CLI inside container
|
|
1055
1481
|
manager
|
|
1056
1482
|
.execInContainer(clusterId, ['sh', '-c', `zeroshot get-log-path ${taskId}`])
|
|
1057
1483
|
.then(({ stdout, stderr, code }) => {
|
|
1058
1484
|
if (code !== 0) {
|
|
1059
1485
|
cleanup();
|
|
1060
1486
|
return reject(
|
|
1061
|
-
new Error(
|
|
1062
|
-
`Failed to get log path for ${taskId} inside container: ${stderr || stdout}`
|
|
1063
|
-
)
|
|
1487
|
+
new Error(`Failed to get log path for ${taskId} inside container: ${stderr || stdout}`)
|
|
1064
1488
|
);
|
|
1065
1489
|
}
|
|
1066
1490
|
|
|
@@ -1072,123 +1496,33 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
|
1072
1496
|
|
|
1073
1497
|
agent._log(`[${agent.id}] Following isolated task logs (streaming): ${logFilePath}`);
|
|
1074
1498
|
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
// The -F flag handles file recreation (rotation)
|
|
1083
|
-
`while [ ! -f "${logFilePath}" ]; do sleep 0.1; done; tail -F -n +1 "${logFilePath}"`,
|
|
1084
|
-
]);
|
|
1085
|
-
|
|
1086
|
-
// Stream stdout directly - lines arrive as they're written
|
|
1087
|
-
tailProcess.stdout.on('data', (data) => {
|
|
1088
|
-
const chunk = data.toString();
|
|
1089
|
-
fullOutput += chunk;
|
|
1090
|
-
processNewContent(chunk);
|
|
1499
|
+
startIsolatedTail({
|
|
1500
|
+
agent,
|
|
1501
|
+
manager,
|
|
1502
|
+
clusterId,
|
|
1503
|
+
logFilePath,
|
|
1504
|
+
state,
|
|
1505
|
+
onLine,
|
|
1091
1506
|
});
|
|
1092
1507
|
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
agent._log(`[${agent.id}] tail process exited with code ${exitCode}`);
|
|
1105
|
-
}
|
|
1508
|
+
startIsolatedStatusChecks({
|
|
1509
|
+
agent,
|
|
1510
|
+
manager,
|
|
1511
|
+
clusterId,
|
|
1512
|
+
logFilePath,
|
|
1513
|
+
taskId,
|
|
1514
|
+
providerName,
|
|
1515
|
+
state,
|
|
1516
|
+
cleanup,
|
|
1517
|
+
resolve,
|
|
1518
|
+
onLine,
|
|
1106
1519
|
});
|
|
1107
1520
|
|
|
1108
|
-
tailProcess.on('error', (err) => {
|
|
1109
|
-
agent._log(`[${agent.id}] tail process error: ${err.message}`);
|
|
1110
|
-
});
|
|
1111
|
-
|
|
1112
|
-
// Check task status periodically (every 2 seconds - much less frequent than polling)
|
|
1113
|
-
// This is the only remaining docker exec - but now at 2s intervals instead of 500ms
|
|
1114
|
-
statusCheckInterval = setInterval(async () => {
|
|
1115
|
-
if (taskExited) return;
|
|
1116
|
-
|
|
1117
|
-
try {
|
|
1118
|
-
const statusResult = await manager.execInContainer(clusterId, [
|
|
1119
|
-
'sh',
|
|
1120
|
-
'-c',
|
|
1121
|
-
`zeroshot status ${taskId} 2>/dev/null || echo "not_found"`,
|
|
1122
|
-
]);
|
|
1123
|
-
|
|
1124
|
-
const statusOutput = statusResult.stdout;
|
|
1125
|
-
const isSuccess = /Status:\s+completed/i.test(statusOutput);
|
|
1126
|
-
const isError = /Status:\s+failed/i.test(statusOutput);
|
|
1127
|
-
const isNotFound = statusOutput.includes('not_found');
|
|
1128
|
-
|
|
1129
|
-
if (isSuccess || isError || isNotFound) {
|
|
1130
|
-
taskExited = true;
|
|
1131
|
-
|
|
1132
|
-
// Give tail a moment to flush remaining output
|
|
1133
|
-
await new Promise((r) => setTimeout(r, 200));
|
|
1134
|
-
|
|
1135
|
-
// Read final output to ensure we have everything
|
|
1136
|
-
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
1137
|
-
'sh',
|
|
1138
|
-
'-c',
|
|
1139
|
-
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
1140
|
-
]);
|
|
1141
|
-
|
|
1142
|
-
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
1143
|
-
fullOutput = finalReadResult.stdout;
|
|
1144
|
-
|
|
1145
|
-
// Process any remaining content
|
|
1146
|
-
const remainingLines = fullOutput.split('\n');
|
|
1147
|
-
for (const line of remainingLines) {
|
|
1148
|
-
if (line.trim()) {
|
|
1149
|
-
broadcastLine(line);
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
cleanup();
|
|
1155
|
-
|
|
1156
|
-
// Determine success status
|
|
1157
|
-
const success = isSuccess && !isError;
|
|
1158
|
-
|
|
1159
|
-
// Extract error context using shared helper
|
|
1160
|
-
const errorContext = !success
|
|
1161
|
-
? extractErrorContext({ output: fullOutput, taskId, isNotFound })
|
|
1162
|
-
: null;
|
|
1163
|
-
|
|
1164
|
-
// Parse result from output
|
|
1165
|
-
const parsedResult = agent._parseResultOutput(fullOutput);
|
|
1166
|
-
|
|
1167
|
-
resolve({
|
|
1168
|
-
success,
|
|
1169
|
-
output: fullOutput,
|
|
1170
|
-
taskId,
|
|
1171
|
-
result: parsedResult,
|
|
1172
|
-
error: errorContext,
|
|
1173
|
-
tokenUsage: extractTokenUsage(fullOutput),
|
|
1174
|
-
});
|
|
1175
|
-
}
|
|
1176
|
-
} catch (statusErr) {
|
|
1177
|
-
// Log error but continue checking (transient failures are common)
|
|
1178
|
-
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
1179
|
-
}
|
|
1180
|
-
}, 2000); // Check every 2 seconds (was 500ms in polling mode)
|
|
1181
|
-
|
|
1182
|
-
// Safety timeout (0 = no timeout, task runs until completion)
|
|
1183
1521
|
if (agent.timeout > 0) {
|
|
1184
1522
|
setTimeout(() => {
|
|
1185
|
-
if (!taskExited) {
|
|
1523
|
+
if (!state.taskExited) {
|
|
1186
1524
|
cleanup();
|
|
1187
|
-
reject(
|
|
1188
|
-
new Error(
|
|
1189
|
-
`Task ${taskId} timeout after ${agent.timeout}ms (isolated mode)`
|
|
1190
|
-
)
|
|
1191
|
-
);
|
|
1525
|
+
reject(new Error(`Task ${taskId} timeout after ${agent.timeout}ms (isolated mode)`));
|
|
1192
1526
|
}
|
|
1193
1527
|
}, agent.timeout);
|
|
1194
1528
|
}
|
|
@@ -1204,133 +1538,57 @@ function followClaudeTaskLogsIsolated(agent, taskId) {
|
|
|
1204
1538
|
* Parse agent output to extract structured result data
|
|
1205
1539
|
* GENERIC - returns whatever structured output the agent provides
|
|
1206
1540
|
* Works with any agent schema (planner, validator, worker, etc.)
|
|
1541
|
+
*
|
|
1542
|
+
* Uses clean extraction pipeline from output-extraction.js
|
|
1543
|
+
* Falls back to reformatting if extraction fails and schema is available
|
|
1544
|
+
*
|
|
1207
1545
|
* @param {Object} agent - Agent instance
|
|
1208
1546
|
* @param {String} output - Raw output from agent
|
|
1209
|
-
* @returns {Object} Parsed result data
|
|
1547
|
+
* @returns {Promise<Object>} Parsed result data
|
|
1210
1548
|
*/
|
|
1211
|
-
function parseResultOutput(agent, output) {
|
|
1549
|
+
async function parseResultOutput(agent, output) {
|
|
1212
1550
|
// Empty or error outputs = FAIL
|
|
1213
1551
|
if (!output || output.includes('Task not found') || output.includes('Process terminated')) {
|
|
1214
1552
|
throw new Error('Task execution failed - no output');
|
|
1215
1553
|
}
|
|
1216
1554
|
|
|
1217
|
-
|
|
1218
|
-
|
|
1555
|
+
const providerName = agent._resolveProvider ? agent._resolveProvider() : 'claude';
|
|
1556
|
+
const { extractJsonFromOutput } = require('./output-extraction');
|
|
1219
1557
|
|
|
1220
|
-
//
|
|
1221
|
-
|
|
1222
|
-
// Find the line with "type":"result" which contains the actual result
|
|
1223
|
-
const lines = trimmedOutput.split('\n');
|
|
1224
|
-
const resultLine = lines.find((line) => {
|
|
1225
|
-
try {
|
|
1226
|
-
const content = stripTimestampPrefix(line);
|
|
1227
|
-
if (!content.startsWith('{')) return false;
|
|
1228
|
-
const obj = JSON.parse(content);
|
|
1229
|
-
return obj.type === 'result';
|
|
1230
|
-
} catch {
|
|
1231
|
-
return false;
|
|
1232
|
-
}
|
|
1233
|
-
});
|
|
1558
|
+
// Use clean extraction pipeline
|
|
1559
|
+
let parsed = extractJsonFromOutput(output, providerName);
|
|
1234
1560
|
|
|
1235
|
-
//
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
trimmedOutput = stripTimestampPrefix(resultLine);
|
|
1239
|
-
} else if (lines.length > 1) {
|
|
1240
|
-
// Fallback: use last non-empty line (also strip timestamp)
|
|
1241
|
-
for (let i = lines.length - 1; i >= 0; i--) {
|
|
1242
|
-
const content = stripTimestampPrefix(lines[i]);
|
|
1243
|
-
if (content) {
|
|
1244
|
-
trimmedOutput = content;
|
|
1245
|
-
break;
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1561
|
+
// If extraction failed but we have a schema, attempt reformatting
|
|
1562
|
+
if (!parsed && agent.config.jsonSchema) {
|
|
1563
|
+
const { reformatOutput } = require('./output-reformatter');
|
|
1249
1564
|
|
|
1250
|
-
// Strategy 1: If agent uses JSON output format, try CLI JSON structure first
|
|
1251
|
-
if (agent.config.outputFormat === 'json' && agent.config.jsonSchema) {
|
|
1252
1565
|
try {
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
) {
|
|
1265
|
-
// Check for result wrapper
|
|
1266
|
-
if (claudeOutput.result && typeof claudeOutput.result === 'object') {
|
|
1267
|
-
parsed = claudeOutput.result;
|
|
1268
|
-
}
|
|
1269
|
-
// IMPORTANT: Handle case where result is a string containing markdown-wrapped JSON
|
|
1270
|
-
// Claude CLI with --output-format json returns { result: "```json\n{...}\n```" }
|
|
1271
|
-
else if (claudeOutput.result && typeof claudeOutput.result === 'string') {
|
|
1272
|
-
const resultStr = claudeOutput.result;
|
|
1273
|
-
// Try extracting JSON from markdown code block
|
|
1274
|
-
const jsonMatch = resultStr.match(/```json\s*([\s\S]*?)```/);
|
|
1275
|
-
if (jsonMatch) {
|
|
1276
|
-
try {
|
|
1277
|
-
parsed = JSON.parse(jsonMatch[1].trim());
|
|
1278
|
-
} catch {
|
|
1279
|
-
// Fall through to other strategies
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
// If no markdown block, try parsing result string directly as JSON
|
|
1283
|
-
if (!parsed) {
|
|
1284
|
-
try {
|
|
1285
|
-
parsed = JSON.parse(resultStr);
|
|
1286
|
-
} catch {
|
|
1287
|
-
// Fall through to other strategies
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
}
|
|
1291
|
-
// Use directly if it has meaningful keys (and we haven't found a better parse)
|
|
1292
|
-
if (!parsed) {
|
|
1293
|
-
const keys = Object.keys(claudeOutput);
|
|
1294
|
-
if (keys.length > 0 && keys.some((k) => !['type', 'subtype', 'is_error'].includes(k))) {
|
|
1295
|
-
parsed = claudeOutput;
|
|
1566
|
+
parsed = await reformatOutput({
|
|
1567
|
+
rawOutput: output,
|
|
1568
|
+
schema: agent.config.jsonSchema,
|
|
1569
|
+
providerName,
|
|
1570
|
+
onAttempt: (attempt, lastError) => {
|
|
1571
|
+
if (lastError) {
|
|
1572
|
+
console.warn(`[Agent ${agent.id}] Reformat attempt ${attempt}: ${lastError}`);
|
|
1573
|
+
} else {
|
|
1574
|
+
console.warn(
|
|
1575
|
+
`[Agent ${agent.id}] JSON extraction failed, reformatting (attempt ${attempt})...`
|
|
1576
|
+
);
|
|
1296
1577
|
}
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
} catch {
|
|
1300
|
-
//
|
|
1301
|
-
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
// Strategy 2: Extract JSON from markdown code block (legacy or fallback)
|
|
1305
|
-
if (!parsed) {
|
|
1306
|
-
const jsonMatch = trimmedOutput.match(/```json\s*([\s\S]*?)```/);
|
|
1307
|
-
if (jsonMatch) {
|
|
1308
|
-
try {
|
|
1309
|
-
parsed = JSON.parse(jsonMatch[1].trim());
|
|
1310
|
-
} catch (e) {
|
|
1311
|
-
throw new Error(`JSON parse failed in markdown block: ${e.message}`);
|
|
1312
|
-
}
|
|
1313
|
-
}
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
// Strategy 3: Try parsing the whole output as JSON
|
|
1317
|
-
if (!parsed) {
|
|
1318
|
-
try {
|
|
1319
|
-
const directParse = JSON.parse(trimmedOutput);
|
|
1320
|
-
if (typeof directParse === 'object' && directParse !== null) {
|
|
1321
|
-
parsed = directParse;
|
|
1322
|
-
}
|
|
1323
|
-
} catch {
|
|
1324
|
-
// Not valid JSON, fall through to error
|
|
1578
|
+
},
|
|
1579
|
+
});
|
|
1580
|
+
} catch (reformatError) {
|
|
1581
|
+
// Reformatting failed - fall through to error below
|
|
1582
|
+
console.error(`[Agent ${agent.id}] Reformatting failed: ${reformatError.message}`);
|
|
1325
1583
|
}
|
|
1326
1584
|
}
|
|
1327
1585
|
|
|
1328
|
-
// No strategy worked
|
|
1329
1586
|
if (!parsed) {
|
|
1587
|
+
const trimmedOutput = output.trim();
|
|
1330
1588
|
console.error(`\n${'='.repeat(80)}`);
|
|
1331
1589
|
console.error(`🔴 AGENT OUTPUT MISSING REQUIRED JSON BLOCK`);
|
|
1332
1590
|
console.error(`${'='.repeat(80)}`);
|
|
1333
|
-
console.error(`Agent: ${agent.id}, Role: ${agent.role}`);
|
|
1591
|
+
console.error(`Agent: ${agent.id}, Role: ${agent.role}, Provider: ${providerName}`);
|
|
1334
1592
|
console.error(`Output (last 500 chars): ${trimmedOutput.slice(-500)}`);
|
|
1335
1593
|
console.error(`${'='.repeat(80)}\n`);
|
|
1336
1594
|
throw new Error(`Agent ${agent.id} output missing required JSON block`);
|
|
@@ -1340,6 +1598,10 @@ function parseResultOutput(agent, output) {
|
|
|
1340
1598
|
// This preserves schema enforcement even when we run stream-json for live logs.
|
|
1341
1599
|
// IMPORTANT: For non-validator agents we warn but do not fail the cluster.
|
|
1342
1600
|
if (agent.config.jsonSchema) {
|
|
1601
|
+
// Normalize enum values BEFORE validation (handles case mismatches, common variations)
|
|
1602
|
+
// This is provider-agnostic - works for Claude CLI, Gemini, Codex, etc.
|
|
1603
|
+
normalizeEnumValues(parsed, agent.config.jsonSchema);
|
|
1604
|
+
|
|
1343
1605
|
const Ajv = require('ajv');
|
|
1344
1606
|
const ajv = new Ajv({
|
|
1345
1607
|
allErrors: true,
|
|
@@ -1403,9 +1665,8 @@ function killTask(agent) {
|
|
|
1403
1665
|
// Also kill the underlying zeroshot task if we have a task ID
|
|
1404
1666
|
// This ensures the task process is stopped, not just our polling intervals
|
|
1405
1667
|
if (agent.currentTaskId) {
|
|
1406
|
-
const { exec } = require('child_process');
|
|
1407
1668
|
const ctPath = getClaudeTasksPath();
|
|
1408
|
-
exec(`${ctPath} task kill ${agent.currentTaskId}`, (error) => {
|
|
1669
|
+
exec(`${ctPath} task kill ${agent.currentTaskId}`, { timeout: 10000 }, (error) => {
|
|
1409
1670
|
if (error) {
|
|
1410
1671
|
// Task may have already completed or been killed, ignore errors
|
|
1411
1672
|
agent._log(`Note: Could not kill task ${agent.currentTaskId}: ${error.message}`);
|