@the-open-engine/zeroshot 6.34.1 → 6.34.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +34 -8
- package/cli/json-export.js +77 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/src/agent/agent-lifecycle.js +24 -19
- package/src/agent/agent-task-executor.js +539 -157
- package/src/agent/output-extraction.js +111 -39
- package/src/agent/provider-terminal-failure.js +186 -0
- package/src/ledger.js +364 -31
- package/src/orchestrator.js +33 -1
- package/src/template-resolver.js +5 -0
- package/task-lib/watcher-output-runtime.js +128 -43
|
@@ -54,13 +54,24 @@ const {
|
|
|
54
54
|
validateCompletedResumeIdentity,
|
|
55
55
|
} = require('./provider-session');
|
|
56
56
|
const { extractClaudeVertexModelError } = require('./output-extraction');
|
|
57
|
+
const {
|
|
58
|
+
extractProviderFailure,
|
|
59
|
+
redactTerminalFailureForControlPlane,
|
|
60
|
+
} = require('./provider-terminal-failure');
|
|
57
61
|
const {
|
|
58
62
|
createStructuredOutputInvalidError,
|
|
59
63
|
isStructuredOutputInvalidError,
|
|
60
64
|
} = require('./structured-output-error');
|
|
61
65
|
const TASK_TERMINAL_STATUSES = new Set(['completed', 'failed', 'killed', 'stale']);
|
|
62
66
|
const MAX_CONTROL_PLANE_RECORD_BYTES = 1024 * 1024;
|
|
67
|
+
const MAX_CONTROL_PLANE_OUTPUT_BYTES = 2 * 1024 * 1024;
|
|
68
|
+
const MAX_CONTROL_PLANE_OUTPUT_RECORDS = 1024;
|
|
69
|
+
const MAX_LIVE_OUTPUT_BYTES = 512 * 1024;
|
|
70
|
+
const MAX_LIVE_OUTPUT_RECORDS = 512;
|
|
71
|
+
const CONTROL_PLANE_OMISSION_MARKER_BYTES = 1024;
|
|
63
72
|
const LOG_READ_CHUNK_BYTES = 64 * 1024;
|
|
73
|
+
const MAX_FAILURE_ERROR_BYTES = 4096;
|
|
74
|
+
const FAILURE_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
|
|
64
75
|
function runCommandWithTimeout(command, args, options = {}, callback = null) {
|
|
65
76
|
const timeout = options.timeout ?? 30000;
|
|
66
77
|
if (timeout <= 0) {
|
|
@@ -170,6 +181,22 @@ function buildClaudeEnv(modelSpec, options = {}) {
|
|
|
170
181
|
function sanitizeErrorMessage(error) {
|
|
171
182
|
if (!error) return null;
|
|
172
183
|
|
|
184
|
+
const original = String(error);
|
|
185
|
+
const suffixBytes = Buffer.byteLength(FAILURE_ERROR_TRUNCATION_SUFFIX);
|
|
186
|
+
const contentBudget = MAX_FAILURE_ERROR_BYTES - suffixBytes;
|
|
187
|
+
let boundedError = original;
|
|
188
|
+
if (Buffer.byteLength(original) > MAX_FAILURE_ERROR_BYTES) {
|
|
189
|
+
let bytes = 0;
|
|
190
|
+
let prefix = '';
|
|
191
|
+
for (const character of original) {
|
|
192
|
+
const characterBytes = Buffer.byteLength(character);
|
|
193
|
+
if (bytes + characterBytes > contentBudget) break;
|
|
194
|
+
prefix += character;
|
|
195
|
+
bytes += characterBytes;
|
|
196
|
+
}
|
|
197
|
+
boundedError = `${prefix}${FAILURE_ERROR_TRUNCATION_SUFFIX}`;
|
|
198
|
+
}
|
|
199
|
+
|
|
173
200
|
// Patterns that look like TypeScript type annotations (not real error messages)
|
|
174
201
|
const typeAnnotationPatterns = [
|
|
175
202
|
/^string\s*\|\s*null$/i,
|
|
@@ -182,7 +209,7 @@ function sanitizeErrorMessage(error) {
|
|
|
182
209
|
/^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
|
|
183
210
|
];
|
|
184
211
|
|
|
185
|
-
const trimmedError =
|
|
212
|
+
const trimmedError = boundedError.trim();
|
|
186
213
|
|
|
187
214
|
// Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
|
|
188
215
|
const unionParts = trimmedError.split(/\s*\|\s*/);
|
|
@@ -191,14 +218,16 @@ function sanitizeErrorMessage(error) {
|
|
|
191
218
|
for (const pattern of typeAnnotationPatterns) {
|
|
192
219
|
if (pattern.test(trimmedError) || isUnionType) {
|
|
193
220
|
console.warn(
|
|
194
|
-
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${
|
|
221
|
+
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${boundedError}". ` +
|
|
195
222
|
`This indicates corrupted data. Replacing with generic error.`
|
|
196
223
|
);
|
|
197
|
-
return
|
|
224
|
+
return sanitizeErrorMessage(
|
|
225
|
+
`Task failed with corrupted error data (original: "${boundedError}")`
|
|
226
|
+
);
|
|
198
227
|
}
|
|
199
228
|
}
|
|
200
229
|
|
|
201
|
-
return
|
|
230
|
+
return boundedError;
|
|
202
231
|
}
|
|
203
232
|
|
|
204
233
|
function safeTail(text, maxChars) {
|
|
@@ -290,6 +319,30 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
|
|
|
290
319
|
console.error('[AgentTaskExecutor] Claude CLI returned no messages', payload);
|
|
291
320
|
}
|
|
292
321
|
|
|
322
|
+
function extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug }) {
|
|
323
|
+
if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
|
|
324
|
+
return sanitizeErrorMessage(
|
|
325
|
+
`FILE TOO LARGE (Claude Code 256KB limit). ` +
|
|
326
|
+
`Use offset and limit parameters when reading large files. ` +
|
|
327
|
+
`Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
|
|
331
|
+
return sanitizeErrorMessage(
|
|
332
|
+
`STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
|
|
333
|
+
`This usually happens with AskUserQuestion or interactive prompts. ` +
|
|
334
|
+
`Zeroshot agents must run non-interactively.`
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
if (fullOutput.includes('No messages returned')) {
|
|
338
|
+
logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
|
|
339
|
+
return sanitizeErrorMessage(
|
|
340
|
+
`Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
return null;
|
|
344
|
+
}
|
|
345
|
+
|
|
293
346
|
/**
|
|
294
347
|
* Extract error context from task output.
|
|
295
348
|
* Shared by both isolated and non-isolated modes.
|
|
@@ -302,7 +355,15 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
|
|
|
302
355
|
* @param {Object} [params.debug] - Additional debug context for logging
|
|
303
356
|
* @returns {string|null} Sanitized error context or null if extraction failed
|
|
304
357
|
*/
|
|
305
|
-
function extractErrorContext({
|
|
358
|
+
function extractErrorContext({
|
|
359
|
+
output,
|
|
360
|
+
statusOutput,
|
|
361
|
+
taskId,
|
|
362
|
+
isNotFound = false,
|
|
363
|
+
providerName = getDefaultProviderId(),
|
|
364
|
+
providerFailure = null,
|
|
365
|
+
debug,
|
|
366
|
+
}) {
|
|
306
367
|
// Task not found - explicit error
|
|
307
368
|
if (isNotFound) {
|
|
308
369
|
return sanitizeErrorMessage(`Task ${taskId} not found (may have crashed or been killed)`);
|
|
@@ -316,37 +377,15 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false,
|
|
|
316
377
|
}
|
|
317
378
|
}
|
|
318
379
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
// 256KB file limit error
|
|
323
|
-
if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
|
|
324
|
-
return sanitizeErrorMessage(
|
|
325
|
-
`FILE TOO LARGE (Claude Code 256KB limit). ` +
|
|
326
|
-
`Use offset and limit parameters when reading large files. ` +
|
|
327
|
-
`Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
|
|
328
|
-
);
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
// Streaming mode error (interactive tools in non-interactive mode)
|
|
332
|
-
if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
|
|
333
|
-
return sanitizeErrorMessage(
|
|
334
|
-
`STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
|
|
335
|
-
`This usually happens with AskUserQuestion or interactive prompts. ` +
|
|
336
|
-
`Zeroshot agents must run non-interactively.`
|
|
337
|
-
);
|
|
380
|
+
const terminalFailure = providerFailure || extractProviderFailure(output, providerName);
|
|
381
|
+
if (terminalFailure) {
|
|
382
|
+
return sanitizeErrorMessage(terminalFailure.error);
|
|
338
383
|
}
|
|
339
384
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
return sanitizeErrorMessage(
|
|
344
|
-
`Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
|
|
345
|
-
);
|
|
346
|
-
}
|
|
385
|
+
const fullOutput = output || '';
|
|
386
|
+
const knownFailure = extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug });
|
|
387
|
+
if (knownFailure) return knownFailure;
|
|
347
388
|
|
|
348
|
-
// NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
|
|
349
|
-
// If output is too verbose, that's a prompt problem - fix the prompts, not the data
|
|
350
389
|
const trimmedOutput = (output || '').trim();
|
|
351
390
|
if (!trimmedOutput) {
|
|
352
391
|
return sanitizeErrorMessage(
|
|
@@ -1222,8 +1261,8 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
|
1222
1261
|
const MAX_STATUS_FAILURES = 30;
|
|
1223
1262
|
|
|
1224
1263
|
function createLogFollowState() {
|
|
1225
|
-
|
|
1226
|
-
|
|
1264
|
+
const state = {
|
|
1265
|
+
controlPlaneOutput: createControlPlaneOutputState(),
|
|
1227
1266
|
logFilePath: null,
|
|
1228
1267
|
lastSize: 0,
|
|
1229
1268
|
pollInterval: null,
|
|
@@ -1233,6 +1272,175 @@ function createLogFollowState() {
|
|
|
1233
1272
|
logDecoder: new StringDecoder('utf8'),
|
|
1234
1273
|
consecutiveExecFailures: 0,
|
|
1235
1274
|
};
|
|
1275
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
1276
|
+
return state;
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
function createControlPlaneOutputState(options = {}) {
|
|
1280
|
+
const maxBytes = options.maxBytes || MAX_CONTROL_PLANE_OUTPUT_BYTES;
|
|
1281
|
+
const maxRecords = options.maxRecords || MAX_CONTROL_PLANE_OUTPUT_RECORDS;
|
|
1282
|
+
const liveByteLimit = options.liveByteLimit || MAX_LIVE_OUTPUT_BYTES;
|
|
1283
|
+
const liveRecordLimit = options.liveRecordLimit || MAX_LIVE_OUTPUT_RECORDS;
|
|
1284
|
+
if (maxBytes <= CONTROL_PLANE_OMISSION_MARKER_BYTES || maxRecords < 1) {
|
|
1285
|
+
throw new Error('Control-plane output bounds must retain space for at least one record');
|
|
1286
|
+
}
|
|
1287
|
+
return {
|
|
1288
|
+
records: [],
|
|
1289
|
+
head: 0,
|
|
1290
|
+
byteLength: 0,
|
|
1291
|
+
maxBytes,
|
|
1292
|
+
maxRecords,
|
|
1293
|
+
liveByteLimit,
|
|
1294
|
+
liveRecordLimit,
|
|
1295
|
+
liveBytes: 0,
|
|
1296
|
+
liveRecords: 0,
|
|
1297
|
+
nextSequence: 0,
|
|
1298
|
+
lastLiveSequence: -1,
|
|
1299
|
+
liveSuppressed: false,
|
|
1300
|
+
terminalFlushed: false,
|
|
1301
|
+
omittedBytes: 0,
|
|
1302
|
+
omittedRecords: 0,
|
|
1303
|
+
omittedDigest: createHash('sha256'),
|
|
1304
|
+
prefixOmitted: false,
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
function defineControlPlaneOutputAccessor(state, property) {
|
|
1309
|
+
Object.defineProperty(state, property, {
|
|
1310
|
+
configurable: true,
|
|
1311
|
+
enumerable: true,
|
|
1312
|
+
get: () => snapshotControlPlaneOutput(state.controlPlaneOutput),
|
|
1313
|
+
});
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
function ensureControlPlaneOutputState(state = {}) {
|
|
1317
|
+
if (state.controlPlaneOutput) return state;
|
|
1318
|
+
let existingOutput = '';
|
|
1319
|
+
if (typeof state.output === 'string') {
|
|
1320
|
+
existingOutput = state.output;
|
|
1321
|
+
} else if (typeof state.fullOutput === 'string') {
|
|
1322
|
+
existingOutput = state.fullOutput;
|
|
1323
|
+
}
|
|
1324
|
+
state.controlPlaneOutput = createControlPlaneOutputState();
|
|
1325
|
+
if (Object.hasOwn(state, 'output')) delete state.output;
|
|
1326
|
+
if (Object.hasOwn(state, 'fullOutput')) delete state.fullOutput;
|
|
1327
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
1328
|
+
if (existingOutput) {
|
|
1329
|
+
replayCompleteLogContent(existingOutput, (content) =>
|
|
1330
|
+
appendControlPlaneRecord(state.controlPlaneOutput, {
|
|
1331
|
+
content,
|
|
1332
|
+
timestamp: Date.now(),
|
|
1333
|
+
type: isValidJsonLine(content) ? 'json' : 'text',
|
|
1334
|
+
})
|
|
1335
|
+
);
|
|
1336
|
+
}
|
|
1337
|
+
return state;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
function retainedControlPlaneRecords(outputState) {
|
|
1341
|
+
return outputState.records.slice(outputState.head);
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
function compactControlPlaneRecords(outputState) {
|
|
1345
|
+
if (outputState.head >= 64 && outputState.head * 2 >= outputState.records.length) {
|
|
1346
|
+
outputState.records = outputState.records.slice(outputState.head);
|
|
1347
|
+
outputState.head = 0;
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
function omitOldestControlPlaneRecord(outputState) {
|
|
1352
|
+
const record = outputState.records[outputState.head++];
|
|
1353
|
+
outputState.byteLength -= record.byteLength;
|
|
1354
|
+
outputState.omittedBytes += record.byteLength;
|
|
1355
|
+
outputState.omittedRecords++;
|
|
1356
|
+
outputState.omittedDigest.update(record.text);
|
|
1357
|
+
compactControlPlaneRecords(outputState);
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
function appendControlPlaneRecord(outputState, { content, timestamp, type }) {
|
|
1361
|
+
const text = `${content}\n`;
|
|
1362
|
+
const record = {
|
|
1363
|
+
sequence: outputState.nextSequence++,
|
|
1364
|
+
content,
|
|
1365
|
+
timestamp,
|
|
1366
|
+
type,
|
|
1367
|
+
text,
|
|
1368
|
+
byteLength: Buffer.byteLength(text),
|
|
1369
|
+
};
|
|
1370
|
+
outputState.records.push(record);
|
|
1371
|
+
outputState.byteLength += record.byteLength;
|
|
1372
|
+
|
|
1373
|
+
const payloadLimit = outputState.maxBytes - CONTROL_PLANE_OMISSION_MARKER_BYTES;
|
|
1374
|
+
while (
|
|
1375
|
+
outputState.byteLength > payloadLimit ||
|
|
1376
|
+
outputState.records.length - outputState.head > outputState.maxRecords
|
|
1377
|
+
) {
|
|
1378
|
+
omitOldestControlPlaneRecord(outputState);
|
|
1379
|
+
}
|
|
1380
|
+
return record;
|
|
1381
|
+
}
|
|
1382
|
+
|
|
1383
|
+
function controlPlaneOmissionRecord(outputState) {
|
|
1384
|
+
if (!outputState.prefixOmitted && outputState.omittedRecords === 0) return null;
|
|
1385
|
+
const digest = outputState.omittedRecords
|
|
1386
|
+
? `, sha256=${outputState.omittedDigest.copy().digest('hex')}`
|
|
1387
|
+
: '';
|
|
1388
|
+
const known = outputState.omittedRecords
|
|
1389
|
+
? `records=${outputState.omittedRecords}, byte_length=${outputState.omittedBytes}${digest}`
|
|
1390
|
+
: 'byte_length=unknown';
|
|
1391
|
+
const content =
|
|
1392
|
+
`[ZEROSHOT] Earlier provider output omitted from the bounded control-plane tail (${known}). ` +
|
|
1393
|
+
'Complete output remains in the task log.';
|
|
1394
|
+
return {
|
|
1395
|
+
content,
|
|
1396
|
+
timestamp: Date.now(),
|
|
1397
|
+
type: 'text',
|
|
1398
|
+
text: `${content}\n`,
|
|
1399
|
+
byteLength: Buffer.byteLength(`${content}\n`),
|
|
1400
|
+
};
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
function snapshotControlPlaneOutput(outputState) {
|
|
1404
|
+
const omission = controlPlaneOmissionRecord(outputState);
|
|
1405
|
+
const records = retainedControlPlaneRecords(outputState);
|
|
1406
|
+
return `${omission ? omission.text : ''}${records.map((record) => record.text).join('')}`;
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
function resetControlPlaneTail(outputState, { prefixOmitted = false } = {}) {
|
|
1410
|
+
outputState.records = [];
|
|
1411
|
+
outputState.head = 0;
|
|
1412
|
+
outputState.byteLength = 0;
|
|
1413
|
+
outputState.omittedBytes = 0;
|
|
1414
|
+
outputState.omittedRecords = 0;
|
|
1415
|
+
outputState.omittedDigest = createHash('sha256');
|
|
1416
|
+
outputState.prefixOmitted = prefixOmitted;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1419
|
+
function publishLiveControlPlaneRecord(outputState, record, publish) {
|
|
1420
|
+
if (
|
|
1421
|
+
outputState.liveSuppressed ||
|
|
1422
|
+
outputState.liveBytes + record.byteLength > outputState.liveByteLimit ||
|
|
1423
|
+
outputState.liveRecords + 1 > outputState.liveRecordLimit
|
|
1424
|
+
) {
|
|
1425
|
+
outputState.liveSuppressed = true;
|
|
1426
|
+
return;
|
|
1427
|
+
}
|
|
1428
|
+
publish(record);
|
|
1429
|
+
outputState.liveBytes += record.byteLength;
|
|
1430
|
+
outputState.liveRecords++;
|
|
1431
|
+
outputState.lastLiveSequence = record.sequence;
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
function flushTerminalControlPlaneOutput(outputState, publish) {
|
|
1435
|
+
if (outputState.terminalFlushed) return;
|
|
1436
|
+
outputState.terminalFlushed = true;
|
|
1437
|
+
if (outputState.liveSuppressed) {
|
|
1438
|
+
const omission = controlPlaneOmissionRecord(outputState);
|
|
1439
|
+
if (omission) publish(omission);
|
|
1440
|
+
}
|
|
1441
|
+
for (const record of retainedControlPlaneRecords(outputState)) {
|
|
1442
|
+
if (record.sequence > outputState.lastLiveSequence) publish(record);
|
|
1443
|
+
}
|
|
1236
1444
|
}
|
|
1237
1445
|
|
|
1238
1446
|
function createLogRecordBuffer() {
|
|
@@ -1289,31 +1497,16 @@ function isValidJsonLine(content) {
|
|
|
1289
1497
|
}
|
|
1290
1498
|
}
|
|
1291
1499
|
|
|
1292
|
-
function
|
|
1293
|
-
if (!line.trim()) return;
|
|
1294
|
-
|
|
1295
|
-
const { timestamp, content } = parseTimestampedLine(line);
|
|
1296
|
-
if (shouldSkipLogLine(content)) {
|
|
1297
|
-
return;
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
const isValidJson = isValidJsonLine(content);
|
|
1301
|
-
state.output += content + '\n';
|
|
1302
|
-
|
|
1303
|
-
if (!state.nested) {
|
|
1304
|
-
agent.lastOutputTime = Date.now();
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1500
|
+
function publishAgentOutputRecord(agent, providerName, record) {
|
|
1307
1501
|
agent._publish({
|
|
1308
1502
|
topic: 'AGENT_OUTPUT',
|
|
1309
1503
|
receiver: 'broadcast',
|
|
1310
1504
|
metadata: buildRawLogOnlyMetadata(),
|
|
1311
|
-
timestamp,
|
|
1505
|
+
timestamp: record.timestamp,
|
|
1312
1506
|
content: {
|
|
1313
|
-
text: content,
|
|
1314
1507
|
data: {
|
|
1315
|
-
type:
|
|
1316
|
-
line: content,
|
|
1508
|
+
type: record.type,
|
|
1509
|
+
line: record.content,
|
|
1317
1510
|
agent: agent.id,
|
|
1318
1511
|
role: agent.role,
|
|
1319
1512
|
iteration: agent.iteration,
|
|
@@ -1323,6 +1516,43 @@ function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
|
1323
1516
|
});
|
|
1324
1517
|
}
|
|
1325
1518
|
|
|
1519
|
+
function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
1520
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
1521
|
+
if (!line.trim()) return;
|
|
1522
|
+
|
|
1523
|
+
const { timestamp, content } = parseTimestampedLine(line);
|
|
1524
|
+
if (shouldSkipLogLine(content)) {
|
|
1525
|
+
return;
|
|
1526
|
+
}
|
|
1527
|
+
const controlPlaneContent = redactTerminalFailureForControlPlane(
|
|
1528
|
+
followerState,
|
|
1529
|
+
providerName,
|
|
1530
|
+
content
|
|
1531
|
+
);
|
|
1532
|
+
|
|
1533
|
+
const isValidJson = isValidJsonLine(controlPlaneContent);
|
|
1534
|
+
const record = appendControlPlaneRecord(followerState.controlPlaneOutput, {
|
|
1535
|
+
content: controlPlaneContent,
|
|
1536
|
+
timestamp,
|
|
1537
|
+
type: isValidJson ? 'json' : 'text',
|
|
1538
|
+
});
|
|
1539
|
+
|
|
1540
|
+
if (!followerState.nested) {
|
|
1541
|
+
agent.lastOutputTime = Date.now();
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
|
|
1545
|
+
publishAgentOutputRecord(agent, providerName, item)
|
|
1546
|
+
);
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
function flushAgentOutput(agent, providerName, state) {
|
|
1550
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
1551
|
+
flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
|
|
1552
|
+
publishAgentOutputRecord(agent, providerName, record)
|
|
1553
|
+
);
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1326
1556
|
function appendLogRecordFragment(buffer, fragment) {
|
|
1327
1557
|
const fragmentBytes = Buffer.byteLength(fragment);
|
|
1328
1558
|
buffer.byteLength += fragmentBytes;
|
|
@@ -1512,11 +1742,13 @@ async function evaluateStructuredSuccess({ agent, taskId, state, success, allowR
|
|
|
1512
1742
|
}
|
|
1513
1743
|
}
|
|
1514
1744
|
|
|
1515
|
-
function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
|
|
1745
|
+
function buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }) {
|
|
1516
1746
|
return extractErrorContext({
|
|
1517
1747
|
output: state.output,
|
|
1518
1748
|
statusOutput: stdout,
|
|
1519
1749
|
taskId,
|
|
1750
|
+
providerName,
|
|
1751
|
+
providerFailure,
|
|
1520
1752
|
debug: {
|
|
1521
1753
|
agentId: agent.id,
|
|
1522
1754
|
providerName,
|
|
@@ -1529,6 +1761,19 @@ function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
|
|
|
1529
1761
|
});
|
|
1530
1762
|
}
|
|
1531
1763
|
|
|
1764
|
+
function resolveCompletionFailure({ classified, agent, taskId, providerName, state, stdout }) {
|
|
1765
|
+
if (classified.success) return { providerFailure: null, errorContext: classified.error };
|
|
1766
|
+
const providerFailure =
|
|
1767
|
+
state.providerFailure || extractProviderFailure(state.output, providerName);
|
|
1768
|
+
return {
|
|
1769
|
+
providerFailure,
|
|
1770
|
+
errorContext:
|
|
1771
|
+
providerFailure?.error ||
|
|
1772
|
+
classified.error ||
|
|
1773
|
+
buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }),
|
|
1774
|
+
};
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1532
1777
|
async function buildCompletionResult({
|
|
1533
1778
|
agent,
|
|
1534
1779
|
taskId,
|
|
@@ -1562,10 +1807,14 @@ async function buildCompletionResult({
|
|
|
1562
1807
|
if (vertexModelError) {
|
|
1563
1808
|
classified.success = false;
|
|
1564
1809
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1810
|
+
const { providerFailure, errorContext } = resolveCompletionFailure({
|
|
1811
|
+
classified,
|
|
1812
|
+
agent,
|
|
1813
|
+
taskId,
|
|
1814
|
+
providerName,
|
|
1815
|
+
state,
|
|
1816
|
+
stdout,
|
|
1817
|
+
});
|
|
1569
1818
|
|
|
1570
1819
|
return {
|
|
1571
1820
|
success: classified.success,
|
|
@@ -1581,6 +1830,7 @@ async function buildCompletionResult({
|
|
|
1581
1830
|
taskInfo,
|
|
1582
1831
|
logicalSuccess: classified.success,
|
|
1583
1832
|
}),
|
|
1833
|
+
providerFailure,
|
|
1584
1834
|
vertexModelError,
|
|
1585
1835
|
};
|
|
1586
1836
|
}
|
|
@@ -1616,7 +1866,29 @@ function finalizeLogFollow(agent, state) {
|
|
|
1616
1866
|
}
|
|
1617
1867
|
}
|
|
1618
1868
|
|
|
1619
|
-
function
|
|
1869
|
+
function settleHostStatusFailure({ agent, providerName, state, resolve, text, data, error }) {
|
|
1870
|
+
if (state.resolved) return;
|
|
1871
|
+
state.resolved = true;
|
|
1872
|
+
finalizeLogFollow(agent, state);
|
|
1873
|
+
flushAgentOutput(agent, providerName, state);
|
|
1874
|
+
agent._publish({
|
|
1875
|
+
topic: 'AGENT_ERROR',
|
|
1876
|
+
receiver: 'broadcast',
|
|
1877
|
+
content: { text, data },
|
|
1878
|
+
});
|
|
1879
|
+
resolve({ success: false, output: state.output, error });
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
function handleStatusExecError({
|
|
1883
|
+
agent,
|
|
1884
|
+
providerName,
|
|
1885
|
+
state,
|
|
1886
|
+
ctPath,
|
|
1887
|
+
taskId,
|
|
1888
|
+
error,
|
|
1889
|
+
stderr,
|
|
1890
|
+
resolve,
|
|
1891
|
+
}) {
|
|
1620
1892
|
if (!error) {
|
|
1621
1893
|
return false;
|
|
1622
1894
|
}
|
|
@@ -1641,30 +1913,20 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
|
|
|
1641
1913
|
`[Agent ${agent.id}] ⚠️ Task ${taskId} not found - will restart to ensure completion`
|
|
1642
1914
|
);
|
|
1643
1915
|
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
},
|
|
1659
|
-
},
|
|
1660
|
-
});
|
|
1661
|
-
|
|
1662
|
-
resolve({
|
|
1663
|
-
success: false,
|
|
1664
|
-
output: state.output,
|
|
1665
|
-
error: `Task not found - restarting for safety`,
|
|
1666
|
-
});
|
|
1667
|
-
}
|
|
1916
|
+
settleHostStatusFailure({
|
|
1917
|
+
agent,
|
|
1918
|
+
providerName,
|
|
1919
|
+
state,
|
|
1920
|
+
resolve,
|
|
1921
|
+
text: `Task ${taskId} not found - restarting for safety`,
|
|
1922
|
+
data: {
|
|
1923
|
+
taskId,
|
|
1924
|
+
error: 'task_not_found',
|
|
1925
|
+
role: agent.role,
|
|
1926
|
+
iteration: agent.iteration,
|
|
1927
|
+
},
|
|
1928
|
+
error: 'Task not found - restarting for safety',
|
|
1929
|
+
});
|
|
1668
1930
|
|
|
1669
1931
|
return true;
|
|
1670
1932
|
}
|
|
@@ -1682,31 +1944,21 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
|
|
|
1682
1944
|
console.error(` Stderr: ${stderr || 'none'}`);
|
|
1683
1945
|
console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
|
|
1684
1946
|
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
},
|
|
1701
|
-
},
|
|
1702
|
-
});
|
|
1703
|
-
|
|
1704
|
-
resolve({
|
|
1705
|
-
success: false,
|
|
1706
|
-
output: state.output,
|
|
1707
|
-
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
1708
|
-
});
|
|
1709
|
-
}
|
|
1947
|
+
settleHostStatusFailure({
|
|
1948
|
+
agent,
|
|
1949
|
+
providerName,
|
|
1950
|
+
state,
|
|
1951
|
+
resolve,
|
|
1952
|
+
text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
|
|
1953
|
+
data: {
|
|
1954
|
+
taskId,
|
|
1955
|
+
error: 'polling_timeout',
|
|
1956
|
+
attempts: state.consecutiveExecFailures,
|
|
1957
|
+
role: agent.role,
|
|
1958
|
+
iteration: agent.iteration,
|
|
1959
|
+
},
|
|
1960
|
+
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
1961
|
+
});
|
|
1710
1962
|
|
|
1711
1963
|
return true;
|
|
1712
1964
|
}
|
|
@@ -1761,6 +2013,7 @@ function handleStatusCompletion({
|
|
|
1761
2013
|
state.resolved = true;
|
|
1762
2014
|
|
|
1763
2015
|
finalizeLogFollow(agent, state);
|
|
2016
|
+
flushAgentOutput(agent, providerName, state);
|
|
1764
2017
|
|
|
1765
2018
|
buildCompletionResult({
|
|
1766
2019
|
agent,
|
|
@@ -1783,6 +2036,7 @@ function buildKillHandler({ agent, taskId, state, providerName, resolve }) {
|
|
|
1783
2036
|
if (state.resolved) return;
|
|
1784
2037
|
state.resolved = true;
|
|
1785
2038
|
finalizeLogFollow(agent, state);
|
|
2039
|
+
flushAgentOutput(agent, providerName, state);
|
|
1786
2040
|
if (!state.nested) {
|
|
1787
2041
|
agent._stopLivenessCheck();
|
|
1788
2042
|
}
|
|
@@ -1842,7 +2096,18 @@ function createLogFollower({
|
|
|
1842
2096
|
(error, stdout, stderr) => {
|
|
1843
2097
|
if (state.resolved) return;
|
|
1844
2098
|
|
|
1845
|
-
if (
|
|
2099
|
+
if (
|
|
2100
|
+
handleStatusExecError({
|
|
2101
|
+
agent,
|
|
2102
|
+
providerName,
|
|
2103
|
+
state,
|
|
2104
|
+
ctPath,
|
|
2105
|
+
taskId,
|
|
2106
|
+
error,
|
|
2107
|
+
stderr,
|
|
2108
|
+
resolve,
|
|
2109
|
+
})
|
|
2110
|
+
) {
|
|
1846
2111
|
return;
|
|
1847
2112
|
}
|
|
1848
2113
|
|
|
@@ -2283,7 +2548,7 @@ async function spawnClaudeTaskIsolatedExecution(agent, context, options = {}) {
|
|
|
2283
2548
|
* - Result: 10-20% overall latency reduction
|
|
2284
2549
|
*/
|
|
2285
2550
|
function createIsolatedLogState(skipStructuredResultCheck = false, nested = false) {
|
|
2286
|
-
|
|
2551
|
+
const state = {
|
|
2287
2552
|
taskExited: false,
|
|
2288
2553
|
resolved: false,
|
|
2289
2554
|
terminationPromise: null,
|
|
@@ -2291,8 +2556,10 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
|
|
|
2291
2556
|
durableTaskStatus: null,
|
|
2292
2557
|
lifecycleHandle: null,
|
|
2293
2558
|
logFilePath: null,
|
|
2294
|
-
|
|
2559
|
+
controlPlaneOutput: createControlPlaneOutputState(),
|
|
2295
2560
|
tailProcess: null,
|
|
2561
|
+
rawBytesSeen: 0,
|
|
2562
|
+
ignoreTailOutput: false,
|
|
2296
2563
|
statusCheckInterval: null,
|
|
2297
2564
|
timeoutTimer: null,
|
|
2298
2565
|
lineBuffer: createLogRecordBuffer(),
|
|
@@ -2300,6 +2567,9 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
|
|
|
2300
2567
|
skipStructuredResultCheck,
|
|
2301
2568
|
nested,
|
|
2302
2569
|
};
|
|
2570
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
2571
|
+
defineControlPlaneOutputAccessor(state, 'fullOutput');
|
|
2572
|
+
return state;
|
|
2303
2573
|
}
|
|
2304
2574
|
|
|
2305
2575
|
function buildIsolatedCleanup(state) {
|
|
@@ -2422,6 +2692,89 @@ async function resolveIsolatedLogFilePath(manager, clusterId, taskId, state) {
|
|
|
2422
2692
|
return state.logFilePath;
|
|
2423
2693
|
}
|
|
2424
2694
|
|
|
2695
|
+
async function captureIsolatedFinalOutputTail(
|
|
2696
|
+
manager,
|
|
2697
|
+
clusterId,
|
|
2698
|
+
logFilePath,
|
|
2699
|
+
state,
|
|
2700
|
+
providerName
|
|
2701
|
+
) {
|
|
2702
|
+
stopIsolatedTailForSettlement(state);
|
|
2703
|
+
const sizeResult = await manager.execInContainer(clusterId, [
|
|
2704
|
+
'sh',
|
|
2705
|
+
'-c',
|
|
2706
|
+
`wc -c < "${logFilePath}" 2>/dev/null || echo 0`,
|
|
2707
|
+
]);
|
|
2708
|
+
const fileSize = Number.parseInt(sizeResult.stdout.trim(), 10);
|
|
2709
|
+
const missingBytes = Number.isSafeInteger(fileSize)
|
|
2710
|
+
? Math.max(0, fileSize - state.rawBytesSeen)
|
|
2711
|
+
: 0;
|
|
2712
|
+
|
|
2713
|
+
if (missingBytes > 0) {
|
|
2714
|
+
const readBytes = Math.min(missingBytes, MAX_CONTROL_PLANE_OUTPUT_BYTES);
|
|
2715
|
+
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
2716
|
+
'sh',
|
|
2717
|
+
'-c',
|
|
2718
|
+
`tail -c ${readBytes} "${logFilePath}" 2>/dev/null || echo ""`,
|
|
2719
|
+
]);
|
|
2720
|
+
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
2721
|
+
if (missingBytes > readBytes) {
|
|
2722
|
+
resetControlPlaneTail(state.controlPlaneOutput, { prefixOmitted: true });
|
|
2723
|
+
state.lineBuffer = createLogRecordBuffer();
|
|
2724
|
+
}
|
|
2725
|
+
appendIsolatedContent(state, finalReadResult.stdout, (line) =>
|
|
2726
|
+
retainIsolatedLine(state, providerName, line)
|
|
2727
|
+
);
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2731
|
+
if (state.lineBuffer.byteLength > 0) {
|
|
2732
|
+
completeLogRecord(
|
|
2733
|
+
state.lineBuffer,
|
|
2734
|
+
(line) => retainIsolatedLine(state, providerName, line),
|
|
2735
|
+
true
|
|
2736
|
+
);
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
|
|
2740
|
+
function resolveIsolatedFailure({
|
|
2741
|
+
success,
|
|
2742
|
+
structuredError,
|
|
2743
|
+
agent,
|
|
2744
|
+
taskId,
|
|
2745
|
+
providerName,
|
|
2746
|
+
state,
|
|
2747
|
+
status,
|
|
2748
|
+
isNotFound,
|
|
2749
|
+
clusterId,
|
|
2750
|
+
logFilePath,
|
|
2751
|
+
}) {
|
|
2752
|
+
if (success) return { providerFailure: null, errorContext: structuredError };
|
|
2753
|
+
const providerFailure =
|
|
2754
|
+
state.providerFailure || extractProviderFailure(state.fullOutput, providerName);
|
|
2755
|
+
const errorContext =
|
|
2756
|
+
providerFailure?.error ||
|
|
2757
|
+
structuredError ||
|
|
2758
|
+
extractErrorContext({
|
|
2759
|
+
output: state.fullOutput,
|
|
2760
|
+
statusOutput: status ? `Status: ${status}` : '',
|
|
2761
|
+
taskId,
|
|
2762
|
+
isNotFound,
|
|
2763
|
+
providerName,
|
|
2764
|
+
debug: {
|
|
2765
|
+
agentId: agent.id,
|
|
2766
|
+
providerName,
|
|
2767
|
+
pid: agent.processPid,
|
|
2768
|
+
cwd: agent.config.cwd || process.cwd(),
|
|
2769
|
+
worktreePath: agent.worktree?.path || null,
|
|
2770
|
+
isolation: true,
|
|
2771
|
+
clusterId,
|
|
2772
|
+
logFilePath,
|
|
2773
|
+
},
|
|
2774
|
+
});
|
|
2775
|
+
return { providerFailure, errorContext };
|
|
2776
|
+
}
|
|
2777
|
+
|
|
2425
2778
|
function settleIsolatedTerminalStatus({
|
|
2426
2779
|
agent,
|
|
2427
2780
|
manager,
|
|
@@ -2434,7 +2787,6 @@ function settleIsolatedTerminalStatus({
|
|
|
2434
2787
|
cleanup,
|
|
2435
2788
|
resolve,
|
|
2436
2789
|
reject,
|
|
2437
|
-
onLine,
|
|
2438
2790
|
}) {
|
|
2439
2791
|
if (state.resolved) return Promise.resolve();
|
|
2440
2792
|
if (state.terminalSettlementPromise) return state.terminalSettlementPromise;
|
|
@@ -2448,17 +2800,9 @@ function settleIsolatedTerminalStatus({
|
|
|
2448
2800
|
const logFilePath = await resolveIsolatedLogFilePath(manager, clusterId, taskId, state);
|
|
2449
2801
|
await new Promise((settle) => setTimeout(settle, 200));
|
|
2450
2802
|
if (state.resolved) return;
|
|
2451
|
-
|
|
2452
|
-
'sh',
|
|
2453
|
-
'-c',
|
|
2454
|
-
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
2455
|
-
]);
|
|
2803
|
+
await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state, providerName);
|
|
2456
2804
|
if (state.resolved) return;
|
|
2457
|
-
|
|
2458
|
-
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
2459
|
-
state.fullOutput = finalReadResult.stdout;
|
|
2460
|
-
replayCompleteLogContent(state.fullOutput, onLine);
|
|
2461
|
-
}
|
|
2805
|
+
flushIsolatedOutput(agent, providerName, taskId, state);
|
|
2462
2806
|
|
|
2463
2807
|
const vertexModelError =
|
|
2464
2808
|
providerName === 'claude'
|
|
@@ -2483,26 +2827,18 @@ function settleIsolatedTerminalStatus({
|
|
|
2483
2827
|
success = evaluated.success;
|
|
2484
2828
|
structuredError = evaluated.error;
|
|
2485
2829
|
}
|
|
2486
|
-
const errorContext =
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
cwd: agent.config.cwd || process.cwd(),
|
|
2499
|
-
worktreePath: agent.worktree?.path || null,
|
|
2500
|
-
isolation: true,
|
|
2501
|
-
clusterId,
|
|
2502
|
-
logFilePath,
|
|
2503
|
-
},
|
|
2504
|
-
})
|
|
2505
|
-
: null);
|
|
2830
|
+
const { providerFailure, errorContext } = resolveIsolatedFailure({
|
|
2831
|
+
success,
|
|
2832
|
+
structuredError,
|
|
2833
|
+
agent,
|
|
2834
|
+
taskId,
|
|
2835
|
+
providerName,
|
|
2836
|
+
state,
|
|
2837
|
+
status,
|
|
2838
|
+
isNotFound,
|
|
2839
|
+
clusterId,
|
|
2840
|
+
logFilePath,
|
|
2841
|
+
});
|
|
2506
2842
|
let parsedResult = null;
|
|
2507
2843
|
if (success && !state.skipStructuredResultCheck && !vertexModelError) {
|
|
2508
2844
|
parsedResult = staleCandidate
|
|
@@ -2522,6 +2858,7 @@ function settleIsolatedTerminalStatus({
|
|
|
2522
2858
|
parsedResult,
|
|
2523
2859
|
error: errorContext,
|
|
2524
2860
|
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
2861
|
+
providerFailure,
|
|
2525
2862
|
vertexModelError,
|
|
2526
2863
|
},
|
|
2527
2864
|
});
|
|
@@ -2542,9 +2879,9 @@ function buildIsolatedLifecycleHandle({
|
|
|
2542
2879
|
cleanup,
|
|
2543
2880
|
resolve,
|
|
2544
2881
|
reject,
|
|
2545
|
-
onLine,
|
|
2546
2882
|
}) {
|
|
2547
|
-
const settleCancellation = (reason, details) =>
|
|
2883
|
+
const settleCancellation = (reason, details) => {
|
|
2884
|
+
flushIsolatedOutput(agent, providerName, taskId, state);
|
|
2548
2885
|
settleIsolatedFollower({
|
|
2549
2886
|
agent,
|
|
2550
2887
|
state,
|
|
@@ -2559,6 +2896,7 @@ function buildIsolatedLifecycleHandle({
|
|
|
2559
2896
|
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
2560
2897
|
},
|
|
2561
2898
|
});
|
|
2899
|
+
};
|
|
2562
2900
|
const terminate = (reason = 'Task killed', details = {}) => {
|
|
2563
2901
|
if (state.durableTaskTerminal) {
|
|
2564
2902
|
if (state.nested) settleCancellation(reason, details);
|
|
@@ -2590,7 +2928,6 @@ function buildIsolatedLifecycleHandle({
|
|
|
2590
2928
|
cleanup,
|
|
2591
2929
|
resolve,
|
|
2592
2930
|
reject,
|
|
2593
|
-
onLine,
|
|
2594
2931
|
});
|
|
2595
2932
|
return termination;
|
|
2596
2933
|
}
|
|
@@ -2618,11 +2955,14 @@ function buildIsolatedLifecycleHandle({
|
|
|
2618
2955
|
};
|
|
2619
2956
|
}
|
|
2620
2957
|
|
|
2621
|
-
function
|
|
2958
|
+
function parseIsolatedLogLine(line) {
|
|
2622
2959
|
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
2623
2960
|
const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
|
|
2624
2961
|
const content = timestampMatch ? timestampMatch[2] : line;
|
|
2962
|
+
return { timestamp, content };
|
|
2963
|
+
}
|
|
2625
2964
|
|
|
2965
|
+
function publishIsolatedOutputRecord(agent, providerName, taskId, record) {
|
|
2626
2966
|
agent.messageBus.publish({
|
|
2627
2967
|
cluster_id: agent.cluster.id,
|
|
2628
2968
|
topic: 'AGENT_OUTPUT',
|
|
@@ -2630,31 +2970,67 @@ function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
|
|
|
2630
2970
|
metadata: buildRawLogOnlyMetadata(),
|
|
2631
2971
|
content: {
|
|
2632
2972
|
data: {
|
|
2633
|
-
line: content,
|
|
2973
|
+
line: record.content,
|
|
2634
2974
|
taskId,
|
|
2635
2975
|
iteration: agent.iteration,
|
|
2636
2976
|
provider: providerName,
|
|
2637
2977
|
},
|
|
2638
2978
|
},
|
|
2979
|
+
timestamp: record.timestamp,
|
|
2980
|
+
});
|
|
2981
|
+
}
|
|
2982
|
+
|
|
2983
|
+
function retainIsolatedLine(state, providerName, line) {
|
|
2984
|
+
const { timestamp, content } = parseIsolatedLogLine(line);
|
|
2985
|
+
const controlPlaneContent = redactTerminalFailureForControlPlane(state, providerName, content);
|
|
2986
|
+
return appendControlPlaneRecord(state.controlPlaneOutput, {
|
|
2987
|
+
content: controlPlaneContent,
|
|
2639
2988
|
timestamp,
|
|
2989
|
+
type: isValidJsonLine(controlPlaneContent) ? 'json' : 'text',
|
|
2640
2990
|
});
|
|
2991
|
+
}
|
|
2641
2992
|
|
|
2642
|
-
|
|
2993
|
+
function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
|
|
2994
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
2995
|
+
const record = retainIsolatedLine(followerState, providerName, line);
|
|
2996
|
+
publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
|
|
2997
|
+
publishIsolatedOutputRecord(agent, providerName, taskId, item)
|
|
2998
|
+
);
|
|
2999
|
+
|
|
3000
|
+
if (!followerState.nested) {
|
|
2643
3001
|
agent.lastOutputTime = Date.now();
|
|
2644
3002
|
}
|
|
2645
3003
|
}
|
|
2646
3004
|
|
|
3005
|
+
function flushIsolatedOutput(agent, providerName, taskId, state) {
|
|
3006
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
3007
|
+
flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
|
|
3008
|
+
publishIsolatedOutputRecord(agent, providerName, taskId, record)
|
|
3009
|
+
);
|
|
3010
|
+
}
|
|
3011
|
+
|
|
2647
3012
|
function appendIsolatedContent(state, content, onLine) {
|
|
2648
3013
|
appendContentToBuffer(state, content, onLine, true);
|
|
2649
3014
|
}
|
|
2650
3015
|
|
|
2651
3016
|
function consumeIsolatedTailChunk(state, data, onLine) {
|
|
2652
3017
|
const chunk = typeof data === 'string' ? data : state.tailDecoder.write(data);
|
|
2653
|
-
if (!chunk) return;
|
|
2654
|
-
state.
|
|
3018
|
+
if (!chunk || state.ignoreTailOutput) return;
|
|
3019
|
+
state.rawBytesSeen += Buffer.byteLength(chunk);
|
|
2655
3020
|
appendIsolatedContent(state, chunk, onLine);
|
|
2656
3021
|
}
|
|
2657
3022
|
|
|
3023
|
+
function stopIsolatedTailForSettlement(state) {
|
|
3024
|
+
state.ignoreTailOutput = true;
|
|
3025
|
+
if (!state.tailProcess) return;
|
|
3026
|
+
try {
|
|
3027
|
+
state.tailProcess.kill('SIGTERM');
|
|
3028
|
+
} catch {
|
|
3029
|
+
// Ignore - process may already be dead.
|
|
3030
|
+
}
|
|
3031
|
+
state.tailProcess = null;
|
|
3032
|
+
}
|
|
3033
|
+
|
|
2658
3034
|
function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
|
|
2659
3035
|
state.tailProcess = manager.spawnInContainer(clusterId, [
|
|
2660
3036
|
'sh',
|
|
@@ -2696,7 +3072,6 @@ async function checkIsolatedStatus({
|
|
|
2696
3072
|
cleanup,
|
|
2697
3073
|
resolve,
|
|
2698
3074
|
reject,
|
|
2699
|
-
onLine,
|
|
2700
3075
|
}) {
|
|
2701
3076
|
if (state.taskExited) return;
|
|
2702
3077
|
|
|
@@ -2749,7 +3124,6 @@ async function checkIsolatedStatus({
|
|
|
2749
3124
|
cleanup,
|
|
2750
3125
|
resolve,
|
|
2751
3126
|
reject,
|
|
2752
|
-
onLine,
|
|
2753
3127
|
});
|
|
2754
3128
|
}
|
|
2755
3129
|
|
|
@@ -2764,7 +3138,6 @@ function startIsolatedStatusChecks({
|
|
|
2764
3138
|
cleanup,
|
|
2765
3139
|
resolve,
|
|
2766
3140
|
reject,
|
|
2767
|
-
onLine,
|
|
2768
3141
|
}) {
|
|
2769
3142
|
state.statusCheckInterval = setInterval(() => {
|
|
2770
3143
|
checkIsolatedStatus({
|
|
@@ -2778,7 +3151,6 @@ function startIsolatedStatusChecks({
|
|
|
2778
3151
|
cleanup,
|
|
2779
3152
|
resolve,
|
|
2780
3153
|
reject,
|
|
2781
|
-
onLine,
|
|
2782
3154
|
}).catch((statusErr) => {
|
|
2783
3155
|
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
2784
3156
|
});
|
|
@@ -2812,7 +3184,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
|
|
|
2812
3184
|
cleanup,
|
|
2813
3185
|
resolve,
|
|
2814
3186
|
reject,
|
|
2815
|
-
onLine,
|
|
2816
3187
|
});
|
|
2817
3188
|
// Only register the lifecycle handle on the agent for top-level tasks.
|
|
2818
3189
|
if (!options.nested) {
|
|
@@ -2883,7 +3254,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
|
|
|
2883
3254
|
cleanup,
|
|
2884
3255
|
resolve,
|
|
2885
3256
|
reject,
|
|
2886
|
-
onLine,
|
|
2887
3257
|
});
|
|
2888
3258
|
|
|
2889
3259
|
if (agent.timeout > 0 && !agent.enableLivenessCheck && !options.nested) {
|
|
@@ -3167,7 +3537,19 @@ module.exports = {
|
|
|
3167
3537
|
buildTaskRunArgs,
|
|
3168
3538
|
rebuildProviderSessionAfterCommit,
|
|
3169
3539
|
killTask,
|
|
3540
|
+
createLogFollowState,
|
|
3541
|
+
createIsolatedLogState,
|
|
3542
|
+
createControlPlaneOutputState,
|
|
3543
|
+
appendControlPlaneRecord,
|
|
3170
3544
|
createLogRecordBuffer,
|
|
3171
3545
|
appendContentToBuffer,
|
|
3172
3546
|
consumeIsolatedTailChunk,
|
|
3547
|
+
flushAgentOutput,
|
|
3548
|
+
flushIsolatedOutput,
|
|
3549
|
+
CONTROL_PLANE_OUTPUT_LIMITS: Object.freeze({
|
|
3550
|
+
maxBytes: MAX_CONTROL_PLANE_OUTPUT_BYTES,
|
|
3551
|
+
maxRecords: MAX_CONTROL_PLANE_OUTPUT_RECORDS,
|
|
3552
|
+
liveBytes: MAX_LIVE_OUTPUT_BYTES,
|
|
3553
|
+
liveRecords: MAX_LIVE_OUTPUT_RECORDS,
|
|
3554
|
+
}),
|
|
3173
3555
|
};
|