@the-open-engine/zeroshot 6.34.1 → 6.34.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,13 +54,24 @@ const {
54
54
  validateCompletedResumeIdentity,
55
55
  } = require('./provider-session');
56
56
  const { extractClaudeVertexModelError } = require('./output-extraction');
57
+ const {
58
+ extractProviderFailure,
59
+ redactTerminalFailureForControlPlane,
60
+ } = require('./provider-terminal-failure');
57
61
  const {
58
62
  createStructuredOutputInvalidError,
59
63
  isStructuredOutputInvalidError,
60
64
  } = require('./structured-output-error');
61
65
  const TASK_TERMINAL_STATUSES = new Set(['completed', 'failed', 'killed', 'stale']);
62
66
  const MAX_CONTROL_PLANE_RECORD_BYTES = 1024 * 1024;
67
+ const MAX_CONTROL_PLANE_OUTPUT_BYTES = 2 * 1024 * 1024;
68
+ const MAX_CONTROL_PLANE_OUTPUT_RECORDS = 1024;
69
+ const MAX_LIVE_OUTPUT_BYTES = 512 * 1024;
70
+ const MAX_LIVE_OUTPUT_RECORDS = 512;
71
+ const CONTROL_PLANE_OMISSION_MARKER_BYTES = 1024;
63
72
  const LOG_READ_CHUNK_BYTES = 64 * 1024;
73
+ const MAX_FAILURE_ERROR_BYTES = 4096;
74
+ const FAILURE_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
64
75
  function runCommandWithTimeout(command, args, options = {}, callback = null) {
65
76
  const timeout = options.timeout ?? 30000;
66
77
  if (timeout <= 0) {
@@ -170,6 +181,22 @@ function buildClaudeEnv(modelSpec, options = {}) {
170
181
  function sanitizeErrorMessage(error) {
171
182
  if (!error) return null;
172
183
 
184
+ const original = String(error);
185
+ const suffixBytes = Buffer.byteLength(FAILURE_ERROR_TRUNCATION_SUFFIX);
186
+ const contentBudget = MAX_FAILURE_ERROR_BYTES - suffixBytes;
187
+ let boundedError = original;
188
+ if (Buffer.byteLength(original) > MAX_FAILURE_ERROR_BYTES) {
189
+ let bytes = 0;
190
+ let prefix = '';
191
+ for (const character of original) {
192
+ const characterBytes = Buffer.byteLength(character);
193
+ if (bytes + characterBytes > contentBudget) break;
194
+ prefix += character;
195
+ bytes += characterBytes;
196
+ }
197
+ boundedError = `${prefix}${FAILURE_ERROR_TRUNCATION_SUFFIX}`;
198
+ }
199
+
173
200
  // Patterns that look like TypeScript type annotations (not real error messages)
174
201
  const typeAnnotationPatterns = [
175
202
  /^string\s*\|\s*null$/i,
@@ -182,7 +209,7 @@ function sanitizeErrorMessage(error) {
182
209
  /^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
183
210
  ];
184
211
 
185
- const trimmedError = error.trim();
212
+ const trimmedError = boundedError.trim();
186
213
 
187
214
  // Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
188
215
  const unionParts = trimmedError.split(/\s*\|\s*/);
@@ -191,14 +218,16 @@ function sanitizeErrorMessage(error) {
191
218
  for (const pattern of typeAnnotationPatterns) {
192
219
  if (pattern.test(trimmedError) || isUnionType) {
193
220
  console.warn(
194
- `[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${error}". ` +
221
+ `[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${boundedError}". ` +
195
222
  `This indicates corrupted data. Replacing with generic error.`
196
223
  );
197
- return `Task failed with corrupted error data (original: "${error}")`;
224
+ return sanitizeErrorMessage(
225
+ `Task failed with corrupted error data (original: "${boundedError}")`
226
+ );
198
227
  }
199
228
  }
200
229
 
201
- return error;
230
+ return boundedError;
202
231
  }
203
232
 
204
233
  function safeTail(text, maxChars) {
@@ -290,6 +319,30 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
290
319
  console.error('[AgentTaskExecutor] Claude CLI returned no messages', payload);
291
320
  }
292
321
 
322
+ function extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug }) {
323
+ if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
324
+ return sanitizeErrorMessage(
325
+ `FILE TOO LARGE (Claude Code 256KB limit). ` +
326
+ `Use offset and limit parameters when reading large files. ` +
327
+ `Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
328
+ );
329
+ }
330
+ if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
331
+ return sanitizeErrorMessage(
332
+ `STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
333
+ `This usually happens with AskUserQuestion or interactive prompts. ` +
334
+ `Zeroshot agents must run non-interactively.`
335
+ );
336
+ }
337
+ if (fullOutput.includes('No messages returned')) {
338
+ logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
339
+ return sanitizeErrorMessage(
340
+ `Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
341
+ );
342
+ }
343
+ return null;
344
+ }
345
+
293
346
  /**
294
347
  * Extract error context from task output.
295
348
  * Shared by both isolated and non-isolated modes.
@@ -302,7 +355,15 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
302
355
  * @param {Object} [params.debug] - Additional debug context for logging
303
356
  * @returns {string|null} Sanitized error context or null if extraction failed
304
357
  */
305
- function extractErrorContext({ output, statusOutput, taskId, isNotFound = false, debug }) {
358
+ function extractErrorContext({
359
+ output,
360
+ statusOutput,
361
+ taskId,
362
+ isNotFound = false,
363
+ providerName = getDefaultProviderId(),
364
+ providerFailure = null,
365
+ debug,
366
+ }) {
306
367
  // Task not found - explicit error
307
368
  if (isNotFound) {
308
369
  return sanitizeErrorMessage(`Task ${taskId} not found (may have crashed or been killed)`);
@@ -316,37 +377,15 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false,
316
377
  }
317
378
  }
318
379
 
319
- // KNOWN CLAUDE CODE LIMITATIONS - detect and provide actionable guidance
320
- const fullOutput = output || '';
321
-
322
- // 256KB file limit error
323
- if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
324
- return sanitizeErrorMessage(
325
- `FILE TOO LARGE (Claude Code 256KB limit). ` +
326
- `Use offset and limit parameters when reading large files. ` +
327
- `Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
328
- );
329
- }
330
-
331
- // Streaming mode error (interactive tools in non-interactive mode)
332
- if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
333
- return sanitizeErrorMessage(
334
- `STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
335
- `This usually happens with AskUserQuestion or interactive prompts. ` +
336
- `Zeroshot agents must run non-interactively.`
337
- );
380
+ const terminalFailure = providerFailure || extractProviderFailure(output, providerName);
381
+ if (terminalFailure) {
382
+ return sanitizeErrorMessage(terminalFailure.error);
338
383
  }
339
384
 
340
- // Claude CLI transient failure: no messages returned
341
- if (fullOutput.includes('No messages returned')) {
342
- logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
343
- return sanitizeErrorMessage(
344
- `Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
345
- );
346
- }
385
+ const fullOutput = output || '';
386
+ const knownFailure = extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug });
387
+ if (knownFailure) return knownFailure;
347
388
 
348
- // NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
349
- // If output is too verbose, that's a prompt problem - fix the prompts, not the data
350
389
  const trimmedOutput = (output || '').trim();
351
390
  if (!trimmedOutput) {
352
391
  return sanitizeErrorMessage(
@@ -1222,8 +1261,8 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
1222
1261
  const MAX_STATUS_FAILURES = 30;
1223
1262
 
1224
1263
  function createLogFollowState() {
1225
- return {
1226
- output: '',
1264
+ const state = {
1265
+ controlPlaneOutput: createControlPlaneOutputState(),
1227
1266
  logFilePath: null,
1228
1267
  lastSize: 0,
1229
1268
  pollInterval: null,
@@ -1233,6 +1272,175 @@ function createLogFollowState() {
1233
1272
  logDecoder: new StringDecoder('utf8'),
1234
1273
  consecutiveExecFailures: 0,
1235
1274
  };
1275
+ defineControlPlaneOutputAccessor(state, 'output');
1276
+ return state;
1277
+ }
1278
+
1279
+ function createControlPlaneOutputState(options = {}) {
1280
+ const maxBytes = options.maxBytes || MAX_CONTROL_PLANE_OUTPUT_BYTES;
1281
+ const maxRecords = options.maxRecords || MAX_CONTROL_PLANE_OUTPUT_RECORDS;
1282
+ const liveByteLimit = options.liveByteLimit || MAX_LIVE_OUTPUT_BYTES;
1283
+ const liveRecordLimit = options.liveRecordLimit || MAX_LIVE_OUTPUT_RECORDS;
1284
+ if (maxBytes <= CONTROL_PLANE_OMISSION_MARKER_BYTES || maxRecords < 1) {
1285
+ throw new Error('Control-plane output bounds must retain space for at least one record');
1286
+ }
1287
+ return {
1288
+ records: [],
1289
+ head: 0,
1290
+ byteLength: 0,
1291
+ maxBytes,
1292
+ maxRecords,
1293
+ liveByteLimit,
1294
+ liveRecordLimit,
1295
+ liveBytes: 0,
1296
+ liveRecords: 0,
1297
+ nextSequence: 0,
1298
+ lastLiveSequence: -1,
1299
+ liveSuppressed: false,
1300
+ terminalFlushed: false,
1301
+ omittedBytes: 0,
1302
+ omittedRecords: 0,
1303
+ omittedDigest: createHash('sha256'),
1304
+ prefixOmitted: false,
1305
+ };
1306
+ }
1307
+
1308
+ function defineControlPlaneOutputAccessor(state, property) {
1309
+ Object.defineProperty(state, property, {
1310
+ configurable: true,
1311
+ enumerable: true,
1312
+ get: () => snapshotControlPlaneOutput(state.controlPlaneOutput),
1313
+ });
1314
+ }
1315
+
1316
+ function ensureControlPlaneOutputState(state = {}) {
1317
+ if (state.controlPlaneOutput) return state;
1318
+ let existingOutput = '';
1319
+ if (typeof state.output === 'string') {
1320
+ existingOutput = state.output;
1321
+ } else if (typeof state.fullOutput === 'string') {
1322
+ existingOutput = state.fullOutput;
1323
+ }
1324
+ state.controlPlaneOutput = createControlPlaneOutputState();
1325
+ if (Object.hasOwn(state, 'output')) delete state.output;
1326
+ if (Object.hasOwn(state, 'fullOutput')) delete state.fullOutput;
1327
+ defineControlPlaneOutputAccessor(state, 'output');
1328
+ if (existingOutput) {
1329
+ replayCompleteLogContent(existingOutput, (content) =>
1330
+ appendControlPlaneRecord(state.controlPlaneOutput, {
1331
+ content,
1332
+ timestamp: Date.now(),
1333
+ type: isValidJsonLine(content) ? 'json' : 'text',
1334
+ })
1335
+ );
1336
+ }
1337
+ return state;
1338
+ }
1339
+
1340
+ function retainedControlPlaneRecords(outputState) {
1341
+ return outputState.records.slice(outputState.head);
1342
+ }
1343
+
1344
+ function compactControlPlaneRecords(outputState) {
1345
+ if (outputState.head >= 64 && outputState.head * 2 >= outputState.records.length) {
1346
+ outputState.records = outputState.records.slice(outputState.head);
1347
+ outputState.head = 0;
1348
+ }
1349
+ }
1350
+
1351
+ function omitOldestControlPlaneRecord(outputState) {
1352
+ const record = outputState.records[outputState.head++];
1353
+ outputState.byteLength -= record.byteLength;
1354
+ outputState.omittedBytes += record.byteLength;
1355
+ outputState.omittedRecords++;
1356
+ outputState.omittedDigest.update(record.text);
1357
+ compactControlPlaneRecords(outputState);
1358
+ }
1359
+
1360
+ function appendControlPlaneRecord(outputState, { content, timestamp, type }) {
1361
+ const text = `${content}\n`;
1362
+ const record = {
1363
+ sequence: outputState.nextSequence++,
1364
+ content,
1365
+ timestamp,
1366
+ type,
1367
+ text,
1368
+ byteLength: Buffer.byteLength(text),
1369
+ };
1370
+ outputState.records.push(record);
1371
+ outputState.byteLength += record.byteLength;
1372
+
1373
+ const payloadLimit = outputState.maxBytes - CONTROL_PLANE_OMISSION_MARKER_BYTES;
1374
+ while (
1375
+ outputState.byteLength > payloadLimit ||
1376
+ outputState.records.length - outputState.head > outputState.maxRecords
1377
+ ) {
1378
+ omitOldestControlPlaneRecord(outputState);
1379
+ }
1380
+ return record;
1381
+ }
1382
+
1383
+ function controlPlaneOmissionRecord(outputState) {
1384
+ if (!outputState.prefixOmitted && outputState.omittedRecords === 0) return null;
1385
+ const digest = outputState.omittedRecords
1386
+ ? `, sha256=${outputState.omittedDigest.copy().digest('hex')}`
1387
+ : '';
1388
+ const known = outputState.omittedRecords
1389
+ ? `records=${outputState.omittedRecords}, byte_length=${outputState.omittedBytes}${digest}`
1390
+ : 'byte_length=unknown';
1391
+ const content =
1392
+ `[ZEROSHOT] Earlier provider output omitted from the bounded control-plane tail (${known}). ` +
1393
+ 'Complete output remains in the task log.';
1394
+ return {
1395
+ content,
1396
+ timestamp: Date.now(),
1397
+ type: 'text',
1398
+ text: `${content}\n`,
1399
+ byteLength: Buffer.byteLength(`${content}\n`),
1400
+ };
1401
+ }
1402
+
1403
+ function snapshotControlPlaneOutput(outputState) {
1404
+ const omission = controlPlaneOmissionRecord(outputState);
1405
+ const records = retainedControlPlaneRecords(outputState);
1406
+ return `${omission ? omission.text : ''}${records.map((record) => record.text).join('')}`;
1407
+ }
1408
+
1409
+ function resetControlPlaneTail(outputState, { prefixOmitted = false } = {}) {
1410
+ outputState.records = [];
1411
+ outputState.head = 0;
1412
+ outputState.byteLength = 0;
1413
+ outputState.omittedBytes = 0;
1414
+ outputState.omittedRecords = 0;
1415
+ outputState.omittedDigest = createHash('sha256');
1416
+ outputState.prefixOmitted = prefixOmitted;
1417
+ }
1418
+
1419
+ function publishLiveControlPlaneRecord(outputState, record, publish) {
1420
+ if (
1421
+ outputState.liveSuppressed ||
1422
+ outputState.liveBytes + record.byteLength > outputState.liveByteLimit ||
1423
+ outputState.liveRecords + 1 > outputState.liveRecordLimit
1424
+ ) {
1425
+ outputState.liveSuppressed = true;
1426
+ return;
1427
+ }
1428
+ publish(record);
1429
+ outputState.liveBytes += record.byteLength;
1430
+ outputState.liveRecords++;
1431
+ outputState.lastLiveSequence = record.sequence;
1432
+ }
1433
+
1434
+ function flushTerminalControlPlaneOutput(outputState, publish) {
1435
+ if (outputState.terminalFlushed) return;
1436
+ outputState.terminalFlushed = true;
1437
+ if (outputState.liveSuppressed) {
1438
+ const omission = controlPlaneOmissionRecord(outputState);
1439
+ if (omission) publish(omission);
1440
+ }
1441
+ for (const record of retainedControlPlaneRecords(outputState)) {
1442
+ if (record.sequence > outputState.lastLiveSequence) publish(record);
1443
+ }
1236
1444
  }
1237
1445
 
1238
1446
  function createLogRecordBuffer() {
@@ -1289,31 +1497,16 @@ function isValidJsonLine(content) {
1289
1497
  }
1290
1498
  }
1291
1499
 
1292
- function broadcastAgentLine({ agent, providerName, state, line }) {
1293
- if (!line.trim()) return;
1294
-
1295
- const { timestamp, content } = parseTimestampedLine(line);
1296
- if (shouldSkipLogLine(content)) {
1297
- return;
1298
- }
1299
-
1300
- const isValidJson = isValidJsonLine(content);
1301
- state.output += content + '\n';
1302
-
1303
- if (!state.nested) {
1304
- agent.lastOutputTime = Date.now();
1305
- }
1306
-
1500
+ function publishAgentOutputRecord(agent, providerName, record) {
1307
1501
  agent._publish({
1308
1502
  topic: 'AGENT_OUTPUT',
1309
1503
  receiver: 'broadcast',
1310
1504
  metadata: buildRawLogOnlyMetadata(),
1311
- timestamp,
1505
+ timestamp: record.timestamp,
1312
1506
  content: {
1313
- text: content,
1314
1507
  data: {
1315
- type: isValidJson ? 'json' : 'text',
1316
- line: content,
1508
+ type: record.type,
1509
+ line: record.content,
1317
1510
  agent: agent.id,
1318
1511
  role: agent.role,
1319
1512
  iteration: agent.iteration,
@@ -1323,6 +1516,43 @@ function broadcastAgentLine({ agent, providerName, state, line }) {
1323
1516
  });
1324
1517
  }
1325
1518
 
1519
+ function broadcastAgentLine({ agent, providerName, state, line }) {
1520
+ const followerState = ensureControlPlaneOutputState(state);
1521
+ if (!line.trim()) return;
1522
+
1523
+ const { timestamp, content } = parseTimestampedLine(line);
1524
+ if (shouldSkipLogLine(content)) {
1525
+ return;
1526
+ }
1527
+ const controlPlaneContent = redactTerminalFailureForControlPlane(
1528
+ followerState,
1529
+ providerName,
1530
+ content
1531
+ );
1532
+
1533
+ const isValidJson = isValidJsonLine(controlPlaneContent);
1534
+ const record = appendControlPlaneRecord(followerState.controlPlaneOutput, {
1535
+ content: controlPlaneContent,
1536
+ timestamp,
1537
+ type: isValidJson ? 'json' : 'text',
1538
+ });
1539
+
1540
+ if (!followerState.nested) {
1541
+ agent.lastOutputTime = Date.now();
1542
+ }
1543
+
1544
+ publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
1545
+ publishAgentOutputRecord(agent, providerName, item)
1546
+ );
1547
+ }
1548
+
1549
+ function flushAgentOutput(agent, providerName, state) {
1550
+ const followerState = ensureControlPlaneOutputState(state);
1551
+ flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
1552
+ publishAgentOutputRecord(agent, providerName, record)
1553
+ );
1554
+ }
1555
+
1326
1556
  function appendLogRecordFragment(buffer, fragment) {
1327
1557
  const fragmentBytes = Buffer.byteLength(fragment);
1328
1558
  buffer.byteLength += fragmentBytes;
@@ -1512,11 +1742,13 @@ async function evaluateStructuredSuccess({ agent, taskId, state, success, allowR
1512
1742
  }
1513
1743
  }
1514
1744
 
1515
- function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
1745
+ function buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }) {
1516
1746
  return extractErrorContext({
1517
1747
  output: state.output,
1518
1748
  statusOutput: stdout,
1519
1749
  taskId,
1750
+ providerName,
1751
+ providerFailure,
1520
1752
  debug: {
1521
1753
  agentId: agent.id,
1522
1754
  providerName,
@@ -1529,6 +1761,19 @@ function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
1529
1761
  });
1530
1762
  }
1531
1763
 
1764
+ function resolveCompletionFailure({ classified, agent, taskId, providerName, state, stdout }) {
1765
+ if (classified.success) return { providerFailure: null, errorContext: classified.error };
1766
+ const providerFailure =
1767
+ state.providerFailure || extractProviderFailure(state.output, providerName);
1768
+ return {
1769
+ providerFailure,
1770
+ errorContext:
1771
+ providerFailure?.error ||
1772
+ classified.error ||
1773
+ buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }),
1774
+ };
1775
+ }
1776
+
1532
1777
  async function buildCompletionResult({
1533
1778
  agent,
1534
1779
  taskId,
@@ -1562,10 +1807,14 @@ async function buildCompletionResult({
1562
1807
  if (vertexModelError) {
1563
1808
  classified.success = false;
1564
1809
  }
1565
- let errorContext = classified.error;
1566
- if (!errorContext && !classified.success) {
1567
- errorContext = buildFailureContext({ agent, taskId, providerName, state, stdout });
1568
- }
1810
+ const { providerFailure, errorContext } = resolveCompletionFailure({
1811
+ classified,
1812
+ agent,
1813
+ taskId,
1814
+ providerName,
1815
+ state,
1816
+ stdout,
1817
+ });
1569
1818
 
1570
1819
  return {
1571
1820
  success: classified.success,
@@ -1581,6 +1830,7 @@ async function buildCompletionResult({
1581
1830
  taskInfo,
1582
1831
  logicalSuccess: classified.success,
1583
1832
  }),
1833
+ providerFailure,
1584
1834
  vertexModelError,
1585
1835
  };
1586
1836
  }
@@ -1616,7 +1866,29 @@ function finalizeLogFollow(agent, state) {
1616
1866
  }
1617
1867
  }
1618
1868
 
1619
- function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve }) {
1869
+ function settleHostStatusFailure({ agent, providerName, state, resolve, text, data, error }) {
1870
+ if (state.resolved) return;
1871
+ state.resolved = true;
1872
+ finalizeLogFollow(agent, state);
1873
+ flushAgentOutput(agent, providerName, state);
1874
+ agent._publish({
1875
+ topic: 'AGENT_ERROR',
1876
+ receiver: 'broadcast',
1877
+ content: { text, data },
1878
+ });
1879
+ resolve({ success: false, output: state.output, error });
1880
+ }
1881
+
1882
+ function handleStatusExecError({
1883
+ agent,
1884
+ providerName,
1885
+ state,
1886
+ ctPath,
1887
+ taskId,
1888
+ error,
1889
+ stderr,
1890
+ resolve,
1891
+ }) {
1620
1892
  if (!error) {
1621
1893
  return false;
1622
1894
  }
@@ -1641,30 +1913,20 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
1641
1913
  `[Agent ${agent.id}] ⚠️ Task ${taskId} not found - will restart to ensure completion`
1642
1914
  );
1643
1915
 
1644
- if (!state.resolved) {
1645
- state.resolved = true;
1646
- finalizeLogFollow(agent, state);
1647
-
1648
- agent._publish({
1649
- topic: 'AGENT_ERROR',
1650
- receiver: 'broadcast',
1651
- content: {
1652
- text: `Task ${taskId} not found - restarting for safety`,
1653
- data: {
1654
- taskId,
1655
- error: 'task_not_found',
1656
- role: agent.role,
1657
- iteration: agent.iteration,
1658
- },
1659
- },
1660
- });
1661
-
1662
- resolve({
1663
- success: false,
1664
- output: state.output,
1665
- error: `Task not found - restarting for safety`,
1666
- });
1667
- }
1916
+ settleHostStatusFailure({
1917
+ agent,
1918
+ providerName,
1919
+ state,
1920
+ resolve,
1921
+ text: `Task ${taskId} not found - restarting for safety`,
1922
+ data: {
1923
+ taskId,
1924
+ error: 'task_not_found',
1925
+ role: agent.role,
1926
+ iteration: agent.iteration,
1927
+ },
1928
+ error: 'Task not found - restarting for safety',
1929
+ });
1668
1930
 
1669
1931
  return true;
1670
1932
  }
@@ -1682,31 +1944,21 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
1682
1944
  console.error(` Stderr: ${stderr || 'none'}`);
1683
1945
  console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
1684
1946
 
1685
- if (!state.resolved) {
1686
- state.resolved = true;
1687
- finalizeLogFollow(agent, state);
1688
-
1689
- agent._publish({
1690
- topic: 'AGENT_ERROR',
1691
- receiver: 'broadcast',
1692
- content: {
1693
- text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
1694
- data: {
1695
- taskId,
1696
- error: 'polling_timeout',
1697
- attempts: state.consecutiveExecFailures,
1698
- role: agent.role,
1699
- iteration: agent.iteration,
1700
- },
1701
- },
1702
- });
1703
-
1704
- resolve({
1705
- success: false,
1706
- output: state.output,
1707
- error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
1708
- });
1709
- }
1947
+ settleHostStatusFailure({
1948
+ agent,
1949
+ providerName,
1950
+ state,
1951
+ resolve,
1952
+ text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
1953
+ data: {
1954
+ taskId,
1955
+ error: 'polling_timeout',
1956
+ attempts: state.consecutiveExecFailures,
1957
+ role: agent.role,
1958
+ iteration: agent.iteration,
1959
+ },
1960
+ error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
1961
+ });
1710
1962
 
1711
1963
  return true;
1712
1964
  }
@@ -1761,6 +2013,7 @@ function handleStatusCompletion({
1761
2013
  state.resolved = true;
1762
2014
 
1763
2015
  finalizeLogFollow(agent, state);
2016
+ flushAgentOutput(agent, providerName, state);
1764
2017
 
1765
2018
  buildCompletionResult({
1766
2019
  agent,
@@ -1783,6 +2036,7 @@ function buildKillHandler({ agent, taskId, state, providerName, resolve }) {
1783
2036
  if (state.resolved) return;
1784
2037
  state.resolved = true;
1785
2038
  finalizeLogFollow(agent, state);
2039
+ flushAgentOutput(agent, providerName, state);
1786
2040
  if (!state.nested) {
1787
2041
  agent._stopLivenessCheck();
1788
2042
  }
@@ -1842,7 +2096,18 @@ function createLogFollower({
1842
2096
  (error, stdout, stderr) => {
1843
2097
  if (state.resolved) return;
1844
2098
 
1845
- if (handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, resolve })) {
2099
+ if (
2100
+ handleStatusExecError({
2101
+ agent,
2102
+ providerName,
2103
+ state,
2104
+ ctPath,
2105
+ taskId,
2106
+ error,
2107
+ stderr,
2108
+ resolve,
2109
+ })
2110
+ ) {
1846
2111
  return;
1847
2112
  }
1848
2113
 
@@ -2283,7 +2548,7 @@ async function spawnClaudeTaskIsolatedExecution(agent, context, options = {}) {
2283
2548
  * - Result: 10-20% overall latency reduction
2284
2549
  */
2285
2550
  function createIsolatedLogState(skipStructuredResultCheck = false, nested = false) {
2286
- return {
2551
+ const state = {
2287
2552
  taskExited: false,
2288
2553
  resolved: false,
2289
2554
  terminationPromise: null,
@@ -2291,8 +2556,10 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
2291
2556
  durableTaskStatus: null,
2292
2557
  lifecycleHandle: null,
2293
2558
  logFilePath: null,
2294
- fullOutput: '',
2559
+ controlPlaneOutput: createControlPlaneOutputState(),
2295
2560
  tailProcess: null,
2561
+ rawBytesSeen: 0,
2562
+ ignoreTailOutput: false,
2296
2563
  statusCheckInterval: null,
2297
2564
  timeoutTimer: null,
2298
2565
  lineBuffer: createLogRecordBuffer(),
@@ -2300,6 +2567,9 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
2300
2567
  skipStructuredResultCheck,
2301
2568
  nested,
2302
2569
  };
2570
+ defineControlPlaneOutputAccessor(state, 'output');
2571
+ defineControlPlaneOutputAccessor(state, 'fullOutput');
2572
+ return state;
2303
2573
  }
2304
2574
 
2305
2575
  function buildIsolatedCleanup(state) {
@@ -2422,6 +2692,89 @@ async function resolveIsolatedLogFilePath(manager, clusterId, taskId, state) {
2422
2692
  return state.logFilePath;
2423
2693
  }
2424
2694
 
2695
+ async function captureIsolatedFinalOutputTail(
2696
+ manager,
2697
+ clusterId,
2698
+ logFilePath,
2699
+ state,
2700
+ providerName
2701
+ ) {
2702
+ stopIsolatedTailForSettlement(state);
2703
+ const sizeResult = await manager.execInContainer(clusterId, [
2704
+ 'sh',
2705
+ '-c',
2706
+ `wc -c < "${logFilePath}" 2>/dev/null || echo 0`,
2707
+ ]);
2708
+ const fileSize = Number.parseInt(sizeResult.stdout.trim(), 10);
2709
+ const missingBytes = Number.isSafeInteger(fileSize)
2710
+ ? Math.max(0, fileSize - state.rawBytesSeen)
2711
+ : 0;
2712
+
2713
+ if (missingBytes > 0) {
2714
+ const readBytes = Math.min(missingBytes, MAX_CONTROL_PLANE_OUTPUT_BYTES);
2715
+ const finalReadResult = await manager.execInContainer(clusterId, [
2716
+ 'sh',
2717
+ '-c',
2718
+ `tail -c ${readBytes} "${logFilePath}" 2>/dev/null || echo ""`,
2719
+ ]);
2720
+ if (finalReadResult.code === 0 && finalReadResult.stdout) {
2721
+ if (missingBytes > readBytes) {
2722
+ resetControlPlaneTail(state.controlPlaneOutput, { prefixOmitted: true });
2723
+ state.lineBuffer = createLogRecordBuffer();
2724
+ }
2725
+ appendIsolatedContent(state, finalReadResult.stdout, (line) =>
2726
+ retainIsolatedLine(state, providerName, line)
2727
+ );
2728
+ }
2729
+ }
2730
+
2731
+ if (state.lineBuffer.byteLength > 0) {
2732
+ completeLogRecord(
2733
+ state.lineBuffer,
2734
+ (line) => retainIsolatedLine(state, providerName, line),
2735
+ true
2736
+ );
2737
+ }
2738
+ }
2739
+
2740
+ function resolveIsolatedFailure({
2741
+ success,
2742
+ structuredError,
2743
+ agent,
2744
+ taskId,
2745
+ providerName,
2746
+ state,
2747
+ status,
2748
+ isNotFound,
2749
+ clusterId,
2750
+ logFilePath,
2751
+ }) {
2752
+ if (success) return { providerFailure: null, errorContext: structuredError };
2753
+ const providerFailure =
2754
+ state.providerFailure || extractProviderFailure(state.fullOutput, providerName);
2755
+ const errorContext =
2756
+ providerFailure?.error ||
2757
+ structuredError ||
2758
+ extractErrorContext({
2759
+ output: state.fullOutput,
2760
+ statusOutput: status ? `Status: ${status}` : '',
2761
+ taskId,
2762
+ isNotFound,
2763
+ providerName,
2764
+ debug: {
2765
+ agentId: agent.id,
2766
+ providerName,
2767
+ pid: agent.processPid,
2768
+ cwd: agent.config.cwd || process.cwd(),
2769
+ worktreePath: agent.worktree?.path || null,
2770
+ isolation: true,
2771
+ clusterId,
2772
+ logFilePath,
2773
+ },
2774
+ });
2775
+ return { providerFailure, errorContext };
2776
+ }
2777
+
2425
2778
  function settleIsolatedTerminalStatus({
2426
2779
  agent,
2427
2780
  manager,
@@ -2434,7 +2787,6 @@ function settleIsolatedTerminalStatus({
2434
2787
  cleanup,
2435
2788
  resolve,
2436
2789
  reject,
2437
- onLine,
2438
2790
  }) {
2439
2791
  if (state.resolved) return Promise.resolve();
2440
2792
  if (state.terminalSettlementPromise) return state.terminalSettlementPromise;
@@ -2448,17 +2800,9 @@ function settleIsolatedTerminalStatus({
2448
2800
  const logFilePath = await resolveIsolatedLogFilePath(manager, clusterId, taskId, state);
2449
2801
  await new Promise((settle) => setTimeout(settle, 200));
2450
2802
  if (state.resolved) return;
2451
- const finalReadResult = await manager.execInContainer(clusterId, [
2452
- 'sh',
2453
- '-c',
2454
- `cat "${logFilePath}" 2>/dev/null || echo ""`,
2455
- ]);
2803
+ await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state, providerName);
2456
2804
  if (state.resolved) return;
2457
-
2458
- if (finalReadResult.code === 0 && finalReadResult.stdout) {
2459
- state.fullOutput = finalReadResult.stdout;
2460
- replayCompleteLogContent(state.fullOutput, onLine);
2461
- }
2805
+ flushIsolatedOutput(agent, providerName, taskId, state);
2462
2806
 
2463
2807
  const vertexModelError =
2464
2808
  providerName === 'claude'
@@ -2483,26 +2827,18 @@ function settleIsolatedTerminalStatus({
2483
2827
  success = evaluated.success;
2484
2828
  structuredError = evaluated.error;
2485
2829
  }
2486
- const errorContext =
2487
- structuredError ||
2488
- (!success
2489
- ? extractErrorContext({
2490
- output: state.fullOutput,
2491
- statusOutput: status ? `Status: ${status}` : '',
2492
- taskId,
2493
- isNotFound,
2494
- debug: {
2495
- agentId: agent.id,
2496
- providerName,
2497
- pid: agent.processPid,
2498
- cwd: agent.config.cwd || process.cwd(),
2499
- worktreePath: agent.worktree?.path || null,
2500
- isolation: true,
2501
- clusterId,
2502
- logFilePath,
2503
- },
2504
- })
2505
- : null);
2830
+ const { providerFailure, errorContext } = resolveIsolatedFailure({
2831
+ success,
2832
+ structuredError,
2833
+ agent,
2834
+ taskId,
2835
+ providerName,
2836
+ state,
2837
+ status,
2838
+ isNotFound,
2839
+ clusterId,
2840
+ logFilePath,
2841
+ });
2506
2842
  let parsedResult = null;
2507
2843
  if (success && !state.skipStructuredResultCheck && !vertexModelError) {
2508
2844
  parsedResult = staleCandidate
@@ -2522,6 +2858,7 @@ function settleIsolatedTerminalStatus({
2522
2858
  parsedResult,
2523
2859
  error: errorContext,
2524
2860
  tokenUsage: extractTokenUsage(state.fullOutput, providerName),
2861
+ providerFailure,
2525
2862
  vertexModelError,
2526
2863
  },
2527
2864
  });
@@ -2542,9 +2879,9 @@ function buildIsolatedLifecycleHandle({
2542
2879
  cleanup,
2543
2880
  resolve,
2544
2881
  reject,
2545
- onLine,
2546
2882
  }) {
2547
- const settleCancellation = (reason, details) =>
2883
+ const settleCancellation = (reason, details) => {
2884
+ flushIsolatedOutput(agent, providerName, taskId, state);
2548
2885
  settleIsolatedFollower({
2549
2886
  agent,
2550
2887
  state,
@@ -2559,6 +2896,7 @@ function buildIsolatedLifecycleHandle({
2559
2896
  tokenUsage: extractTokenUsage(state.fullOutput, providerName),
2560
2897
  },
2561
2898
  });
2899
+ };
2562
2900
  const terminate = (reason = 'Task killed', details = {}) => {
2563
2901
  if (state.durableTaskTerminal) {
2564
2902
  if (state.nested) settleCancellation(reason, details);
@@ -2590,7 +2928,6 @@ function buildIsolatedLifecycleHandle({
2590
2928
  cleanup,
2591
2929
  resolve,
2592
2930
  reject,
2593
- onLine,
2594
2931
  });
2595
2932
  return termination;
2596
2933
  }
@@ -2618,11 +2955,14 @@ function buildIsolatedLifecycleHandle({
2618
2955
  };
2619
2956
  }
2620
2957
 
2621
- function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
2958
+ function parseIsolatedLogLine(line) {
2622
2959
  const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
2623
2960
  const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
2624
2961
  const content = timestampMatch ? timestampMatch[2] : line;
2962
+ return { timestamp, content };
2963
+ }
2625
2964
 
2965
+ function publishIsolatedOutputRecord(agent, providerName, taskId, record) {
2626
2966
  agent.messageBus.publish({
2627
2967
  cluster_id: agent.cluster.id,
2628
2968
  topic: 'AGENT_OUTPUT',
@@ -2630,31 +2970,67 @@ function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
2630
2970
  metadata: buildRawLogOnlyMetadata(),
2631
2971
  content: {
2632
2972
  data: {
2633
- line: content,
2973
+ line: record.content,
2634
2974
  taskId,
2635
2975
  iteration: agent.iteration,
2636
2976
  provider: providerName,
2637
2977
  },
2638
2978
  },
2979
+ timestamp: record.timestamp,
2980
+ });
2981
+ }
2982
+
2983
+ function retainIsolatedLine(state, providerName, line) {
2984
+ const { timestamp, content } = parseIsolatedLogLine(line);
2985
+ const controlPlaneContent = redactTerminalFailureForControlPlane(state, providerName, content);
2986
+ return appendControlPlaneRecord(state.controlPlaneOutput, {
2987
+ content: controlPlaneContent,
2639
2988
  timestamp,
2989
+ type: isValidJsonLine(controlPlaneContent) ? 'json' : 'text',
2640
2990
  });
2991
+ }
2641
2992
 
2642
- if (!state?.nested) {
2993
+ function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
2994
+ const followerState = ensureControlPlaneOutputState(state);
2995
+ const record = retainIsolatedLine(followerState, providerName, line);
2996
+ publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
2997
+ publishIsolatedOutputRecord(agent, providerName, taskId, item)
2998
+ );
2999
+
3000
+ if (!followerState.nested) {
2643
3001
  agent.lastOutputTime = Date.now();
2644
3002
  }
2645
3003
  }
2646
3004
 
3005
+ function flushIsolatedOutput(agent, providerName, taskId, state) {
3006
+ const followerState = ensureControlPlaneOutputState(state);
3007
+ flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
3008
+ publishIsolatedOutputRecord(agent, providerName, taskId, record)
3009
+ );
3010
+ }
3011
+
2647
3012
  function appendIsolatedContent(state, content, onLine) {
2648
3013
  appendContentToBuffer(state, content, onLine, true);
2649
3014
  }
2650
3015
 
2651
3016
  function consumeIsolatedTailChunk(state, data, onLine) {
2652
3017
  const chunk = typeof data === 'string' ? data : state.tailDecoder.write(data);
2653
- if (!chunk) return;
2654
- state.fullOutput += chunk;
3018
+ if (!chunk || state.ignoreTailOutput) return;
3019
+ state.rawBytesSeen += Buffer.byteLength(chunk);
2655
3020
  appendIsolatedContent(state, chunk, onLine);
2656
3021
  }
2657
3022
 
3023
+ function stopIsolatedTailForSettlement(state) {
3024
+ state.ignoreTailOutput = true;
3025
+ if (!state.tailProcess) return;
3026
+ try {
3027
+ state.tailProcess.kill('SIGTERM');
3028
+ } catch {
3029
+ // Ignore - process may already be dead.
3030
+ }
3031
+ state.tailProcess = null;
3032
+ }
3033
+
2658
3034
  function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
2659
3035
  state.tailProcess = manager.spawnInContainer(clusterId, [
2660
3036
  'sh',
@@ -2696,7 +3072,6 @@ async function checkIsolatedStatus({
2696
3072
  cleanup,
2697
3073
  resolve,
2698
3074
  reject,
2699
- onLine,
2700
3075
  }) {
2701
3076
  if (state.taskExited) return;
2702
3077
 
@@ -2749,7 +3124,6 @@ async function checkIsolatedStatus({
2749
3124
  cleanup,
2750
3125
  resolve,
2751
3126
  reject,
2752
- onLine,
2753
3127
  });
2754
3128
  }
2755
3129
 
@@ -2764,7 +3138,6 @@ function startIsolatedStatusChecks({
2764
3138
  cleanup,
2765
3139
  resolve,
2766
3140
  reject,
2767
- onLine,
2768
3141
  }) {
2769
3142
  state.statusCheckInterval = setInterval(() => {
2770
3143
  checkIsolatedStatus({
@@ -2778,7 +3151,6 @@ function startIsolatedStatusChecks({
2778
3151
  cleanup,
2779
3152
  resolve,
2780
3153
  reject,
2781
- onLine,
2782
3154
  }).catch((statusErr) => {
2783
3155
  agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
2784
3156
  });
@@ -2812,7 +3184,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
2812
3184
  cleanup,
2813
3185
  resolve,
2814
3186
  reject,
2815
- onLine,
2816
3187
  });
2817
3188
  // Only register the lifecycle handle on the agent for top-level tasks.
2818
3189
  if (!options.nested) {
@@ -2883,7 +3254,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
2883
3254
  cleanup,
2884
3255
  resolve,
2885
3256
  reject,
2886
- onLine,
2887
3257
  });
2888
3258
 
2889
3259
  if (agent.timeout > 0 && !agent.enableLivenessCheck && !options.nested) {
@@ -3167,7 +3537,19 @@ module.exports = {
3167
3537
  buildTaskRunArgs,
3168
3538
  rebuildProviderSessionAfterCommit,
3169
3539
  killTask,
3540
+ createLogFollowState,
3541
+ createIsolatedLogState,
3542
+ createControlPlaneOutputState,
3543
+ appendControlPlaneRecord,
3170
3544
  createLogRecordBuffer,
3171
3545
  appendContentToBuffer,
3172
3546
  consumeIsolatedTailChunk,
3547
+ flushAgentOutput,
3548
+ flushIsolatedOutput,
3549
+ CONTROL_PLANE_OUTPUT_LIMITS: Object.freeze({
3550
+ maxBytes: MAX_CONTROL_PLANE_OUTPUT_BYTES,
3551
+ maxRecords: MAX_CONTROL_PLANE_OUTPUT_RECORDS,
3552
+ liveBytes: MAX_LIVE_OUTPUT_BYTES,
3553
+ liveRecords: MAX_LIVE_OUTPUT_RECORDS,
3554
+ }),
3173
3555
  };