@the-open-engine/zeroshot 6.34.2 → 6.34.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@the-open-engine/zeroshot",
3
- "version": "6.34.2",
3
+ "version": "6.34.3",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@the-open-engine/zeroshot",
9
- "version": "6.34.2",
9
+ "version": "6.34.3",
10
10
  "hasInstallScript": true,
11
11
  "license": "MIT",
12
12
  "dependencies": {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@the-open-engine/zeroshot",
3
- "version": "6.34.2",
3
+ "version": "6.34.3",
4
4
  "description": "Independent executor–verifier orchestration for software changes.",
5
5
  "main": "src/orchestrator.js",
6
6
  "bin": {
@@ -24,6 +24,7 @@ const { findPlatformMismatchReason } = require('./validation-platform');
24
24
  const { calculateRateLimitDelay, isRateLimitError } = require('./rate-limit-backoff');
25
25
  const { updateAgentProviderSession } = require('./provider-session');
26
26
  const { rebuildProviderSessionAfterCommit } = require('./agent-task-executor');
27
+ const providerFailures = require('./provider-terminal-failure');
27
28
  const {
28
29
  buildStructuredOutputClusterFailure,
29
30
  isStructuredOutputInvalidError,
@@ -660,6 +661,7 @@ async function runTaskAttempt(agent, triggeringMessage) {
660
661
  error.code = result.code || result.errorType || null;
661
662
  error.taskId = result.taskId || null;
662
663
  error.vertexModelError = result.vertexModelError || null;
664
+ providerFailures.decorateError(error, result.providerFailure);
663
665
  throw error;
664
666
  }
665
667
 
@@ -795,6 +797,16 @@ ${'='.repeat(80)}`);
795
797
 
796
798
  // Non-validator agents: publish error and stop
797
799
  agent.state = 'error';
800
+ const workerFailure = providerFailures.workerFailure(error);
801
+ // Synchronous terminal listeners must see the final failure truth before persisting a stop.
802
+ agent.cluster.failureInfo = providerFailures.buildFinalFailureInfo({
803
+ agent,
804
+ error,
805
+ attempts: failureAttempts,
806
+ worker: workerFailure,
807
+ unsupportedCapability,
808
+ structuredOutputInvalid,
809
+ });
798
810
 
799
811
  // Hook failure: fail the whole cluster so it gets stopped + persisted (prevents deadlocked "running" clusters).
800
812
  if (error?.hookFailure) {
@@ -872,25 +884,14 @@ ${'='.repeat(80)}`);
872
884
  });
873
885
  }
874
886
 
875
- // Save failure info to cluster for resume capability
876
- agent.cluster.failureInfo = {
877
- ...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
878
- agentId: agent.id,
879
- taskId: error?.taskId || agent.currentTaskId,
880
- iteration: agent.iteration,
881
- error: error.message,
887
+ providerFailures.publishCriticalFailure({
888
+ agent,
889
+ error,
882
890
  attempts: failureAttempts,
883
- ...(unsupportedCapability
884
- ? {
885
- code: error.code,
886
- permanent: true,
887
- provider: error.provider,
888
- capability: error.capability,
889
- }
890
- : {}),
891
- ...(structuredOutputInvalid ? { code: error.code, details: error.details ?? null } : {}),
892
- timestamp: Date.now(),
893
- };
891
+ worker: workerFailure,
892
+ unsupportedCapability,
893
+ structuredOutputInvalid,
894
+ });
894
895
 
895
896
  // Publish error to message bus for visibility in logs
896
897
  agent._publish({
@@ -900,7 +901,7 @@ ${'='.repeat(80)}`);
900
901
  text: `Task execution failed after ${failureAttempts} attempts: ${error.message}`,
901
902
  data: {
902
903
  error: error.message,
903
- stack: error.stack,
904
+ stack: error?.provider ? undefined : error.stack,
904
905
  hookFailure: error?.hookFailure === true,
905
906
  restartExhausted: error?.restartExhausted === true,
906
907
  terminationExhausted: error?.terminationExhausted === true,
@@ -911,6 +912,10 @@ ${'='.repeat(80)}`);
911
912
  iteration: agent.iteration,
912
913
  taskId: error?.taskId || agent.currentTaskId,
913
914
  attempts: failureAttempts,
915
+ ...providerFailures.receiptFields(error),
916
+ ...(error?.provider
917
+ ? { workerCode: workerFailure.code, workerReason: workerFailure.reason }
918
+ : {}),
914
919
  ...(unsupportedCapability
915
920
  ? {
916
921
  code: error.code,
@@ -54,6 +54,10 @@ const {
54
54
  validateCompletedResumeIdentity,
55
55
  } = require('./provider-session');
56
56
  const { extractClaudeVertexModelError } = require('./output-extraction');
57
+ const {
58
+ extractProviderFailure,
59
+ redactTerminalFailureForControlPlane,
60
+ } = require('./provider-terminal-failure');
57
61
  const {
58
62
  createStructuredOutputInvalidError,
59
63
  isStructuredOutputInvalidError,
@@ -66,6 +70,8 @@ const MAX_LIVE_OUTPUT_BYTES = 512 * 1024;
66
70
  const MAX_LIVE_OUTPUT_RECORDS = 512;
67
71
  const CONTROL_PLANE_OMISSION_MARKER_BYTES = 1024;
68
72
  const LOG_READ_CHUNK_BYTES = 64 * 1024;
73
+ const MAX_FAILURE_ERROR_BYTES = 4096;
74
+ const FAILURE_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
69
75
  function runCommandWithTimeout(command, args, options = {}, callback = null) {
70
76
  const timeout = options.timeout ?? 30000;
71
77
  if (timeout <= 0) {
@@ -175,6 +181,22 @@ function buildClaudeEnv(modelSpec, options = {}) {
175
181
  function sanitizeErrorMessage(error) {
176
182
  if (!error) return null;
177
183
 
184
+ const original = String(error);
185
+ const suffixBytes = Buffer.byteLength(FAILURE_ERROR_TRUNCATION_SUFFIX);
186
+ const contentBudget = MAX_FAILURE_ERROR_BYTES - suffixBytes;
187
+ let boundedError = original;
188
+ if (Buffer.byteLength(original) > MAX_FAILURE_ERROR_BYTES) {
189
+ let bytes = 0;
190
+ let prefix = '';
191
+ for (const character of original) {
192
+ const characterBytes = Buffer.byteLength(character);
193
+ if (bytes + characterBytes > contentBudget) break;
194
+ prefix += character;
195
+ bytes += characterBytes;
196
+ }
197
+ boundedError = `${prefix}${FAILURE_ERROR_TRUNCATION_SUFFIX}`;
198
+ }
199
+
178
200
  // Patterns that look like TypeScript type annotations (not real error messages)
179
201
  const typeAnnotationPatterns = [
180
202
  /^string\s*\|\s*null$/i,
@@ -187,7 +209,7 @@ function sanitizeErrorMessage(error) {
187
209
  /^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
188
210
  ];
189
211
 
190
- const trimmedError = error.trim();
212
+ const trimmedError = boundedError.trim();
191
213
 
192
214
  // Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
193
215
  const unionParts = trimmedError.split(/\s*\|\s*/);
@@ -196,14 +218,16 @@ function sanitizeErrorMessage(error) {
196
218
  for (const pattern of typeAnnotationPatterns) {
197
219
  if (pattern.test(trimmedError) || isUnionType) {
198
220
  console.warn(
199
- `[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${error}". ` +
221
+ `[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${boundedError}". ` +
200
222
  `This indicates corrupted data. Replacing with generic error.`
201
223
  );
202
- return `Task failed with corrupted error data (original: "${error}")`;
224
+ return sanitizeErrorMessage(
225
+ `Task failed with corrupted error data (original: "${boundedError}")`
226
+ );
203
227
  }
204
228
  }
205
229
 
206
- return error;
230
+ return boundedError;
207
231
  }
208
232
 
209
233
  function safeTail(text, maxChars) {
@@ -295,6 +319,30 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
295
319
  console.error('[AgentTaskExecutor] Claude CLI returned no messages', payload);
296
320
  }
297
321
 
322
+ function extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug }) {
323
+ if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
324
+ return sanitizeErrorMessage(
325
+ `FILE TOO LARGE (Claude Code 256KB limit). ` +
326
+ `Use offset and limit parameters when reading large files. ` +
327
+ `Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
328
+ );
329
+ }
330
+ if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
331
+ return sanitizeErrorMessage(
332
+ `STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
333
+ `This usually happens with AskUserQuestion or interactive prompts. ` +
334
+ `Zeroshot agents must run non-interactively.`
335
+ );
336
+ }
337
+ if (fullOutput.includes('No messages returned')) {
338
+ logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
339
+ return sanitizeErrorMessage(
340
+ `Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
341
+ );
342
+ }
343
+ return null;
344
+ }
345
+
298
346
  /**
299
347
  * Extract error context from task output.
300
348
  * Shared by both isolated and non-isolated modes.
@@ -307,7 +355,15 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
307
355
  * @param {Object} [params.debug] - Additional debug context for logging
308
356
  * @returns {string|null} Sanitized error context or null if extraction failed
309
357
  */
310
- function extractErrorContext({ output, statusOutput, taskId, isNotFound = false, debug }) {
358
+ function extractErrorContext({
359
+ output,
360
+ statusOutput,
361
+ taskId,
362
+ isNotFound = false,
363
+ providerName = getDefaultProviderId(),
364
+ providerFailure = null,
365
+ debug,
366
+ }) {
311
367
  // Task not found - explicit error
312
368
  if (isNotFound) {
313
369
  return sanitizeErrorMessage(`Task ${taskId} not found (may have crashed or been killed)`);
@@ -321,37 +377,15 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false,
321
377
  }
322
378
  }
323
379
 
324
- // KNOWN CLAUDE CODE LIMITATIONS - detect and provide actionable guidance
325
- const fullOutput = output || '';
326
-
327
- // 256KB file limit error
328
- if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
329
- return sanitizeErrorMessage(
330
- `FILE TOO LARGE (Claude Code 256KB limit). ` +
331
- `Use offset and limit parameters when reading large files. ` +
332
- `Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
333
- );
380
+ const terminalFailure = providerFailure || extractProviderFailure(output, providerName);
381
+ if (terminalFailure) {
382
+ return sanitizeErrorMessage(terminalFailure.error);
334
383
  }
335
384
 
336
- // Streaming mode error (interactive tools in non-interactive mode)
337
- if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
338
- return sanitizeErrorMessage(
339
- `STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
340
- `This usually happens with AskUserQuestion or interactive prompts. ` +
341
- `Zeroshot agents must run non-interactively.`
342
- );
343
- }
344
-
345
- // Claude CLI transient failure: no messages returned
346
- if (fullOutput.includes('No messages returned')) {
347
- logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
348
- return sanitizeErrorMessage(
349
- `Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
350
- );
351
- }
385
+ const fullOutput = output || '';
386
+ const knownFailure = extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug });
387
+ if (knownFailure) return knownFailure;
352
388
 
353
- // NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
354
- // If output is too verbose, that's a prompt problem - fix the prompts, not the data
355
389
  const trimmedOutput = (output || '').trim();
356
390
  if (!trimmedOutput) {
357
391
  return sanitizeErrorMessage(
@@ -1490,10 +1524,15 @@ function broadcastAgentLine({ agent, providerName, state, line }) {
1490
1524
  if (shouldSkipLogLine(content)) {
1491
1525
  return;
1492
1526
  }
1527
+ const controlPlaneContent = redactTerminalFailureForControlPlane(
1528
+ followerState,
1529
+ providerName,
1530
+ content
1531
+ );
1493
1532
 
1494
- const isValidJson = isValidJsonLine(content);
1533
+ const isValidJson = isValidJsonLine(controlPlaneContent);
1495
1534
  const record = appendControlPlaneRecord(followerState.controlPlaneOutput, {
1496
- content,
1535
+ content: controlPlaneContent,
1497
1536
  timestamp,
1498
1537
  type: isValidJson ? 'json' : 'text',
1499
1538
  });
@@ -1703,11 +1742,13 @@ async function evaluateStructuredSuccess({ agent, taskId, state, success, allowR
1703
1742
  }
1704
1743
  }
1705
1744
 
1706
- function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
1745
+ function buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }) {
1707
1746
  return extractErrorContext({
1708
1747
  output: state.output,
1709
1748
  statusOutput: stdout,
1710
1749
  taskId,
1750
+ providerName,
1751
+ providerFailure,
1711
1752
  debug: {
1712
1753
  agentId: agent.id,
1713
1754
  providerName,
@@ -1720,6 +1761,19 @@ function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
1720
1761
  });
1721
1762
  }
1722
1763
 
1764
+ function resolveCompletionFailure({ classified, agent, taskId, providerName, state, stdout }) {
1765
+ if (classified.success) return { providerFailure: null, errorContext: classified.error };
1766
+ const providerFailure =
1767
+ state.providerFailure || extractProviderFailure(state.output, providerName);
1768
+ return {
1769
+ providerFailure,
1770
+ errorContext:
1771
+ providerFailure?.error ||
1772
+ classified.error ||
1773
+ buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }),
1774
+ };
1775
+ }
1776
+
1723
1777
  async function buildCompletionResult({
1724
1778
  agent,
1725
1779
  taskId,
@@ -1753,10 +1807,14 @@ async function buildCompletionResult({
1753
1807
  if (vertexModelError) {
1754
1808
  classified.success = false;
1755
1809
  }
1756
- let errorContext = classified.error;
1757
- if (!errorContext && !classified.success) {
1758
- errorContext = buildFailureContext({ agent, taskId, providerName, state, stdout });
1759
- }
1810
+ const { providerFailure, errorContext } = resolveCompletionFailure({
1811
+ classified,
1812
+ agent,
1813
+ taskId,
1814
+ providerName,
1815
+ state,
1816
+ stdout,
1817
+ });
1760
1818
 
1761
1819
  return {
1762
1820
  success: classified.success,
@@ -1772,6 +1830,7 @@ async function buildCompletionResult({
1772
1830
  taskInfo,
1773
1831
  logicalSuccess: classified.success,
1774
1832
  }),
1833
+ providerFailure,
1775
1834
  vertexModelError,
1776
1835
  };
1777
1836
  }
@@ -2633,7 +2692,13 @@ async function resolveIsolatedLogFilePath(manager, clusterId, taskId, state) {
2633
2692
  return state.logFilePath;
2634
2693
  }
2635
2694
 
2636
- async function captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state) {
2695
+ async function captureIsolatedFinalOutputTail(
2696
+ manager,
2697
+ clusterId,
2698
+ logFilePath,
2699
+ state,
2700
+ providerName
2701
+ ) {
2637
2702
  stopIsolatedTailForSettlement(state);
2638
2703
  const sizeResult = await manager.execInContainer(clusterId, [
2639
2704
  'sh',
@@ -2658,16 +2723,58 @@ async function captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, s
2658
2723
  state.lineBuffer = createLogRecordBuffer();
2659
2724
  }
2660
2725
  appendIsolatedContent(state, finalReadResult.stdout, (line) =>
2661
- retainIsolatedLine(state, line)
2726
+ retainIsolatedLine(state, providerName, line)
2662
2727
  );
2663
2728
  }
2664
2729
  }
2665
2730
 
2666
2731
  if (state.lineBuffer.byteLength > 0) {
2667
- completeLogRecord(state.lineBuffer, (line) => retainIsolatedLine(state, line), true);
2732
+ completeLogRecord(
2733
+ state.lineBuffer,
2734
+ (line) => retainIsolatedLine(state, providerName, line),
2735
+ true
2736
+ );
2668
2737
  }
2669
2738
  }
2670
2739
 
2740
+ function resolveIsolatedFailure({
2741
+ success,
2742
+ structuredError,
2743
+ agent,
2744
+ taskId,
2745
+ providerName,
2746
+ state,
2747
+ status,
2748
+ isNotFound,
2749
+ clusterId,
2750
+ logFilePath,
2751
+ }) {
2752
+ if (success) return { providerFailure: null, errorContext: structuredError };
2753
+ const providerFailure =
2754
+ state.providerFailure || extractProviderFailure(state.fullOutput, providerName);
2755
+ const errorContext =
2756
+ providerFailure?.error ||
2757
+ structuredError ||
2758
+ extractErrorContext({
2759
+ output: state.fullOutput,
2760
+ statusOutput: status ? `Status: ${status}` : '',
2761
+ taskId,
2762
+ isNotFound,
2763
+ providerName,
2764
+ debug: {
2765
+ agentId: agent.id,
2766
+ providerName,
2767
+ pid: agent.processPid,
2768
+ cwd: agent.config.cwd || process.cwd(),
2769
+ worktreePath: agent.worktree?.path || null,
2770
+ isolation: true,
2771
+ clusterId,
2772
+ logFilePath,
2773
+ },
2774
+ });
2775
+ return { providerFailure, errorContext };
2776
+ }
2777
+
2671
2778
  function settleIsolatedTerminalStatus({
2672
2779
  agent,
2673
2780
  manager,
@@ -2693,7 +2800,7 @@ function settleIsolatedTerminalStatus({
2693
2800
  const logFilePath = await resolveIsolatedLogFilePath(manager, clusterId, taskId, state);
2694
2801
  await new Promise((settle) => setTimeout(settle, 200));
2695
2802
  if (state.resolved) return;
2696
- await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state);
2803
+ await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state, providerName);
2697
2804
  if (state.resolved) return;
2698
2805
  flushIsolatedOutput(agent, providerName, taskId, state);
2699
2806
 
@@ -2720,26 +2827,18 @@ function settleIsolatedTerminalStatus({
2720
2827
  success = evaluated.success;
2721
2828
  structuredError = evaluated.error;
2722
2829
  }
2723
- const errorContext =
2724
- structuredError ||
2725
- (!success
2726
- ? extractErrorContext({
2727
- output: state.fullOutput,
2728
- statusOutput: status ? `Status: ${status}` : '',
2729
- taskId,
2730
- isNotFound,
2731
- debug: {
2732
- agentId: agent.id,
2733
- providerName,
2734
- pid: agent.processPid,
2735
- cwd: agent.config.cwd || process.cwd(),
2736
- worktreePath: agent.worktree?.path || null,
2737
- isolation: true,
2738
- clusterId,
2739
- logFilePath,
2740
- },
2741
- })
2742
- : null);
2830
+ const { providerFailure, errorContext } = resolveIsolatedFailure({
2831
+ success,
2832
+ structuredError,
2833
+ agent,
2834
+ taskId,
2835
+ providerName,
2836
+ state,
2837
+ status,
2838
+ isNotFound,
2839
+ clusterId,
2840
+ logFilePath,
2841
+ });
2743
2842
  let parsedResult = null;
2744
2843
  if (success && !state.skipStructuredResultCheck && !vertexModelError) {
2745
2844
  parsedResult = staleCandidate
@@ -2759,6 +2858,7 @@ function settleIsolatedTerminalStatus({
2759
2858
  parsedResult,
2760
2859
  error: errorContext,
2761
2860
  tokenUsage: extractTokenUsage(state.fullOutput, providerName),
2861
+ providerFailure,
2762
2862
  vertexModelError,
2763
2863
  },
2764
2864
  });
@@ -2880,18 +2980,19 @@ function publishIsolatedOutputRecord(agent, providerName, taskId, record) {
2880
2980
  });
2881
2981
  }
2882
2982
 
2883
- function retainIsolatedLine(state, line) {
2983
+ function retainIsolatedLine(state, providerName, line) {
2884
2984
  const { timestamp, content } = parseIsolatedLogLine(line);
2985
+ const controlPlaneContent = redactTerminalFailureForControlPlane(state, providerName, content);
2885
2986
  return appendControlPlaneRecord(state.controlPlaneOutput, {
2886
- content,
2987
+ content: controlPlaneContent,
2887
2988
  timestamp,
2888
- type: isValidJsonLine(content) ? 'json' : 'text',
2989
+ type: isValidJsonLine(controlPlaneContent) ? 'json' : 'text',
2889
2990
  });
2890
2991
  }
2891
2992
 
2892
2993
  function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
2893
2994
  const followerState = ensureControlPlaneOutputState(state);
2894
- const record = retainIsolatedLine(followerState, line);
2995
+ const record = retainIsolatedLine(followerState, providerName, line);
2895
2996
  publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
2896
2997
  publishIsolatedOutputRecord(agent, providerName, taskId, item)
2897
2998
  );
@@ -16,8 +16,51 @@
16
16
  * 4. Direct JSON parse of entire output
17
17
  */
18
18
 
19
+ const { createHash } = require('node:crypto');
19
20
  const { parseProviderChunk } = require('../providers');
20
21
 
22
+ const MAX_CLI_ERROR_BYTES = 4096;
23
+ const CLI_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
24
+
25
+ function truncateUtf8(text, maxBytes = MAX_CLI_ERROR_BYTES) {
26
+ if (Buffer.byteLength(text) <= maxBytes) return text;
27
+
28
+ const suffixBytes = Buffer.byteLength(CLI_ERROR_TRUNCATION_SUFFIX);
29
+ const contentBudget = Math.max(0, maxBytes - suffixBytes);
30
+ let bytes = 0;
31
+ let truncated = '';
32
+ for (const character of text) {
33
+ const characterBytes = Buffer.byteLength(character);
34
+ if (bytes + characterBytes > contentBudget) break;
35
+ truncated += character;
36
+ bytes += characterBytes;
37
+ }
38
+ return `${truncated}${CLI_ERROR_TRUNCATION_SUFFIX}`;
39
+ }
40
+
41
+ function primitiveErrorText(value) {
42
+ return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
43
+ ? String(value)
44
+ : '';
45
+ }
46
+
47
+ function cliErrorDetail(value, fallback) {
48
+ const candidate = Array.isArray(value)
49
+ ? value
50
+ .map(primitiveErrorText)
51
+ .filter((message) => message.trim())
52
+ .join('; ')
53
+ : primitiveErrorText(value);
54
+ const raw = candidate.trim() ? candidate : fallback;
55
+ return {
56
+ error: truncateUtf8(raw.trim()),
57
+ diagnostic: {
58
+ byteLength: Buffer.byteLength(raw),
59
+ sha256: createHash('sha256').update(raw).digest('hex'),
60
+ },
61
+ };
62
+ }
63
+
21
64
  /**
22
65
  * Strip timestamp prefix from log lines.
23
66
  * Format: [epochMs]content or [epochMs]{json...}
@@ -308,60 +351,87 @@ function extractClaudeVertexModelError(output, { useVertex = false } = {}) {
308
351
  * @param {string} providerName - Active provider whose terminal errors may be inspected
309
352
  * @returns {{error: string, provider: string}|null} Error info or null
310
353
  */
311
- function extractCliError(output, providerName = 'claude') {
354
+ function claudeFailureFromObject(obj) {
355
+ if (obj.type !== 'result') return null;
356
+ if (obj.is_error === true) {
357
+ const detail = cliErrorDetail(
358
+ Array.isArray(obj.errors) ? obj.errors : obj.error || obj.result,
359
+ 'Unknown CLI error'
360
+ );
361
+ return { ...detail, provider: 'claude' };
362
+ }
363
+ if (obj.subtype !== 'error') return null;
364
+ return {
365
+ ...cliErrorDetail(obj.error || obj.result, 'CLI returned error'),
366
+ provider: 'claude',
367
+ };
368
+ }
369
+
370
+ function codexFailureFromObject(obj) {
371
+ if (obj.type !== 'turn.failed') return null;
372
+ return {
373
+ ...cliErrorDetail(obj.error?.message || obj.error?.code || obj.error, 'Turn failed'),
374
+ provider: 'codex',
375
+ };
376
+ }
377
+
378
+ function geminiFailureFromObject(obj) {
379
+ const geminiFailure =
380
+ (obj.type === 'result' && obj.status === 'error') ||
381
+ (obj.type === 'error' && obj.severity === 'error');
382
+ if (!geminiFailure) return null;
383
+ return {
384
+ ...cliErrorDetail(obj.error?.message || obj.message, 'Gemini CLI error'),
385
+ provider: 'gemini',
386
+ };
387
+ }
388
+
389
+ function opencodeFailureFromObject(obj) {
390
+ if (obj.type !== 'session.error' && obj.type !== 'error') return null;
391
+ return {
392
+ ...cliErrorDetail(
393
+ obj.error?.data?.message || obj.error?.message || obj.error?.name,
394
+ 'Session error'
395
+ ),
396
+ provider: 'opencode',
397
+ };
398
+ }
399
+
400
+ function failureFromProviderObject(obj, providerName) {
401
+ if (providerName === 'claude') return claudeFailureFromObject(obj);
402
+ if (providerName === 'codex') return codexFailureFromObject(obj);
403
+ if (providerName === 'gemini') return geminiFailureFromObject(obj);
404
+ if (providerName === 'opencode') return opencodeFailureFromObject(obj);
405
+ return null;
406
+ }
407
+
408
+ function extractCliFailure(output, providerName = 'claude') {
312
409
  if (!output || typeof output !== 'string') return null;
313
410
 
314
411
  const lines = output.split('\n');
412
+ // Codex terminal truth is a turn.failed record at the newest end of JSONL output.
413
+ // Search it newest-first so a preserved terminal tail wins over older provider chatter.
414
+ if (providerName === 'codex') lines.reverse();
315
415
 
316
416
  for (const line of lines) {
317
417
  const content = stripTimestamp(line);
318
418
  if (!content.startsWith('{')) continue;
319
-
320
- let obj;
321
419
  try {
322
- obj = JSON.parse(content);
420
+ const failure = failureFromProviderObject(JSON.parse(content), providerName);
421
+ if (failure) return failure;
323
422
  } catch {
324
- continue;
325
- }
326
-
327
- if (providerName === 'claude' && obj.type === 'result' && obj.is_error === true) {
328
- const errorMsg = Array.isArray(obj.errors)
329
- ? obj.errors.join('; ')
330
- : obj.error || obj.result || 'Unknown CLI error';
331
- return { error: errorMsg, provider: 'claude' };
332
- }
333
-
334
- if (providerName === 'claude' && obj.type === 'result' && obj.subtype === 'error') {
335
- const errorMsg = obj.error || obj.result || 'CLI returned error';
336
- return { error: errorMsg, provider: 'claude' };
337
- }
338
-
339
- if (providerName === 'codex' && obj.type === 'turn.failed') {
340
- const errorMsg = obj.error?.message || obj.error || 'Turn failed';
341
- return { error: errorMsg, provider: 'codex' };
342
- }
343
-
344
- if (
345
- providerName === 'gemini' &&
346
- ((obj.type === 'result' && obj.status === 'error') ||
347
- (obj.type === 'error' && obj.severity === 'error'))
348
- ) {
349
- return {
350
- error: obj.error?.message || obj.message || 'Gemini CLI error',
351
- provider: 'gemini',
352
- };
353
- }
354
-
355
- if (providerName === 'opencode' && (obj.type === 'session.error' || obj.type === 'error')) {
356
- const errorMsg =
357
- obj.error?.data?.message || obj.error?.message || obj.error?.name || 'Session error';
358
- return { error: errorMsg, provider: 'opencode' };
423
+ // Ignore non-JSON output lines.
359
424
  }
360
425
  }
361
426
 
362
427
  return null;
363
428
  }
364
429
 
430
+ function extractCliError(output, providerName = 'claude') {
431
+ const failure = extractCliFailure(output, providerName);
432
+ return failure ? { error: failure.error, provider: failure.provider } : null;
433
+ }
434
+
365
435
  /**
366
436
  * Detects fatal standalone output lines that indicate no task output was produced.
367
437
  * Only matches when the line itself is the fatal message (not when it appears inside JSON).
@@ -419,8 +489,10 @@ function extractJsonFromOutput(output, providerName = 'claude') {
419
489
  }
420
490
 
421
491
  module.exports = {
492
+ MAX_CLI_ERROR_BYTES,
422
493
  extractJsonFromOutput,
423
494
  extractModelTextFromOutput,
495
+ extractCliFailure,
424
496
  extractCliError,
425
497
  extractClaudeVertexModelError,
426
498
  extractFromResultWrapper,
@@ -0,0 +1,186 @@
1
+ // @ts-nocheck
2
+
3
+ const { getProvider } = require('../providers');
4
+ const { extractCliFailure } = require('./output-extraction');
5
+
6
+ function categoryForProviderFailure(error, classification) {
7
+ const isPermanent = classification.retryable === false;
8
+ const authenticationPattern =
9
+ /(?:invalid[_ -]?api[_ -]?key|api[_ -]?key.*invalid|unauthori[sz]ed|forbidden|authentication|permission denied)/i;
10
+ if (isPermanent && authenticationPattern.test(error)) return 'authentication';
11
+
12
+ const quotaPattern = /(?:insufficient[_ -]?quota|quota exceeded|resource_exhausted)/i;
13
+ if (isPermanent && quotaPattern.test(error)) return 'quota';
14
+ if (isPermanent) return 'permanent';
15
+ return classification.kind === 'unknown-retryable' ? 'unknown' : 'transient';
16
+ }
17
+
18
+ function classifyProviderFailure(providerName, error) {
19
+ let rawClassification = { retryable: true, kind: 'unknown-retryable' };
20
+ try {
21
+ rawClassification = getProvider(providerName).adapter.classifyError(new Error(error));
22
+ } catch {
23
+ // Extraction remains available if a provider adapter cannot be loaded.
24
+ }
25
+ return {
26
+ retryable: rawClassification?.retryable !== false,
27
+ kind:
28
+ typeof rawClassification?.kind === 'string' ? rawClassification.kind : 'unknown-retryable',
29
+ };
30
+ }
31
+
32
+ function extractProviderFailure(output, providerName) {
33
+ const cliError = extractCliFailure(output, providerName);
34
+ if (!cliError) return null;
35
+
36
+ const classification = classifyProviderFailure(providerName, cliError.error);
37
+ const category = categoryForProviderFailure(cliError.error, classification);
38
+ return {
39
+ error: `Provider ${cliError.provider} failed (${category}; ${classification.kind})`,
40
+ provider: cliError.provider,
41
+ event: cliError.provider === 'codex' ? 'turn.failed' : 'terminal_error',
42
+ category,
43
+ classification,
44
+ diagnostic: cliError.diagnostic,
45
+ };
46
+ }
47
+
48
+ function redactTerminalFailureForControlPlane(state, providerName, content) {
49
+ const failure = extractProviderFailure(content, providerName);
50
+ if (!failure) return content;
51
+
52
+ state.providerFailure = failure;
53
+ let eventType = 'provider.failure';
54
+ try {
55
+ const parsed = JSON.parse(content);
56
+ if (typeof parsed?.type === 'string') eventType = parsed.type;
57
+ } catch {
58
+ // extractProviderFailure already proved a supported terminal envelope.
59
+ }
60
+ return JSON.stringify({
61
+ type: eventType,
62
+ ...(providerName === 'claude' ? { is_error: true } : {}),
63
+ ...(providerName === 'gemini' ? { status: 'error', severity: 'error' } : {}),
64
+ error: { message: failure.error },
65
+ zeroshot_failure: {
66
+ provider: failure.provider,
67
+ event: failure.event,
68
+ category: failure.category,
69
+ kind: failure.classification.kind,
70
+ retryable: failure.classification.retryable,
71
+ diagnostic: failure.diagnostic,
72
+ },
73
+ });
74
+ }
75
+
76
+ function decorateError(error, failure) {
77
+ if (!failure) return error;
78
+ error.provider = failure.provider || null;
79
+ error.providerEvent = failure.event || null;
80
+ error.providerCategory = failure.category || null;
81
+ error.classification = failure.classification || null;
82
+ error.providerDiagnostic = failure.diagnostic || null;
83
+ if (failure.classification?.retryable === false) error.permanent = true;
84
+ return error;
85
+ }
86
+
87
+ function receiptFields(error) {
88
+ if (!error?.provider) return {};
89
+ return {
90
+ provider: error.provider,
91
+ event: error.providerEvent,
92
+ category: error.providerCategory,
93
+ kind: error.classification?.kind,
94
+ retryable: error.classification?.retryable,
95
+ diagnostic: error.providerDiagnostic,
96
+ };
97
+ }
98
+
99
+ function workerFailure(error) {
100
+ const authenticationFailure =
101
+ error?.provider &&
102
+ error?.classification?.retryable === false &&
103
+ error?.providerCategory === 'authentication';
104
+ return authenticationFailure
105
+ ? { code: 'refusal', reason: 'authentication_required' }
106
+ : { code: 'crash', reason: 'declared_failure' };
107
+ }
108
+
109
+ function publishCriticalFailure({
110
+ agent,
111
+ error,
112
+ attempts,
113
+ worker,
114
+ unsupportedCapability,
115
+ structuredOutputInvalid,
116
+ }) {
117
+ const specific =
118
+ error?.hookFailure ||
119
+ structuredOutputInvalid ||
120
+ unsupportedCapability ||
121
+ error?.vertexModelError ||
122
+ error?.terminationExhausted;
123
+ const critical =
124
+ agent.role === 'implementation' ||
125
+ agent.role === 'coordinator' ||
126
+ agent.id === 'consensus-coordinator';
127
+ if (!critical || specific) return worker;
128
+
129
+ agent._publish({
130
+ topic: 'CLUSTER_FAILED',
131
+ receiver: 'broadcast',
132
+ content: {
133
+ text: `Critical agent ${agent.id} exhausted its retry budget`,
134
+ data: {
135
+ reason: error?.provider ? 'provider_execution_failed' : 'critical_agent_exhausted',
136
+ agentId: agent.id,
137
+ role: agent.role,
138
+ attempts,
139
+ code: worker.code,
140
+ workerReason: worker.reason,
141
+ ...receiptFields(error),
142
+ },
143
+ },
144
+ });
145
+ return worker;
146
+ }
147
+
148
+ function buildFinalFailureInfo({
149
+ agent,
150
+ error,
151
+ attempts,
152
+ worker,
153
+ unsupportedCapability,
154
+ structuredOutputInvalid,
155
+ }) {
156
+ return {
157
+ ...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
158
+ agentId: agent.id,
159
+ taskId: error?.taskId || agent.currentTaskId,
160
+ iteration: agent.iteration,
161
+ error: error.message,
162
+ attempts,
163
+ ...receiptFields(error),
164
+ ...(error?.provider ? { code: worker.code, workerReason: worker.reason } : {}),
165
+ ...(unsupportedCapability
166
+ ? {
167
+ code: error.code,
168
+ permanent: true,
169
+ provider: error.provider,
170
+ capability: error.capability,
171
+ }
172
+ : {}),
173
+ ...(structuredOutputInvalid ? { code: error.code, details: error.details ?? null } : {}),
174
+ timestamp: Date.now(),
175
+ };
176
+ }
177
+
178
+ module.exports = {
179
+ buildFinalFailureInfo,
180
+ decorateError,
181
+ extractProviderFailure,
182
+ publishCriticalFailure,
183
+ receiptFields,
184
+ redactTerminalFailureForControlPlane,
185
+ workerFailure,
186
+ };
@@ -275,6 +275,7 @@ class Orchestrator {
275
275
  // Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
276
276
  this.closed = false;
277
277
  this._conductorWatchdogs = new Set();
278
+ this._clusterRunBoundaries = new Map();
278
279
 
279
280
  // Track if clusters are loaded (for lazy loading pattern)
280
281
  this._clustersLoaded = options.skipLoad === true;
@@ -1575,11 +1576,17 @@ class Orchestrator {
1575
1576
  }
1576
1577
 
1577
1578
  _registerAgentErrorHandler(messageBus, clusterId) {
1579
+ this._recordClusterRunBoundary(messageBus, clusterId);
1578
1580
  this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
1579
1581
  const agentRole = message.content?.data?.role;
1580
1582
  const attempts = message.content?.data?.attempts || 1;
1581
1583
  const hookFailure = message.content?.data?.hookFailure === true;
1582
1584
  const restartExhausted = message.content?.data?.restartExhausted === true;
1585
+ const durableClusterFailure = this._findCurrentRunClusterFailure(
1586
+ messageBus,
1587
+ clusterId,
1588
+ message.sequence
1589
+ );
1583
1590
 
1584
1591
  await this._saveClusters();
1585
1592
 
@@ -1587,7 +1594,10 @@ class Orchestrator {
1587
1594
  agentRole === 'implementation' ||
1588
1595
  agentRole === 'coordinator' ||
1589
1596
  message.sender === 'consensus-coordinator';
1590
- const shouldStop = shouldStopForRole && (hookFailure || restartExhausted || attempts >= 3);
1597
+ const shouldStop =
1598
+ !durableClusterFailure &&
1599
+ shouldStopForRole &&
1600
+ (hookFailure || restartExhausted || attempts >= 3);
1591
1601
 
1592
1602
  if (shouldStop) {
1593
1603
  this._log(`\n${'='.repeat(80)}`);
@@ -1610,6 +1620,25 @@ class Orchestrator {
1610
1620
  });
1611
1621
  }
1612
1622
 
1623
+ _recordClusterRunBoundary(messageBus, clusterId) {
1624
+ const latest = messageBus.findLast({ cluster_id: clusterId, orderBySequence: true });
1625
+ this._clusterRunBoundaries.set(clusterId, latest?.sequence ?? null);
1626
+ }
1627
+
1628
+ _findCurrentRunClusterFailure(messageBus, clusterId, throughId) {
1629
+ const boundary = this._clusterRunBoundaries.get(clusterId);
1630
+ return (
1631
+ messageBus.query({
1632
+ cluster_id: clusterId,
1633
+ topic: 'CLUSTER_FAILED',
1634
+ order: 'desc',
1635
+ limit: 1,
1636
+ ...(boundary === null ? {} : { afterId: boundary }),
1637
+ ...(throughId === undefined ? {} : { throughId }),
1638
+ })[0] || null
1639
+ );
1640
+ }
1641
+
1613
1642
  _registerPushBlockedHandler(messageBus, clusterId) {
1614
1643
  this._subscribeToClusterTopic(messageBus, clusterId, 'PUSH_BLOCKED', async (message) => {
1615
1644
  const reason = message.content?.data?.blocked_reason || 'unknown';
@@ -2425,6 +2454,7 @@ class Orchestrator {
2425
2454
 
2426
2455
  // Now remove from memory after persisting
2427
2456
  this.clusters.delete(clusterId);
2457
+ this._clusterRunBoundaries.delete(clusterId);
2428
2458
  }
2429
2459
 
2430
2460
  /**
@@ -2462,6 +2492,7 @@ class Orchestrator {
2462
2492
  watchdog.dispose();
2463
2493
  }
2464
2494
  this._conductorWatchdogs.clear();
2495
+ this._clusterRunBoundaries.clear();
2465
2496
 
2466
2497
  for (const cluster of this.clusters.values()) {
2467
2498
  if (typeof cluster.snapshotter?.stop === 'function') {
@@ -2902,6 +2933,7 @@ class Orchestrator {
2902
2933
  }
2903
2934
 
2904
2935
  async _restartClusterAgents(cluster) {
2936
+ this._recordClusterRunBoundary(cluster.messageBus, cluster.id);
2905
2937
  cluster.state = 'running';
2906
2938
  cluster.pid = process.pid;
2907
2939
  for (const agent of cluster.agents) {