@the-open-engine/zeroshot 6.34.2 → 6.34.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/src/agent/agent-lifecycle.js +24 -19
- package/src/agent/agent-task-executor.js +169 -68
- package/src/agent/output-extraction.js +111 -39
- package/src/agent/provider-terminal-failure.js +186 -0
- package/src/orchestrator.js +33 -1
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@the-open-engine/zeroshot",
|
|
3
|
-
"version": "6.34.
|
|
3
|
+
"version": "6.34.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@the-open-engine/zeroshot",
|
|
9
|
-
"version": "6.34.
|
|
9
|
+
"version": "6.34.3",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "MIT",
|
|
12
12
|
"dependencies": {
|
package/package.json
CHANGED
|
@@ -24,6 +24,7 @@ const { findPlatformMismatchReason } = require('./validation-platform');
|
|
|
24
24
|
const { calculateRateLimitDelay, isRateLimitError } = require('./rate-limit-backoff');
|
|
25
25
|
const { updateAgentProviderSession } = require('./provider-session');
|
|
26
26
|
const { rebuildProviderSessionAfterCommit } = require('./agent-task-executor');
|
|
27
|
+
const providerFailures = require('./provider-terminal-failure');
|
|
27
28
|
const {
|
|
28
29
|
buildStructuredOutputClusterFailure,
|
|
29
30
|
isStructuredOutputInvalidError,
|
|
@@ -660,6 +661,7 @@ async function runTaskAttempt(agent, triggeringMessage) {
|
|
|
660
661
|
error.code = result.code || result.errorType || null;
|
|
661
662
|
error.taskId = result.taskId || null;
|
|
662
663
|
error.vertexModelError = result.vertexModelError || null;
|
|
664
|
+
providerFailures.decorateError(error, result.providerFailure);
|
|
663
665
|
throw error;
|
|
664
666
|
}
|
|
665
667
|
|
|
@@ -795,6 +797,16 @@ ${'='.repeat(80)}`);
|
|
|
795
797
|
|
|
796
798
|
// Non-validator agents: publish error and stop
|
|
797
799
|
agent.state = 'error';
|
|
800
|
+
const workerFailure = providerFailures.workerFailure(error);
|
|
801
|
+
// Synchronous terminal listeners must see the final failure truth before persisting a stop.
|
|
802
|
+
agent.cluster.failureInfo = providerFailures.buildFinalFailureInfo({
|
|
803
|
+
agent,
|
|
804
|
+
error,
|
|
805
|
+
attempts: failureAttempts,
|
|
806
|
+
worker: workerFailure,
|
|
807
|
+
unsupportedCapability,
|
|
808
|
+
structuredOutputInvalid,
|
|
809
|
+
});
|
|
798
810
|
|
|
799
811
|
// Hook failure: fail the whole cluster so it gets stopped + persisted (prevents deadlocked "running" clusters).
|
|
800
812
|
if (error?.hookFailure) {
|
|
@@ -872,25 +884,14 @@ ${'='.repeat(80)}`);
|
|
|
872
884
|
});
|
|
873
885
|
}
|
|
874
886
|
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
agentId: agent.id,
|
|
879
|
-
taskId: error?.taskId || agent.currentTaskId,
|
|
880
|
-
iteration: agent.iteration,
|
|
881
|
-
error: error.message,
|
|
887
|
+
providerFailures.publishCriticalFailure({
|
|
888
|
+
agent,
|
|
889
|
+
error,
|
|
882
890
|
attempts: failureAttempts,
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
provider: error.provider,
|
|
888
|
-
capability: error.capability,
|
|
889
|
-
}
|
|
890
|
-
: {}),
|
|
891
|
-
...(structuredOutputInvalid ? { code: error.code, details: error.details ?? null } : {}),
|
|
892
|
-
timestamp: Date.now(),
|
|
893
|
-
};
|
|
891
|
+
worker: workerFailure,
|
|
892
|
+
unsupportedCapability,
|
|
893
|
+
structuredOutputInvalid,
|
|
894
|
+
});
|
|
894
895
|
|
|
895
896
|
// Publish error to message bus for visibility in logs
|
|
896
897
|
agent._publish({
|
|
@@ -900,7 +901,7 @@ ${'='.repeat(80)}`);
|
|
|
900
901
|
text: `Task execution failed after ${failureAttempts} attempts: ${error.message}`,
|
|
901
902
|
data: {
|
|
902
903
|
error: error.message,
|
|
903
|
-
stack: error.stack,
|
|
904
|
+
stack: error?.provider ? undefined : error.stack,
|
|
904
905
|
hookFailure: error?.hookFailure === true,
|
|
905
906
|
restartExhausted: error?.restartExhausted === true,
|
|
906
907
|
terminationExhausted: error?.terminationExhausted === true,
|
|
@@ -911,6 +912,10 @@ ${'='.repeat(80)}`);
|
|
|
911
912
|
iteration: agent.iteration,
|
|
912
913
|
taskId: error?.taskId || agent.currentTaskId,
|
|
913
914
|
attempts: failureAttempts,
|
|
915
|
+
...providerFailures.receiptFields(error),
|
|
916
|
+
...(error?.provider
|
|
917
|
+
? { workerCode: workerFailure.code, workerReason: workerFailure.reason }
|
|
918
|
+
: {}),
|
|
914
919
|
...(unsupportedCapability
|
|
915
920
|
? {
|
|
916
921
|
code: error.code,
|
|
@@ -54,6 +54,10 @@ const {
|
|
|
54
54
|
validateCompletedResumeIdentity,
|
|
55
55
|
} = require('./provider-session');
|
|
56
56
|
const { extractClaudeVertexModelError } = require('./output-extraction');
|
|
57
|
+
const {
|
|
58
|
+
extractProviderFailure,
|
|
59
|
+
redactTerminalFailureForControlPlane,
|
|
60
|
+
} = require('./provider-terminal-failure');
|
|
57
61
|
const {
|
|
58
62
|
createStructuredOutputInvalidError,
|
|
59
63
|
isStructuredOutputInvalidError,
|
|
@@ -66,6 +70,8 @@ const MAX_LIVE_OUTPUT_BYTES = 512 * 1024;
|
|
|
66
70
|
const MAX_LIVE_OUTPUT_RECORDS = 512;
|
|
67
71
|
const CONTROL_PLANE_OMISSION_MARKER_BYTES = 1024;
|
|
68
72
|
const LOG_READ_CHUNK_BYTES = 64 * 1024;
|
|
73
|
+
const MAX_FAILURE_ERROR_BYTES = 4096;
|
|
74
|
+
const FAILURE_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
|
|
69
75
|
function runCommandWithTimeout(command, args, options = {}, callback = null) {
|
|
70
76
|
const timeout = options.timeout ?? 30000;
|
|
71
77
|
if (timeout <= 0) {
|
|
@@ -175,6 +181,22 @@ function buildClaudeEnv(modelSpec, options = {}) {
|
|
|
175
181
|
function sanitizeErrorMessage(error) {
|
|
176
182
|
if (!error) return null;
|
|
177
183
|
|
|
184
|
+
const original = String(error);
|
|
185
|
+
const suffixBytes = Buffer.byteLength(FAILURE_ERROR_TRUNCATION_SUFFIX);
|
|
186
|
+
const contentBudget = MAX_FAILURE_ERROR_BYTES - suffixBytes;
|
|
187
|
+
let boundedError = original;
|
|
188
|
+
if (Buffer.byteLength(original) > MAX_FAILURE_ERROR_BYTES) {
|
|
189
|
+
let bytes = 0;
|
|
190
|
+
let prefix = '';
|
|
191
|
+
for (const character of original) {
|
|
192
|
+
const characterBytes = Buffer.byteLength(character);
|
|
193
|
+
if (bytes + characterBytes > contentBudget) break;
|
|
194
|
+
prefix += character;
|
|
195
|
+
bytes += characterBytes;
|
|
196
|
+
}
|
|
197
|
+
boundedError = `${prefix}${FAILURE_ERROR_TRUNCATION_SUFFIX}`;
|
|
198
|
+
}
|
|
199
|
+
|
|
178
200
|
// Patterns that look like TypeScript type annotations (not real error messages)
|
|
179
201
|
const typeAnnotationPatterns = [
|
|
180
202
|
/^string\s*\|\s*null$/i,
|
|
@@ -187,7 +209,7 @@ function sanitizeErrorMessage(error) {
|
|
|
187
209
|
/^[A-Z][a-zA-Z]*\s*\|\s*(?:null|undefined)$/, // e.g., "Error | null"
|
|
188
210
|
];
|
|
189
211
|
|
|
190
|
-
const trimmedError =
|
|
212
|
+
const trimmedError = boundedError.trim();
|
|
191
213
|
|
|
192
214
|
// Check if it's a union type like "string | number | boolean" (ReDoS-safe approach)
|
|
193
215
|
const unionParts = trimmedError.split(/\s*\|\s*/);
|
|
@@ -196,14 +218,16 @@ function sanitizeErrorMessage(error) {
|
|
|
196
218
|
for (const pattern of typeAnnotationPatterns) {
|
|
197
219
|
if (pattern.test(trimmedError) || isUnionType) {
|
|
198
220
|
console.warn(
|
|
199
|
-
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${
|
|
221
|
+
`[agent-task-executor] WARNING: Error message looks like a TypeScript type annotation: "${boundedError}". ` +
|
|
200
222
|
`This indicates corrupted data. Replacing with generic error.`
|
|
201
223
|
);
|
|
202
|
-
return
|
|
224
|
+
return sanitizeErrorMessage(
|
|
225
|
+
`Task failed with corrupted error data (original: "${boundedError}")`
|
|
226
|
+
);
|
|
203
227
|
}
|
|
204
228
|
}
|
|
205
229
|
|
|
206
|
-
return
|
|
230
|
+
return boundedError;
|
|
207
231
|
}
|
|
208
232
|
|
|
209
233
|
function safeTail(text, maxChars) {
|
|
@@ -295,6 +319,30 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
|
|
|
295
319
|
console.error('[AgentTaskExecutor] Claude CLI returned no messages', payload);
|
|
296
320
|
}
|
|
297
321
|
|
|
322
|
+
function extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug }) {
|
|
323
|
+
if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
|
|
324
|
+
return sanitizeErrorMessage(
|
|
325
|
+
`FILE TOO LARGE (Claude Code 256KB limit). ` +
|
|
326
|
+
`Use offset and limit parameters when reading large files. ` +
|
|
327
|
+
`Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
if (fullOutput.includes('only prompt commands are supported in streaming mode')) {
|
|
331
|
+
return sanitizeErrorMessage(
|
|
332
|
+
`STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
|
|
333
|
+
`This usually happens with AskUserQuestion or interactive prompts. ` +
|
|
334
|
+
`Zeroshot agents must run non-interactively.`
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
if (fullOutput.includes('No messages returned')) {
|
|
338
|
+
logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
|
|
339
|
+
return sanitizeErrorMessage(
|
|
340
|
+
`Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
return null;
|
|
344
|
+
}
|
|
345
|
+
|
|
298
346
|
/**
|
|
299
347
|
* Extract error context from task output.
|
|
300
348
|
* Shared by both isolated and non-isolated modes.
|
|
@@ -307,7 +355,15 @@ function logNoMessagesReturned({ taskId, output, statusOutput, debug }) {
|
|
|
307
355
|
* @param {Object} [params.debug] - Additional debug context for logging
|
|
308
356
|
* @returns {string|null} Sanitized error context or null if extraction failed
|
|
309
357
|
*/
|
|
310
|
-
function extractErrorContext({
|
|
358
|
+
function extractErrorContext({
|
|
359
|
+
output,
|
|
360
|
+
statusOutput,
|
|
361
|
+
taskId,
|
|
362
|
+
isNotFound = false,
|
|
363
|
+
providerName = getDefaultProviderId(),
|
|
364
|
+
providerFailure = null,
|
|
365
|
+
debug,
|
|
366
|
+
}) {
|
|
311
367
|
// Task not found - explicit error
|
|
312
368
|
if (isNotFound) {
|
|
313
369
|
return sanitizeErrorMessage(`Task ${taskId} not found (may have crashed or been killed)`);
|
|
@@ -321,37 +377,15 @@ function extractErrorContext({ output, statusOutput, taskId, isNotFound = false,
|
|
|
321
377
|
}
|
|
322
378
|
}
|
|
323
379
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
// 256KB file limit error
|
|
328
|
-
if (fullOutput.includes('exceeds maximum allowed size') || fullOutput.includes('256KB')) {
|
|
329
|
-
return sanitizeErrorMessage(
|
|
330
|
-
`FILE TOO LARGE (Claude Code 256KB limit). ` +
|
|
331
|
-
`Use offset and limit parameters when reading large files. ` +
|
|
332
|
-
`Example: Read tool with offset=0, limit=1000 to read first 1000 lines.`
|
|
333
|
-
);
|
|
380
|
+
const terminalFailure = providerFailure || extractProviderFailure(output, providerName);
|
|
381
|
+
if (terminalFailure) {
|
|
382
|
+
return sanitizeErrorMessage(terminalFailure.error);
|
|
334
383
|
}
|
|
335
384
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
`STREAMING MODE ERROR: Agent tried to use interactive tools in streaming mode. ` +
|
|
340
|
-
`This usually happens with AskUserQuestion or interactive prompts. ` +
|
|
341
|
-
`Zeroshot agents must run non-interactively.`
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
// Claude CLI transient failure: no messages returned
|
|
346
|
-
if (fullOutput.includes('No messages returned')) {
|
|
347
|
-
logNoMessagesReturned({ taskId, output: fullOutput, statusOutput, debug });
|
|
348
|
-
return sanitizeErrorMessage(
|
|
349
|
-
`Claude CLI returned no messages. This is usually transient; retry the task or resume the cluster.`
|
|
350
|
-
);
|
|
351
|
-
}
|
|
385
|
+
const fullOutput = output || '';
|
|
386
|
+
const knownFailure = extractKnownCliFailure({ fullOutput, taskId, statusOutput, debug });
|
|
387
|
+
if (knownFailure) return knownFailure;
|
|
352
388
|
|
|
353
|
-
// NEVER TRUNCATE OUTPUT - truncation corrupts structured JSON and causes false "crash" status
|
|
354
|
-
// If output is too verbose, that's a prompt problem - fix the prompts, not the data
|
|
355
389
|
const trimmedOutput = (output || '').trim();
|
|
356
390
|
if (!trimmedOutput) {
|
|
357
391
|
return sanitizeErrorMessage(
|
|
@@ -1490,10 +1524,15 @@ function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
|
1490
1524
|
if (shouldSkipLogLine(content)) {
|
|
1491
1525
|
return;
|
|
1492
1526
|
}
|
|
1527
|
+
const controlPlaneContent = redactTerminalFailureForControlPlane(
|
|
1528
|
+
followerState,
|
|
1529
|
+
providerName,
|
|
1530
|
+
content
|
|
1531
|
+
);
|
|
1493
1532
|
|
|
1494
|
-
const isValidJson = isValidJsonLine(
|
|
1533
|
+
const isValidJson = isValidJsonLine(controlPlaneContent);
|
|
1495
1534
|
const record = appendControlPlaneRecord(followerState.controlPlaneOutput, {
|
|
1496
|
-
content,
|
|
1535
|
+
content: controlPlaneContent,
|
|
1497
1536
|
timestamp,
|
|
1498
1537
|
type: isValidJson ? 'json' : 'text',
|
|
1499
1538
|
});
|
|
@@ -1703,11 +1742,13 @@ async function evaluateStructuredSuccess({ agent, taskId, state, success, allowR
|
|
|
1703
1742
|
}
|
|
1704
1743
|
}
|
|
1705
1744
|
|
|
1706
|
-
function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
|
|
1745
|
+
function buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }) {
|
|
1707
1746
|
return extractErrorContext({
|
|
1708
1747
|
output: state.output,
|
|
1709
1748
|
statusOutput: stdout,
|
|
1710
1749
|
taskId,
|
|
1750
|
+
providerName,
|
|
1751
|
+
providerFailure,
|
|
1711
1752
|
debug: {
|
|
1712
1753
|
agentId: agent.id,
|
|
1713
1754
|
providerName,
|
|
@@ -1720,6 +1761,19 @@ function buildFailureContext({ agent, taskId, providerName, state, stdout }) {
|
|
|
1720
1761
|
});
|
|
1721
1762
|
}
|
|
1722
1763
|
|
|
1764
|
+
function resolveCompletionFailure({ classified, agent, taskId, providerName, state, stdout }) {
|
|
1765
|
+
if (classified.success) return { providerFailure: null, errorContext: classified.error };
|
|
1766
|
+
const providerFailure =
|
|
1767
|
+
state.providerFailure || extractProviderFailure(state.output, providerName);
|
|
1768
|
+
return {
|
|
1769
|
+
providerFailure,
|
|
1770
|
+
errorContext:
|
|
1771
|
+
providerFailure?.error ||
|
|
1772
|
+
classified.error ||
|
|
1773
|
+
buildFailureContext({ agent, taskId, providerName, state, stdout, providerFailure }),
|
|
1774
|
+
};
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1723
1777
|
async function buildCompletionResult({
|
|
1724
1778
|
agent,
|
|
1725
1779
|
taskId,
|
|
@@ -1753,10 +1807,14 @@ async function buildCompletionResult({
|
|
|
1753
1807
|
if (vertexModelError) {
|
|
1754
1808
|
classified.success = false;
|
|
1755
1809
|
}
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1810
|
+
const { providerFailure, errorContext } = resolveCompletionFailure({
|
|
1811
|
+
classified,
|
|
1812
|
+
agent,
|
|
1813
|
+
taskId,
|
|
1814
|
+
providerName,
|
|
1815
|
+
state,
|
|
1816
|
+
stdout,
|
|
1817
|
+
});
|
|
1760
1818
|
|
|
1761
1819
|
return {
|
|
1762
1820
|
success: classified.success,
|
|
@@ -1772,6 +1830,7 @@ async function buildCompletionResult({
|
|
|
1772
1830
|
taskInfo,
|
|
1773
1831
|
logicalSuccess: classified.success,
|
|
1774
1832
|
}),
|
|
1833
|
+
providerFailure,
|
|
1775
1834
|
vertexModelError,
|
|
1776
1835
|
};
|
|
1777
1836
|
}
|
|
@@ -2633,7 +2692,13 @@ async function resolveIsolatedLogFilePath(manager, clusterId, taskId, state) {
|
|
|
2633
2692
|
return state.logFilePath;
|
|
2634
2693
|
}
|
|
2635
2694
|
|
|
2636
|
-
async function captureIsolatedFinalOutputTail(
|
|
2695
|
+
async function captureIsolatedFinalOutputTail(
|
|
2696
|
+
manager,
|
|
2697
|
+
clusterId,
|
|
2698
|
+
logFilePath,
|
|
2699
|
+
state,
|
|
2700
|
+
providerName
|
|
2701
|
+
) {
|
|
2637
2702
|
stopIsolatedTailForSettlement(state);
|
|
2638
2703
|
const sizeResult = await manager.execInContainer(clusterId, [
|
|
2639
2704
|
'sh',
|
|
@@ -2658,16 +2723,58 @@ async function captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, s
|
|
|
2658
2723
|
state.lineBuffer = createLogRecordBuffer();
|
|
2659
2724
|
}
|
|
2660
2725
|
appendIsolatedContent(state, finalReadResult.stdout, (line) =>
|
|
2661
|
-
retainIsolatedLine(state, line)
|
|
2726
|
+
retainIsolatedLine(state, providerName, line)
|
|
2662
2727
|
);
|
|
2663
2728
|
}
|
|
2664
2729
|
}
|
|
2665
2730
|
|
|
2666
2731
|
if (state.lineBuffer.byteLength > 0) {
|
|
2667
|
-
completeLogRecord(
|
|
2732
|
+
completeLogRecord(
|
|
2733
|
+
state.lineBuffer,
|
|
2734
|
+
(line) => retainIsolatedLine(state, providerName, line),
|
|
2735
|
+
true
|
|
2736
|
+
);
|
|
2668
2737
|
}
|
|
2669
2738
|
}
|
|
2670
2739
|
|
|
2740
|
+
function resolveIsolatedFailure({
|
|
2741
|
+
success,
|
|
2742
|
+
structuredError,
|
|
2743
|
+
agent,
|
|
2744
|
+
taskId,
|
|
2745
|
+
providerName,
|
|
2746
|
+
state,
|
|
2747
|
+
status,
|
|
2748
|
+
isNotFound,
|
|
2749
|
+
clusterId,
|
|
2750
|
+
logFilePath,
|
|
2751
|
+
}) {
|
|
2752
|
+
if (success) return { providerFailure: null, errorContext: structuredError };
|
|
2753
|
+
const providerFailure =
|
|
2754
|
+
state.providerFailure || extractProviderFailure(state.fullOutput, providerName);
|
|
2755
|
+
const errorContext =
|
|
2756
|
+
providerFailure?.error ||
|
|
2757
|
+
structuredError ||
|
|
2758
|
+
extractErrorContext({
|
|
2759
|
+
output: state.fullOutput,
|
|
2760
|
+
statusOutput: status ? `Status: ${status}` : '',
|
|
2761
|
+
taskId,
|
|
2762
|
+
isNotFound,
|
|
2763
|
+
providerName,
|
|
2764
|
+
debug: {
|
|
2765
|
+
agentId: agent.id,
|
|
2766
|
+
providerName,
|
|
2767
|
+
pid: agent.processPid,
|
|
2768
|
+
cwd: agent.config.cwd || process.cwd(),
|
|
2769
|
+
worktreePath: agent.worktree?.path || null,
|
|
2770
|
+
isolation: true,
|
|
2771
|
+
clusterId,
|
|
2772
|
+
logFilePath,
|
|
2773
|
+
},
|
|
2774
|
+
});
|
|
2775
|
+
return { providerFailure, errorContext };
|
|
2776
|
+
}
|
|
2777
|
+
|
|
2671
2778
|
function settleIsolatedTerminalStatus({
|
|
2672
2779
|
agent,
|
|
2673
2780
|
manager,
|
|
@@ -2693,7 +2800,7 @@ function settleIsolatedTerminalStatus({
|
|
|
2693
2800
|
const logFilePath = await resolveIsolatedLogFilePath(manager, clusterId, taskId, state);
|
|
2694
2801
|
await new Promise((settle) => setTimeout(settle, 200));
|
|
2695
2802
|
if (state.resolved) return;
|
|
2696
|
-
await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state);
|
|
2803
|
+
await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state, providerName);
|
|
2697
2804
|
if (state.resolved) return;
|
|
2698
2805
|
flushIsolatedOutput(agent, providerName, taskId, state);
|
|
2699
2806
|
|
|
@@ -2720,26 +2827,18 @@ function settleIsolatedTerminalStatus({
|
|
|
2720
2827
|
success = evaluated.success;
|
|
2721
2828
|
structuredError = evaluated.error;
|
|
2722
2829
|
}
|
|
2723
|
-
const errorContext =
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
cwd: agent.config.cwd || process.cwd(),
|
|
2736
|
-
worktreePath: agent.worktree?.path || null,
|
|
2737
|
-
isolation: true,
|
|
2738
|
-
clusterId,
|
|
2739
|
-
logFilePath,
|
|
2740
|
-
},
|
|
2741
|
-
})
|
|
2742
|
-
: null);
|
|
2830
|
+
const { providerFailure, errorContext } = resolveIsolatedFailure({
|
|
2831
|
+
success,
|
|
2832
|
+
structuredError,
|
|
2833
|
+
agent,
|
|
2834
|
+
taskId,
|
|
2835
|
+
providerName,
|
|
2836
|
+
state,
|
|
2837
|
+
status,
|
|
2838
|
+
isNotFound,
|
|
2839
|
+
clusterId,
|
|
2840
|
+
logFilePath,
|
|
2841
|
+
});
|
|
2743
2842
|
let parsedResult = null;
|
|
2744
2843
|
if (success && !state.skipStructuredResultCheck && !vertexModelError) {
|
|
2745
2844
|
parsedResult = staleCandidate
|
|
@@ -2759,6 +2858,7 @@ function settleIsolatedTerminalStatus({
|
|
|
2759
2858
|
parsedResult,
|
|
2760
2859
|
error: errorContext,
|
|
2761
2860
|
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
2861
|
+
providerFailure,
|
|
2762
2862
|
vertexModelError,
|
|
2763
2863
|
},
|
|
2764
2864
|
});
|
|
@@ -2880,18 +2980,19 @@ function publishIsolatedOutputRecord(agent, providerName, taskId, record) {
|
|
|
2880
2980
|
});
|
|
2881
2981
|
}
|
|
2882
2982
|
|
|
2883
|
-
function retainIsolatedLine(state, line) {
|
|
2983
|
+
function retainIsolatedLine(state, providerName, line) {
|
|
2884
2984
|
const { timestamp, content } = parseIsolatedLogLine(line);
|
|
2985
|
+
const controlPlaneContent = redactTerminalFailureForControlPlane(state, providerName, content);
|
|
2885
2986
|
return appendControlPlaneRecord(state.controlPlaneOutput, {
|
|
2886
|
-
content,
|
|
2987
|
+
content: controlPlaneContent,
|
|
2887
2988
|
timestamp,
|
|
2888
|
-
type: isValidJsonLine(
|
|
2989
|
+
type: isValidJsonLine(controlPlaneContent) ? 'json' : 'text',
|
|
2889
2990
|
});
|
|
2890
2991
|
}
|
|
2891
2992
|
|
|
2892
2993
|
function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
|
|
2893
2994
|
const followerState = ensureControlPlaneOutputState(state);
|
|
2894
|
-
const record = retainIsolatedLine(followerState, line);
|
|
2995
|
+
const record = retainIsolatedLine(followerState, providerName, line);
|
|
2895
2996
|
publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
|
|
2896
2997
|
publishIsolatedOutputRecord(agent, providerName, taskId, item)
|
|
2897
2998
|
);
|
|
@@ -16,8 +16,51 @@
|
|
|
16
16
|
* 4. Direct JSON parse of entire output
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
const { createHash } = require('node:crypto');
|
|
19
20
|
const { parseProviderChunk } = require('../providers');
|
|
20
21
|
|
|
22
|
+
const MAX_CLI_ERROR_BYTES = 4096;
|
|
23
|
+
const CLI_ERROR_TRUNCATION_SUFFIX = '… [truncated]';
|
|
24
|
+
|
|
25
|
+
function truncateUtf8(text, maxBytes = MAX_CLI_ERROR_BYTES) {
|
|
26
|
+
if (Buffer.byteLength(text) <= maxBytes) return text;
|
|
27
|
+
|
|
28
|
+
const suffixBytes = Buffer.byteLength(CLI_ERROR_TRUNCATION_SUFFIX);
|
|
29
|
+
const contentBudget = Math.max(0, maxBytes - suffixBytes);
|
|
30
|
+
let bytes = 0;
|
|
31
|
+
let truncated = '';
|
|
32
|
+
for (const character of text) {
|
|
33
|
+
const characterBytes = Buffer.byteLength(character);
|
|
34
|
+
if (bytes + characterBytes > contentBudget) break;
|
|
35
|
+
truncated += character;
|
|
36
|
+
bytes += characterBytes;
|
|
37
|
+
}
|
|
38
|
+
return `${truncated}${CLI_ERROR_TRUNCATION_SUFFIX}`;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function primitiveErrorText(value) {
|
|
42
|
+
return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'
|
|
43
|
+
? String(value)
|
|
44
|
+
: '';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function cliErrorDetail(value, fallback) {
|
|
48
|
+
const candidate = Array.isArray(value)
|
|
49
|
+
? value
|
|
50
|
+
.map(primitiveErrorText)
|
|
51
|
+
.filter((message) => message.trim())
|
|
52
|
+
.join('; ')
|
|
53
|
+
: primitiveErrorText(value);
|
|
54
|
+
const raw = candidate.trim() ? candidate : fallback;
|
|
55
|
+
return {
|
|
56
|
+
error: truncateUtf8(raw.trim()),
|
|
57
|
+
diagnostic: {
|
|
58
|
+
byteLength: Buffer.byteLength(raw),
|
|
59
|
+
sha256: createHash('sha256').update(raw).digest('hex'),
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
|
|
21
64
|
/**
|
|
22
65
|
* Strip timestamp prefix from log lines.
|
|
23
66
|
* Format: [epochMs]content or [epochMs]{json...}
|
|
@@ -308,60 +351,87 @@ function extractClaudeVertexModelError(output, { useVertex = false } = {}) {
|
|
|
308
351
|
* @param {string} providerName - Active provider whose terminal errors may be inspected
|
|
309
352
|
* @returns {{error: string, provider: string}|null} Error info or null
|
|
310
353
|
*/
|
|
311
|
-
function
|
|
354
|
+
function claudeFailureFromObject(obj) {
|
|
355
|
+
if (obj.type !== 'result') return null;
|
|
356
|
+
if (obj.is_error === true) {
|
|
357
|
+
const detail = cliErrorDetail(
|
|
358
|
+
Array.isArray(obj.errors) ? obj.errors : obj.error || obj.result,
|
|
359
|
+
'Unknown CLI error'
|
|
360
|
+
);
|
|
361
|
+
return { ...detail, provider: 'claude' };
|
|
362
|
+
}
|
|
363
|
+
if (obj.subtype !== 'error') return null;
|
|
364
|
+
return {
|
|
365
|
+
...cliErrorDetail(obj.error || obj.result, 'CLI returned error'),
|
|
366
|
+
provider: 'claude',
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function codexFailureFromObject(obj) {
|
|
371
|
+
if (obj.type !== 'turn.failed') return null;
|
|
372
|
+
return {
|
|
373
|
+
...cliErrorDetail(obj.error?.message || obj.error?.code || obj.error, 'Turn failed'),
|
|
374
|
+
provider: 'codex',
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function geminiFailureFromObject(obj) {
|
|
379
|
+
const geminiFailure =
|
|
380
|
+
(obj.type === 'result' && obj.status === 'error') ||
|
|
381
|
+
(obj.type === 'error' && obj.severity === 'error');
|
|
382
|
+
if (!geminiFailure) return null;
|
|
383
|
+
return {
|
|
384
|
+
...cliErrorDetail(obj.error?.message || obj.message, 'Gemini CLI error'),
|
|
385
|
+
provider: 'gemini',
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function opencodeFailureFromObject(obj) {
|
|
390
|
+
if (obj.type !== 'session.error' && obj.type !== 'error') return null;
|
|
391
|
+
return {
|
|
392
|
+
...cliErrorDetail(
|
|
393
|
+
obj.error?.data?.message || obj.error?.message || obj.error?.name,
|
|
394
|
+
'Session error'
|
|
395
|
+
),
|
|
396
|
+
provider: 'opencode',
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
function failureFromProviderObject(obj, providerName) {
|
|
401
|
+
if (providerName === 'claude') return claudeFailureFromObject(obj);
|
|
402
|
+
if (providerName === 'codex') return codexFailureFromObject(obj);
|
|
403
|
+
if (providerName === 'gemini') return geminiFailureFromObject(obj);
|
|
404
|
+
if (providerName === 'opencode') return opencodeFailureFromObject(obj);
|
|
405
|
+
return null;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function extractCliFailure(output, providerName = 'claude') {
|
|
312
409
|
if (!output || typeof output !== 'string') return null;
|
|
313
410
|
|
|
314
411
|
const lines = output.split('\n');
|
|
412
|
+
// Codex terminal truth is a turn.failed record at the newest end of JSONL output.
|
|
413
|
+
// Search it newest-first so a preserved terminal tail wins over older provider chatter.
|
|
414
|
+
if (providerName === 'codex') lines.reverse();
|
|
315
415
|
|
|
316
416
|
for (const line of lines) {
|
|
317
417
|
const content = stripTimestamp(line);
|
|
318
418
|
if (!content.startsWith('{')) continue;
|
|
319
|
-
|
|
320
|
-
let obj;
|
|
321
419
|
try {
|
|
322
|
-
|
|
420
|
+
const failure = failureFromProviderObject(JSON.parse(content), providerName);
|
|
421
|
+
if (failure) return failure;
|
|
323
422
|
} catch {
|
|
324
|
-
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
if (providerName === 'claude' && obj.type === 'result' && obj.is_error === true) {
|
|
328
|
-
const errorMsg = Array.isArray(obj.errors)
|
|
329
|
-
? obj.errors.join('; ')
|
|
330
|
-
: obj.error || obj.result || 'Unknown CLI error';
|
|
331
|
-
return { error: errorMsg, provider: 'claude' };
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
if (providerName === 'claude' && obj.type === 'result' && obj.subtype === 'error') {
|
|
335
|
-
const errorMsg = obj.error || obj.result || 'CLI returned error';
|
|
336
|
-
return { error: errorMsg, provider: 'claude' };
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
if (providerName === 'codex' && obj.type === 'turn.failed') {
|
|
340
|
-
const errorMsg = obj.error?.message || obj.error || 'Turn failed';
|
|
341
|
-
return { error: errorMsg, provider: 'codex' };
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
if (
|
|
345
|
-
providerName === 'gemini' &&
|
|
346
|
-
((obj.type === 'result' && obj.status === 'error') ||
|
|
347
|
-
(obj.type === 'error' && obj.severity === 'error'))
|
|
348
|
-
) {
|
|
349
|
-
return {
|
|
350
|
-
error: obj.error?.message || obj.message || 'Gemini CLI error',
|
|
351
|
-
provider: 'gemini',
|
|
352
|
-
};
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
if (providerName === 'opencode' && (obj.type === 'session.error' || obj.type === 'error')) {
|
|
356
|
-
const errorMsg =
|
|
357
|
-
obj.error?.data?.message || obj.error?.message || obj.error?.name || 'Session error';
|
|
358
|
-
return { error: errorMsg, provider: 'opencode' };
|
|
423
|
+
// Ignore non-JSON output lines.
|
|
359
424
|
}
|
|
360
425
|
}
|
|
361
426
|
|
|
362
427
|
return null;
|
|
363
428
|
}
|
|
364
429
|
|
|
430
|
+
function extractCliError(output, providerName = 'claude') {
|
|
431
|
+
const failure = extractCliFailure(output, providerName);
|
|
432
|
+
return failure ? { error: failure.error, provider: failure.provider } : null;
|
|
433
|
+
}
|
|
434
|
+
|
|
365
435
|
/**
|
|
366
436
|
* Detects fatal standalone output lines that indicate no task output was produced.
|
|
367
437
|
* Only matches when the line itself is the fatal message (not when it appears inside JSON).
|
|
@@ -419,8 +489,10 @@ function extractJsonFromOutput(output, providerName = 'claude') {
|
|
|
419
489
|
}
|
|
420
490
|
|
|
421
491
|
module.exports = {
|
|
492
|
+
MAX_CLI_ERROR_BYTES,
|
|
422
493
|
extractJsonFromOutput,
|
|
423
494
|
extractModelTextFromOutput,
|
|
495
|
+
extractCliFailure,
|
|
424
496
|
extractCliError,
|
|
425
497
|
extractClaudeVertexModelError,
|
|
426
498
|
extractFromResultWrapper,
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
|
|
3
|
+
const { getProvider } = require('../providers');
|
|
4
|
+
const { extractCliFailure } = require('./output-extraction');
|
|
5
|
+
|
|
6
|
+
function categoryForProviderFailure(error, classification) {
|
|
7
|
+
const isPermanent = classification.retryable === false;
|
|
8
|
+
const authenticationPattern =
|
|
9
|
+
/(?:invalid[_ -]?api[_ -]?key|api[_ -]?key.*invalid|unauthori[sz]ed|forbidden|authentication|permission denied)/i;
|
|
10
|
+
if (isPermanent && authenticationPattern.test(error)) return 'authentication';
|
|
11
|
+
|
|
12
|
+
const quotaPattern = /(?:insufficient[_ -]?quota|quota exceeded|resource_exhausted)/i;
|
|
13
|
+
if (isPermanent && quotaPattern.test(error)) return 'quota';
|
|
14
|
+
if (isPermanent) return 'permanent';
|
|
15
|
+
return classification.kind === 'unknown-retryable' ? 'unknown' : 'transient';
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function classifyProviderFailure(providerName, error) {
|
|
19
|
+
let rawClassification = { retryable: true, kind: 'unknown-retryable' };
|
|
20
|
+
try {
|
|
21
|
+
rawClassification = getProvider(providerName).adapter.classifyError(new Error(error));
|
|
22
|
+
} catch {
|
|
23
|
+
// Extraction remains available if a provider adapter cannot be loaded.
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
retryable: rawClassification?.retryable !== false,
|
|
27
|
+
kind:
|
|
28
|
+
typeof rawClassification?.kind === 'string' ? rawClassification.kind : 'unknown-retryable',
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function extractProviderFailure(output, providerName) {
|
|
33
|
+
const cliError = extractCliFailure(output, providerName);
|
|
34
|
+
if (!cliError) return null;
|
|
35
|
+
|
|
36
|
+
const classification = classifyProviderFailure(providerName, cliError.error);
|
|
37
|
+
const category = categoryForProviderFailure(cliError.error, classification);
|
|
38
|
+
return {
|
|
39
|
+
error: `Provider ${cliError.provider} failed (${category}; ${classification.kind})`,
|
|
40
|
+
provider: cliError.provider,
|
|
41
|
+
event: cliError.provider === 'codex' ? 'turn.failed' : 'terminal_error',
|
|
42
|
+
category,
|
|
43
|
+
classification,
|
|
44
|
+
diagnostic: cliError.diagnostic,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function redactTerminalFailureForControlPlane(state, providerName, content) {
|
|
49
|
+
const failure = extractProviderFailure(content, providerName);
|
|
50
|
+
if (!failure) return content;
|
|
51
|
+
|
|
52
|
+
state.providerFailure = failure;
|
|
53
|
+
let eventType = 'provider.failure';
|
|
54
|
+
try {
|
|
55
|
+
const parsed = JSON.parse(content);
|
|
56
|
+
if (typeof parsed?.type === 'string') eventType = parsed.type;
|
|
57
|
+
} catch {
|
|
58
|
+
// extractProviderFailure already proved a supported terminal envelope.
|
|
59
|
+
}
|
|
60
|
+
return JSON.stringify({
|
|
61
|
+
type: eventType,
|
|
62
|
+
...(providerName === 'claude' ? { is_error: true } : {}),
|
|
63
|
+
...(providerName === 'gemini' ? { status: 'error', severity: 'error' } : {}),
|
|
64
|
+
error: { message: failure.error },
|
|
65
|
+
zeroshot_failure: {
|
|
66
|
+
provider: failure.provider,
|
|
67
|
+
event: failure.event,
|
|
68
|
+
category: failure.category,
|
|
69
|
+
kind: failure.classification.kind,
|
|
70
|
+
retryable: failure.classification.retryable,
|
|
71
|
+
diagnostic: failure.diagnostic,
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function decorateError(error, failure) {
|
|
77
|
+
if (!failure) return error;
|
|
78
|
+
error.provider = failure.provider || null;
|
|
79
|
+
error.providerEvent = failure.event || null;
|
|
80
|
+
error.providerCategory = failure.category || null;
|
|
81
|
+
error.classification = failure.classification || null;
|
|
82
|
+
error.providerDiagnostic = failure.diagnostic || null;
|
|
83
|
+
if (failure.classification?.retryable === false) error.permanent = true;
|
|
84
|
+
return error;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function receiptFields(error) {
|
|
88
|
+
if (!error?.provider) return {};
|
|
89
|
+
return {
|
|
90
|
+
provider: error.provider,
|
|
91
|
+
event: error.providerEvent,
|
|
92
|
+
category: error.providerCategory,
|
|
93
|
+
kind: error.classification?.kind,
|
|
94
|
+
retryable: error.classification?.retryable,
|
|
95
|
+
diagnostic: error.providerDiagnostic,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function workerFailure(error) {
|
|
100
|
+
const authenticationFailure =
|
|
101
|
+
error?.provider &&
|
|
102
|
+
error?.classification?.retryable === false &&
|
|
103
|
+
error?.providerCategory === 'authentication';
|
|
104
|
+
return authenticationFailure
|
|
105
|
+
? { code: 'refusal', reason: 'authentication_required' }
|
|
106
|
+
: { code: 'crash', reason: 'declared_failure' };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function publishCriticalFailure({
|
|
110
|
+
agent,
|
|
111
|
+
error,
|
|
112
|
+
attempts,
|
|
113
|
+
worker,
|
|
114
|
+
unsupportedCapability,
|
|
115
|
+
structuredOutputInvalid,
|
|
116
|
+
}) {
|
|
117
|
+
const specific =
|
|
118
|
+
error?.hookFailure ||
|
|
119
|
+
structuredOutputInvalid ||
|
|
120
|
+
unsupportedCapability ||
|
|
121
|
+
error?.vertexModelError ||
|
|
122
|
+
error?.terminationExhausted;
|
|
123
|
+
const critical =
|
|
124
|
+
agent.role === 'implementation' ||
|
|
125
|
+
agent.role === 'coordinator' ||
|
|
126
|
+
agent.id === 'consensus-coordinator';
|
|
127
|
+
if (!critical || specific) return worker;
|
|
128
|
+
|
|
129
|
+
agent._publish({
|
|
130
|
+
topic: 'CLUSTER_FAILED',
|
|
131
|
+
receiver: 'broadcast',
|
|
132
|
+
content: {
|
|
133
|
+
text: `Critical agent ${agent.id} exhausted its retry budget`,
|
|
134
|
+
data: {
|
|
135
|
+
reason: error?.provider ? 'provider_execution_failed' : 'critical_agent_exhausted',
|
|
136
|
+
agentId: agent.id,
|
|
137
|
+
role: agent.role,
|
|
138
|
+
attempts,
|
|
139
|
+
code: worker.code,
|
|
140
|
+
workerReason: worker.reason,
|
|
141
|
+
...receiptFields(error),
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
});
|
|
145
|
+
return worker;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function buildFinalFailureInfo({
|
|
149
|
+
agent,
|
|
150
|
+
error,
|
|
151
|
+
attempts,
|
|
152
|
+
worker,
|
|
153
|
+
unsupportedCapability,
|
|
154
|
+
structuredOutputInvalid,
|
|
155
|
+
}) {
|
|
156
|
+
return {
|
|
157
|
+
...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
|
|
158
|
+
agentId: agent.id,
|
|
159
|
+
taskId: error?.taskId || agent.currentTaskId,
|
|
160
|
+
iteration: agent.iteration,
|
|
161
|
+
error: error.message,
|
|
162
|
+
attempts,
|
|
163
|
+
...receiptFields(error),
|
|
164
|
+
...(error?.provider ? { code: worker.code, workerReason: worker.reason } : {}),
|
|
165
|
+
...(unsupportedCapability
|
|
166
|
+
? {
|
|
167
|
+
code: error.code,
|
|
168
|
+
permanent: true,
|
|
169
|
+
provider: error.provider,
|
|
170
|
+
capability: error.capability,
|
|
171
|
+
}
|
|
172
|
+
: {}),
|
|
173
|
+
...(structuredOutputInvalid ? { code: error.code, details: error.details ?? null } : {}),
|
|
174
|
+
timestamp: Date.now(),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
module.exports = {
|
|
179
|
+
buildFinalFailureInfo,
|
|
180
|
+
decorateError,
|
|
181
|
+
extractProviderFailure,
|
|
182
|
+
publishCriticalFailure,
|
|
183
|
+
receiptFields,
|
|
184
|
+
redactTerminalFailureForControlPlane,
|
|
185
|
+
workerFailure,
|
|
186
|
+
};
|
package/src/orchestrator.js
CHANGED
|
@@ -275,6 +275,7 @@ class Orchestrator {
|
|
|
275
275
|
// Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
|
|
276
276
|
this.closed = false;
|
|
277
277
|
this._conductorWatchdogs = new Set();
|
|
278
|
+
this._clusterRunBoundaries = new Map();
|
|
278
279
|
|
|
279
280
|
// Track if clusters are loaded (for lazy loading pattern)
|
|
280
281
|
this._clustersLoaded = options.skipLoad === true;
|
|
@@ -1575,11 +1576,17 @@ class Orchestrator {
|
|
|
1575
1576
|
}
|
|
1576
1577
|
|
|
1577
1578
|
_registerAgentErrorHandler(messageBus, clusterId) {
|
|
1579
|
+
this._recordClusterRunBoundary(messageBus, clusterId);
|
|
1578
1580
|
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
|
|
1579
1581
|
const agentRole = message.content?.data?.role;
|
|
1580
1582
|
const attempts = message.content?.data?.attempts || 1;
|
|
1581
1583
|
const hookFailure = message.content?.data?.hookFailure === true;
|
|
1582
1584
|
const restartExhausted = message.content?.data?.restartExhausted === true;
|
|
1585
|
+
const durableClusterFailure = this._findCurrentRunClusterFailure(
|
|
1586
|
+
messageBus,
|
|
1587
|
+
clusterId,
|
|
1588
|
+
message.sequence
|
|
1589
|
+
);
|
|
1583
1590
|
|
|
1584
1591
|
await this._saveClusters();
|
|
1585
1592
|
|
|
@@ -1587,7 +1594,10 @@ class Orchestrator {
|
|
|
1587
1594
|
agentRole === 'implementation' ||
|
|
1588
1595
|
agentRole === 'coordinator' ||
|
|
1589
1596
|
message.sender === 'consensus-coordinator';
|
|
1590
|
-
const shouldStop =
|
|
1597
|
+
const shouldStop =
|
|
1598
|
+
!durableClusterFailure &&
|
|
1599
|
+
shouldStopForRole &&
|
|
1600
|
+
(hookFailure || restartExhausted || attempts >= 3);
|
|
1591
1601
|
|
|
1592
1602
|
if (shouldStop) {
|
|
1593
1603
|
this._log(`\n${'='.repeat(80)}`);
|
|
@@ -1610,6 +1620,25 @@ class Orchestrator {
|
|
|
1610
1620
|
});
|
|
1611
1621
|
}
|
|
1612
1622
|
|
|
1623
|
+
_recordClusterRunBoundary(messageBus, clusterId) {
|
|
1624
|
+
const latest = messageBus.findLast({ cluster_id: clusterId, orderBySequence: true });
|
|
1625
|
+
this._clusterRunBoundaries.set(clusterId, latest?.sequence ?? null);
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
_findCurrentRunClusterFailure(messageBus, clusterId, throughId) {
|
|
1629
|
+
const boundary = this._clusterRunBoundaries.get(clusterId);
|
|
1630
|
+
return (
|
|
1631
|
+
messageBus.query({
|
|
1632
|
+
cluster_id: clusterId,
|
|
1633
|
+
topic: 'CLUSTER_FAILED',
|
|
1634
|
+
order: 'desc',
|
|
1635
|
+
limit: 1,
|
|
1636
|
+
...(boundary === null ? {} : { afterId: boundary }),
|
|
1637
|
+
...(throughId === undefined ? {} : { throughId }),
|
|
1638
|
+
})[0] || null
|
|
1639
|
+
);
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1613
1642
|
_registerPushBlockedHandler(messageBus, clusterId) {
|
|
1614
1643
|
this._subscribeToClusterTopic(messageBus, clusterId, 'PUSH_BLOCKED', async (message) => {
|
|
1615
1644
|
const reason = message.content?.data?.blocked_reason || 'unknown';
|
|
@@ -2425,6 +2454,7 @@ class Orchestrator {
|
|
|
2425
2454
|
|
|
2426
2455
|
// Now remove from memory after persisting
|
|
2427
2456
|
this.clusters.delete(clusterId);
|
|
2457
|
+
this._clusterRunBoundaries.delete(clusterId);
|
|
2428
2458
|
}
|
|
2429
2459
|
|
|
2430
2460
|
/**
|
|
@@ -2462,6 +2492,7 @@ class Orchestrator {
|
|
|
2462
2492
|
watchdog.dispose();
|
|
2463
2493
|
}
|
|
2464
2494
|
this._conductorWatchdogs.clear();
|
|
2495
|
+
this._clusterRunBoundaries.clear();
|
|
2465
2496
|
|
|
2466
2497
|
for (const cluster of this.clusters.values()) {
|
|
2467
2498
|
if (typeof cluster.snapshotter?.stop === 'function') {
|
|
@@ -2902,6 +2933,7 @@ class Orchestrator {
|
|
|
2902
2933
|
}
|
|
2903
2934
|
|
|
2904
2935
|
async _restartClusterAgents(cluster) {
|
|
2936
|
+
this._recordClusterRunBoundary(cluster.messageBus, cluster.id);
|
|
2905
2937
|
cluster.state = 'running';
|
|
2906
2938
|
cluster.pid = process.pid;
|
|
2907
2939
|
for (const agent of cluster.agents) {
|