@the-open-engine/zeroshot 6.6.0 → 6.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/zeroshot-cluster-worker.js +16 -0
- package/cli/commands/providers.js +4 -1
- package/cli/index.js +153 -24
- package/docs/openengine-cluster-protocol/v1/legacy-worker.md +117 -0
- package/lib/agent-cli-provider/adapters/claude.d.ts.map +1 -1
- package/lib/agent-cli-provider/adapters/claude.js +23 -10
- package/lib/agent-cli-provider/adapters/claude.js.map +1 -1
- package/lib/agent-cli-provider/adapters/codex.d.ts.map +1 -1
- package/lib/agent-cli-provider/adapters/codex.js +4 -0
- package/lib/agent-cli-provider/adapters/codex.js.map +1 -1
- package/lib/agent-cli-provider/adapters/common.d.ts.map +1 -1
- package/lib/agent-cli-provider/adapters/common.js +2 -1
- package/lib/agent-cli-provider/adapters/common.js.map +1 -1
- package/lib/agent-cli-provider/contract-options.d.ts.map +1 -1
- package/lib/agent-cli-provider/contract-options.js +2 -1
- package/lib/agent-cli-provider/contract-options.js.map +1 -1
- package/lib/agent-cli-provider/index.d.ts +1 -1
- package/lib/agent-cli-provider/index.d.ts.map +1 -1
- package/lib/agent-cli-provider/index.js.map +1 -1
- package/lib/agent-cli-provider/provider-registry.d.ts +1 -1
- package/lib/agent-cli-provider/provider-registry.js +1 -1
- package/lib/agent-cli-provider/provider-registry.js.map +1 -1
- package/lib/agent-cli-provider/single-agent-runtime.js +6 -2
- package/lib/agent-cli-provider/single-agent-runtime.js.map +1 -1
- package/lib/agent-cli-provider/types.d.ts +3 -1
- package/lib/agent-cli-provider/types.d.ts.map +1 -1
- package/lib/agent-cli-provider/types.js.map +1 -1
- package/lib/cluster-worker/contracts.js +194 -0
- package/lib/cluster-worker/engine-adapter.js +300 -0
- package/lib/cluster-worker/executable.js +229 -0
- package/lib/cluster-worker/index.d.ts +227 -0
- package/lib/cluster-worker/index.js +294 -0
- package/lib/cluster-worker/object-utils.js +17 -0
- package/lib/cluster-worker/process-stdio.js +37 -0
- package/lib/cluster-worker/profiles.js +81 -0
- package/lib/cluster-worker/runtime-engine.js +50 -0
- package/lib/cluster-worker/runtime-support.js +298 -0
- package/lib/cluster-worker/state-machine.js +147 -0
- package/lib/cluster-worker/terminal-normalizer.js +95 -0
- package/lib/cluster-worker/worker-internals.js +20 -0
- package/lib/detached-startup.js +127 -11
- package/lib/process-liveness.js +26 -0
- package/lib/settings.js +1 -0
- package/lib/start-cluster.js +93 -18
- package/package.json +8 -2
- package/protocol/openengine-cluster/v1/worker.schema.json +1174 -0
- package/scripts/assert-release-published.js +65 -11
- package/scripts/run-lint-staged-no-stash.js +68 -0
- package/src/agent/agent-lifecycle.js +368 -91
- package/src/agent/agent-task-executor.js +384 -101
- package/src/agent-cli-provider/adapters/claude.ts +29 -11
- package/src/agent-cli-provider/adapters/codex.ts +4 -0
- package/src/agent-cli-provider/adapters/common.ts +2 -1
- package/src/agent-cli-provider/contract-options.ts +2 -1
- package/src/agent-cli-provider/index.ts +1 -0
- package/src/agent-cli-provider/provider-registry.ts +1 -1
- package/src/agent-cli-provider/single-agent-runtime.ts +8 -2
- package/src/agent-cli-provider/types.ts +3 -1
- package/src/agent-wrapper.js +9 -2
- package/src/config-validator.js +10 -11
- package/src/orchestrator.js +482 -67
- package/task-lib/attachable-watcher.js +10 -1
- package/task-lib/commands/kill.js +34 -9
- package/task-lib/commands/status.js +13 -3
- package/task-lib/process-termination.js +202 -0
- package/task-lib/runner.js +8 -20
- package/task-lib/store.js +28 -6
- package/task-lib/watcher.js +14 -2
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* - Message handling and routing
|
|
8
8
|
* - Trigger action execution (execute_task, stop_cluster)
|
|
9
9
|
* - Task execution with retry logic
|
|
10
|
-
* -
|
|
10
|
+
* - Bounded task runtime and provider-output inactivity monitoring
|
|
11
11
|
*
|
|
12
12
|
* State machine: idle → evaluating → building_context → executing → idle
|
|
13
13
|
*/
|
|
@@ -17,11 +17,7 @@ const { executeHook } = require('./agent-hook-executor');
|
|
|
17
17
|
const IsolationManager = require('../isolation-manager');
|
|
18
18
|
const crypto = require('crypto');
|
|
19
19
|
const { bufferMessage, scheduleDrain, drainBufferedMessages } = require('../message-buffer');
|
|
20
|
-
const {
|
|
21
|
-
analyzeProcessHealth,
|
|
22
|
-
isPlatformSupported,
|
|
23
|
-
STUCK_THRESHOLD,
|
|
24
|
-
} = require('./agent-stuck-detector');
|
|
20
|
+
const { isPlatformSupported } = require('./agent-stuck-detector');
|
|
25
21
|
const { normalizeProviderName } = require('../../lib/provider-names');
|
|
26
22
|
const { loadSettings } = require('../../lib/settings');
|
|
27
23
|
const { findPlatformMismatchReason } = require('./validation-platform');
|
|
@@ -36,6 +32,7 @@ class HookExecutionError extends Error {
|
|
|
36
32
|
this.hookFailure = true;
|
|
37
33
|
this.hookRetries = options?.hookRetries;
|
|
38
34
|
this.originalHookError = options?.originalHookError;
|
|
35
|
+
this.taskId = options?.taskId || null;
|
|
39
36
|
}
|
|
40
37
|
}
|
|
41
38
|
|
|
@@ -217,21 +214,28 @@ async function stop(agent) {
|
|
|
217
214
|
|
|
218
215
|
// Kill current task if any
|
|
219
216
|
if (agent.currentTask) {
|
|
220
|
-
agent._killTask();
|
|
217
|
+
await agent._killTask('Task stopped by cluster shutdown');
|
|
221
218
|
}
|
|
222
219
|
|
|
223
220
|
// Wait for in-flight execution to complete (up to 5 seconds)
|
|
224
221
|
// This prevents write-after-close race conditions
|
|
225
222
|
if (agent._currentExecution) {
|
|
223
|
+
let executionTimeout = null;
|
|
226
224
|
try {
|
|
227
225
|
await Promise.race([
|
|
228
226
|
agent._currentExecution,
|
|
229
|
-
new Promise((resolve) =>
|
|
227
|
+
new Promise((resolve) => {
|
|
228
|
+
executionTimeout = setTimeout(resolve, 5000);
|
|
229
|
+
}),
|
|
230
230
|
]);
|
|
231
231
|
} catch {
|
|
232
232
|
// Ignore errors from cancelled execution
|
|
233
|
+
} finally {
|
|
234
|
+
if (executionTimeout) {
|
|
235
|
+
clearTimeout(executionTimeout);
|
|
236
|
+
}
|
|
237
|
+
agent._currentExecution = null;
|
|
233
238
|
}
|
|
234
|
-
agent._currentExecution = null;
|
|
235
239
|
}
|
|
236
240
|
|
|
237
241
|
agent._log(`Agent ${agent.id} stopped`);
|
|
@@ -427,7 +431,7 @@ function publishTaskStarted(agent, triggeringMessage) {
|
|
|
427
431
|
|
|
428
432
|
function attachResultMetadata(agent, result) {
|
|
429
433
|
// Add task ID to result for debugging and hooks
|
|
430
|
-
result.taskId = agent.currentTaskId;
|
|
434
|
+
result.taskId = result.taskId || agent.currentTaskId;
|
|
431
435
|
result.agentId = agent.id;
|
|
432
436
|
result.iteration = agent.iteration;
|
|
433
437
|
}
|
|
@@ -467,6 +471,15 @@ function publishTokenUsage(agent, result) {
|
|
|
467
471
|
});
|
|
468
472
|
}
|
|
469
473
|
|
|
474
|
+
function clearTransientTaskState(agent) {
|
|
475
|
+
stopLivenessCheck(agent);
|
|
476
|
+
agent.currentTask = null;
|
|
477
|
+
agent.currentTaskId = null;
|
|
478
|
+
agent.processPid = null;
|
|
479
|
+
agent.lastOutputTime = null;
|
|
480
|
+
agent.taskStartedAt = null;
|
|
481
|
+
}
|
|
482
|
+
|
|
470
483
|
async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
|
|
471
484
|
// Execute onComplete hook WITH RETRY
|
|
472
485
|
// Hook failure shouldn't retry the entire task - just the hook
|
|
@@ -510,7 +523,11 @@ ${'='.repeat(80)}`);
|
|
|
510
523
|
`Hook execution failed after ${hookMaxRetries} attempts. ` +
|
|
511
524
|
`Task completed successfully but hook could not publish result. ` +
|
|
512
525
|
`Original error: ${hookError.message}`,
|
|
513
|
-
{
|
|
526
|
+
{
|
|
527
|
+
hookRetries: hookMaxRetries,
|
|
528
|
+
originalHookError: hookError.message,
|
|
529
|
+
taskId: result?.taskId || null,
|
|
530
|
+
}
|
|
514
531
|
);
|
|
515
532
|
}
|
|
516
533
|
}
|
|
@@ -554,7 +571,10 @@ async function runTaskAttempt(agent, triggeringMessage) {
|
|
|
554
571
|
|
|
555
572
|
// Check if task execution failed
|
|
556
573
|
if (!result.success) {
|
|
557
|
-
|
|
574
|
+
const error = new Error(result.error || 'Task execution failed');
|
|
575
|
+
error.code = result.code || result.errorType || null;
|
|
576
|
+
error.taskId = result.taskId || null;
|
|
577
|
+
throw error;
|
|
558
578
|
}
|
|
559
579
|
|
|
560
580
|
const fallbackReason = await maybeRetryValidatorInDocker(agent, result);
|
|
@@ -573,6 +593,7 @@ async function runTaskAttempt(agent, triggeringMessage) {
|
|
|
573
593
|
|
|
574
594
|
publishTaskCompleted(agent, result);
|
|
575
595
|
publishTokenUsage(agent, result);
|
|
596
|
+
clearTransientTaskState(agent);
|
|
576
597
|
await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
|
|
577
598
|
}
|
|
578
599
|
|
|
@@ -595,11 +616,12 @@ async function handleLockContention() {
|
|
|
595
616
|
}
|
|
596
617
|
|
|
597
618
|
async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
|
|
619
|
+
const failureAttempts = error?.terminationAttempts ?? maxRetries;
|
|
598
620
|
console.error(`
|
|
599
621
|
${'='.repeat(80)}`);
|
|
600
622
|
console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
|
|
601
623
|
console.error(`${'='.repeat(80)}`);
|
|
602
|
-
console.error(`All ${
|
|
624
|
+
console.error(`All ${failureAttempts} attempts failed`);
|
|
603
625
|
console.error(`Final error: ${error.message}`);
|
|
604
626
|
console.error(`Stack: ${error.stack}`);
|
|
605
627
|
console.error(`${'='.repeat(80)}
|
|
@@ -613,7 +635,7 @@ ${'='.repeat(80)}`);
|
|
|
613
635
|
${'='.repeat(80)}`);
|
|
614
636
|
console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
|
|
615
637
|
console.error(`${'='.repeat(80)}`);
|
|
616
|
-
console.error(`Validator ${agent.id} crashed ${
|
|
638
|
+
console.error(`Validator ${agent.id} crashed ${failureAttempts} times`);
|
|
617
639
|
console.error(`Error: ${error.message}`);
|
|
618
640
|
console.error(`REJECTING validation - broken code will NOT be merged`);
|
|
619
641
|
console.error(`Investigation required before retry`);
|
|
@@ -627,16 +649,16 @@ ${'='.repeat(80)}`);
|
|
|
627
649
|
topic: hook.config.topic,
|
|
628
650
|
receiver: hook.config.receiver || 'broadcast',
|
|
629
651
|
content: {
|
|
630
|
-
text: `REJECTED: Validator crashed ${
|
|
652
|
+
text: `REJECTED: Validator crashed ${failureAttempts} times - ${error.message}`,
|
|
631
653
|
data: {
|
|
632
654
|
approved: false, // REJECT!
|
|
633
655
|
crashedAfterRetries: true,
|
|
634
656
|
errors: JSON.stringify([
|
|
635
|
-
`VALIDATOR CRASHED ${
|
|
657
|
+
`VALIDATOR CRASHED ${failureAttempts}x: ${error.message}`,
|
|
636
658
|
`Validation could not be performed - REJECTING to prevent broken code merge`,
|
|
637
659
|
`Investigation required before retry`,
|
|
638
660
|
]),
|
|
639
|
-
attempts:
|
|
661
|
+
attempts: failureAttempts,
|
|
640
662
|
requiresInvestigation: true,
|
|
641
663
|
},
|
|
642
664
|
},
|
|
@@ -670,13 +692,32 @@ ${'='.repeat(80)}`);
|
|
|
670
692
|
});
|
|
671
693
|
}
|
|
672
694
|
|
|
695
|
+
if (error?.terminationExhausted) {
|
|
696
|
+
agent._publish({
|
|
697
|
+
topic: 'CLUSTER_FAILED',
|
|
698
|
+
receiver: 'broadcast',
|
|
699
|
+
content: {
|
|
700
|
+
text: `Cluster failed: task termination could not be verified for ${agent.id}`,
|
|
701
|
+
data: {
|
|
702
|
+
reason: 'task_termination_unverified',
|
|
703
|
+
agentId: agent.id,
|
|
704
|
+
role: agent.role,
|
|
705
|
+
taskId: error.taskId || agent.currentTaskId,
|
|
706
|
+
attempts: failureAttempts,
|
|
707
|
+
error: error.message,
|
|
708
|
+
},
|
|
709
|
+
},
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
|
|
673
713
|
// Save failure info to cluster for resume capability
|
|
674
714
|
agent.cluster.failureInfo = {
|
|
715
|
+
...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
|
|
675
716
|
agentId: agent.id,
|
|
676
|
-
taskId: agent.currentTaskId,
|
|
717
|
+
taskId: error?.taskId || agent.currentTaskId,
|
|
677
718
|
iteration: agent.iteration,
|
|
678
719
|
error: error.message,
|
|
679
|
-
attempts:
|
|
720
|
+
attempts: failureAttempts,
|
|
680
721
|
timestamp: Date.now(),
|
|
681
722
|
};
|
|
682
723
|
|
|
@@ -685,18 +726,20 @@ ${'='.repeat(80)}`);
|
|
|
685
726
|
topic: 'AGENT_ERROR',
|
|
686
727
|
receiver: 'broadcast',
|
|
687
728
|
content: {
|
|
688
|
-
text: `Task execution failed after ${
|
|
729
|
+
text: `Task execution failed after ${failureAttempts} attempts: ${error.message}`,
|
|
689
730
|
data: {
|
|
690
731
|
error: error.message,
|
|
691
732
|
stack: error.stack,
|
|
692
733
|
hookFailure: error?.hookFailure === true,
|
|
734
|
+
restartExhausted: error?.restartExhausted === true,
|
|
735
|
+
terminationExhausted: error?.terminationExhausted === true,
|
|
693
736
|
hookRetries: error?.hookRetries ?? undefined,
|
|
694
737
|
originalHookError: error?.originalHookError ?? undefined,
|
|
695
738
|
agent: agent.id,
|
|
696
739
|
role: agent.role,
|
|
697
740
|
iteration: agent.iteration,
|
|
698
|
-
taskId: agent.currentTaskId,
|
|
699
|
-
attempts:
|
|
741
|
+
taskId: error?.taskId || agent.currentTaskId,
|
|
742
|
+
attempts: failureAttempts,
|
|
700
743
|
hookFailureContext: error.message.includes('Hook uses result')
|
|
701
744
|
? {
|
|
702
745
|
taskId: agent.currentTaskId || 'UNKNOWN',
|
|
@@ -723,7 +766,9 @@ ${'='.repeat(80)}`);
|
|
|
723
766
|
orchestrator: agent.orchestrator,
|
|
724
767
|
});
|
|
725
768
|
|
|
726
|
-
|
|
769
|
+
if (!error?.terminationExhausted) {
|
|
770
|
+
agent.state = 'idle';
|
|
771
|
+
}
|
|
727
772
|
}
|
|
728
773
|
|
|
729
774
|
async function scheduleRetry(agent, error, attempt, maxRetries, _baseDelay) {
|
|
@@ -805,6 +850,97 @@ function maybeExtendMaxRetries({
|
|
|
805
850
|
return { maxRetries, sigtermRetryGranted, noMessagesRetryGranted };
|
|
806
851
|
}
|
|
807
852
|
|
|
853
|
+
const RECOVERABLE_STUCK_TASK_CODES = new Set(['PROVIDER_INACTIVITY_TIMEOUT', 'AGENT_TASK_TIMEOUT']);
|
|
854
|
+
|
|
855
|
+
function readRestartHistory(agent) {
|
|
856
|
+
const lifecycle = agent.messageBus.query({
|
|
857
|
+
cluster_id: agent.cluster.id,
|
|
858
|
+
topic: 'AGENT_LIFECYCLE',
|
|
859
|
+
sender: agent.id,
|
|
860
|
+
});
|
|
861
|
+
let lastCompletedIndex = -1;
|
|
862
|
+
let totalRestarts = 0;
|
|
863
|
+
|
|
864
|
+
lifecycle.forEach((message, index) => {
|
|
865
|
+
const event = message.content?.data?.event;
|
|
866
|
+
if (event === 'TASK_COMPLETED') {
|
|
867
|
+
lastCompletedIndex = index;
|
|
868
|
+
} else if (event === 'AGENT_RESTART_ATTEMPT') {
|
|
869
|
+
totalRestarts += 1;
|
|
870
|
+
}
|
|
871
|
+
});
|
|
872
|
+
|
|
873
|
+
const restartsSinceCompletion = lifecycle
|
|
874
|
+
.slice(lastCompletedIndex + 1)
|
|
875
|
+
.filter((message) => message.content?.data?.event === 'AGENT_RESTART_ATTEMPT').length;
|
|
876
|
+
|
|
877
|
+
return { restartsSinceCompletion, totalRestarts };
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
function resolveRestartBudget(agent, settings) {
|
|
881
|
+
const history = readRestartHistory(agent);
|
|
882
|
+
const maxRestartAttempts = settings.maxRestartAttempts ?? 3;
|
|
883
|
+
const maxTotalRestarts = settings.maxTotalRestarts ?? 10;
|
|
884
|
+
const allowed =
|
|
885
|
+
history.restartsSinceCompletion < maxRestartAttempts &&
|
|
886
|
+
history.totalRestarts < maxTotalRestarts;
|
|
887
|
+
|
|
888
|
+
return {
|
|
889
|
+
...history,
|
|
890
|
+
maxRestartAttempts,
|
|
891
|
+
maxTotalRestarts,
|
|
892
|
+
allowed,
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
async function handleRecoverableStuckTaskFailure({
|
|
897
|
+
agent,
|
|
898
|
+
triggeringMessage,
|
|
899
|
+
error,
|
|
900
|
+
attempt,
|
|
901
|
+
maxRetries,
|
|
902
|
+
baseDelay,
|
|
903
|
+
settings,
|
|
904
|
+
}) {
|
|
905
|
+
if (!RECOVERABLE_STUCK_TASK_CODES.has(error.code)) {
|
|
906
|
+
return { handled: false, maxRetries };
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
logTaskAttemptFailure(agent, attempt, maxRetries, error);
|
|
910
|
+
const budget = resolveRestartBudget(agent, settings);
|
|
911
|
+
if (!budget.allowed) {
|
|
912
|
+
agent._publishLifecycle('AGENT_RESTART_EXHAUSTED', {
|
|
913
|
+
taskId: error.taskId,
|
|
914
|
+
reason: error.message,
|
|
915
|
+
code: error.code,
|
|
916
|
+
attempts: attempt,
|
|
917
|
+
restartsSinceCompletion: budget.restartsSinceCompletion,
|
|
918
|
+
totalRestarts: budget.totalRestarts,
|
|
919
|
+
maxRestartAttempts: budget.maxRestartAttempts,
|
|
920
|
+
maxTotalRestarts: budget.maxTotalRestarts,
|
|
921
|
+
});
|
|
922
|
+
error.restartExhausted = true;
|
|
923
|
+
await handleFinalFailure(agent, triggeringMessage, error, attempt);
|
|
924
|
+
return { handled: true, shouldStop: true, maxRetries };
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
const nextRestartAttempt = budget.restartsSinceCompletion + 1;
|
|
928
|
+
const nextTotalRestart = budget.totalRestarts + 1;
|
|
929
|
+
agent._publishLifecycle('AGENT_RESTART_ATTEMPT', {
|
|
930
|
+
taskId: error.taskId,
|
|
931
|
+
reason: error.message,
|
|
932
|
+
code: error.code,
|
|
933
|
+
attempt: nextRestartAttempt,
|
|
934
|
+
totalRestartAttempt: nextTotalRestart,
|
|
935
|
+
maxRestartAttempts: budget.maxRestartAttempts,
|
|
936
|
+
maxTotalRestarts: budget.maxTotalRestarts,
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
const extendedMaxRetries = Math.max(maxRetries, attempt + 1);
|
|
940
|
+
await scheduleRetry(agent, error, attempt, extendedMaxRetries, baseDelay);
|
|
941
|
+
return { handled: true, shouldStop: false, maxRetries: extendedMaxRetries };
|
|
942
|
+
}
|
|
943
|
+
|
|
808
944
|
/**
|
|
809
945
|
* Execute claude-zeroshots with built context
|
|
810
946
|
* Default: uses settings.maxRetries (default 3) for exponential backoff retries.
|
|
@@ -849,6 +985,30 @@ async function executeTask(agent, triggeringMessage) {
|
|
|
849
985
|
await handleFinalFailure(agent, triggeringMessage, error, 1);
|
|
850
986
|
return;
|
|
851
987
|
}
|
|
988
|
+
agent._publishLifecycle('TASK_FAILED', {
|
|
989
|
+
iteration: agent.iteration,
|
|
990
|
+
taskId: error.taskId || agent.currentTaskId,
|
|
991
|
+
error: error.message,
|
|
992
|
+
code: error.code || null,
|
|
993
|
+
attempt,
|
|
994
|
+
});
|
|
995
|
+
clearTransientTaskState(agent);
|
|
996
|
+
const stuckTaskResult = await handleRecoverableStuckTaskFailure({
|
|
997
|
+
agent,
|
|
998
|
+
triggeringMessage,
|
|
999
|
+
error,
|
|
1000
|
+
attempt,
|
|
1001
|
+
maxRetries,
|
|
1002
|
+
baseDelay,
|
|
1003
|
+
settings,
|
|
1004
|
+
});
|
|
1005
|
+
if (stuckTaskResult.handled) {
|
|
1006
|
+
maxRetries = stuckTaskResult.maxRetries;
|
|
1007
|
+
if (stuckTaskResult.shouldStop) {
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
continue;
|
|
1011
|
+
}
|
|
852
1012
|
const updated = maybeExtendMaxRetries({
|
|
853
1013
|
error,
|
|
854
1014
|
attempt,
|
|
@@ -878,89 +1038,201 @@ function startLivenessCheck(agent) {
|
|
|
878
1038
|
clearInterval(agent.livenessCheckInterval);
|
|
879
1039
|
}
|
|
880
1040
|
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
1041
|
+
const settings = loadSettings();
|
|
1042
|
+
const warningsBeforeKill = Math.max(1, settings.staleWarningsBeforeKill ?? 2);
|
|
1043
|
+
const staleDuration = Math.max(1, agent.staleDuration);
|
|
1044
|
+
const configuredTimeout = agent.timeout > 0 ? agent.timeout : null;
|
|
1045
|
+
const shortestLimit = configuredTimeout
|
|
1046
|
+
? Math.min(staleDuration, configuredTimeout)
|
|
1047
|
+
: staleDuration;
|
|
1048
|
+
const checkIntervalMs = Math.min(60 * 1000, Math.max(10, Math.floor(shortestLimit / 4)));
|
|
1049
|
+
|
|
1050
|
+
agent.consecutiveStaleWarnings = 0;
|
|
1051
|
+
agent.livenessTerminationStarted = false;
|
|
1052
|
+
agent.livenessTerminationContext = null;
|
|
1053
|
+
agent.livenessTerminationAttempts = 0;
|
|
1054
|
+
agent.livenessTerminationRetryAt = 0;
|
|
1055
|
+
|
|
1056
|
+
agent.livenessCheckInterval = setInterval(() => {
|
|
1057
|
+
const hasRecoverableTask =
|
|
1058
|
+
Boolean(agent.currentTask) || Boolean(agent.isolation?.enabled && agent.currentTaskId);
|
|
1059
|
+
if (!hasRecoverableTask || agent.livenessTerminationStarted) {
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
const now = Date.now();
|
|
1064
|
+
if (agent.livenessTerminationContext) {
|
|
1065
|
+
if (now >= agent.livenessTerminationRetryAt) {
|
|
1066
|
+
attemptLivenessTermination(agent, settings);
|
|
1067
|
+
}
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
888
1070
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1071
|
+
const taskStartedAt = agent.taskStartedAt || agent.lastOutputTime || now;
|
|
1072
|
+
const lastOutputTime = agent.lastOutputTime || taskStartedAt;
|
|
1073
|
+
const taskRuntime = now - taskStartedAt;
|
|
1074
|
+
const timeSinceLastOutput = now - lastOutputTime;
|
|
892
1075
|
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1076
|
+
if (configuredTimeout && taskRuntime >= configuredTimeout) {
|
|
1077
|
+
const reason = `Task timed out after ${configuredTimeout}ms`;
|
|
1078
|
+
beginLivenessTermination(agent, settings, reason, 'AGENT_TASK_TIMEOUT', {
|
|
1079
|
+
taskId: agent.currentTaskId,
|
|
1080
|
+
taskRuntime,
|
|
1081
|
+
timeout: configuredTimeout,
|
|
1082
|
+
});
|
|
896
1083
|
return;
|
|
897
1084
|
}
|
|
898
1085
|
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
if (timeSinceLastOutput < agent.staleDuration) {
|
|
903
|
-
return; // Output is recent, definitely not stuck
|
|
904
|
-
}
|
|
1086
|
+
if (timeSinceLastOutput < staleDuration) {
|
|
1087
|
+
agent.consecutiveStaleWarnings = 0;
|
|
1088
|
+
return;
|
|
905
1089
|
}
|
|
906
1090
|
|
|
907
|
-
|
|
908
|
-
agent.
|
|
909
|
-
|
|
910
|
-
|
|
1091
|
+
agent.consecutiveStaleWarnings += 1;
|
|
1092
|
+
agent._publishLifecycle('AGENT_STALE_WARNING', {
|
|
1093
|
+
taskId: agent.currentTaskId,
|
|
1094
|
+
timeSinceLastOutput,
|
|
1095
|
+
staleDuration,
|
|
1096
|
+
lastOutputTime,
|
|
1097
|
+
consecutiveWarnings: agent.consecutiveStaleWarnings,
|
|
1098
|
+
warningsBeforeKill,
|
|
1099
|
+
processDiagnosticsAvailable: isPlatformSupported(),
|
|
1100
|
+
analysis: `Provider produced no output for ${timeSinceLastOutput}ms`,
|
|
1101
|
+
});
|
|
911
1102
|
|
|
912
|
-
|
|
913
|
-
|
|
1103
|
+
if (agent.consecutiveStaleWarnings < warningsBeforeKill) {
|
|
1104
|
+
return;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
const reason = `Provider produced no output for ${timeSinceLastOutput}ms`;
|
|
1108
|
+
beginLivenessTermination(agent, settings, reason, 'PROVIDER_INACTIVITY_TIMEOUT', {
|
|
1109
|
+
taskId: agent.currentTaskId,
|
|
1110
|
+
timeSinceLastOutput,
|
|
1111
|
+
staleDuration,
|
|
1112
|
+
consecutiveWarnings: agent.consecutiveStaleWarnings,
|
|
1113
|
+
});
|
|
1114
|
+
}, checkIntervalMs);
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
const MAX_LIVENESS_TERMINATION_ATTEMPTS = 3;
|
|
1118
|
+
|
|
1119
|
+
function beginLivenessTermination(agent, settings, reason, code, eventData) {
|
|
1120
|
+
agent.livenessTerminationContext = {
|
|
1121
|
+
taskId: agent.currentTaskId,
|
|
1122
|
+
reason,
|
|
1123
|
+
code,
|
|
1124
|
+
eventData,
|
|
1125
|
+
eventPublished: false,
|
|
1126
|
+
};
|
|
1127
|
+
agent.livenessTerminationAttempts = 0;
|
|
1128
|
+
agent.livenessTerminationRetryAt = 0;
|
|
1129
|
+
attemptLivenessTermination(agent, settings);
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
function publishLivenessTerminationEvent(agent, context) {
|
|
1133
|
+
if (context.eventPublished) return;
|
|
1134
|
+
context.eventPublished = true;
|
|
1135
|
+
agent._publishLifecycle(
|
|
1136
|
+
context.code === 'AGENT_TASK_TIMEOUT' ? 'AGENT_TASK_TIMEOUT' : 'AGENT_INACTIVITY_TIMEOUT',
|
|
1137
|
+
context.eventData
|
|
1138
|
+
);
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
function terminationRetryDelay(settings, attempt) {
|
|
1142
|
+
const baseDelay = Math.max(0, settings.backoffBaseMs ?? 2000);
|
|
1143
|
+
const maxDelay = Math.max(0, settings.backoffMaxMs ?? 30000);
|
|
1144
|
+
return Math.min(maxDelay, baseDelay * Math.pow(2, Math.max(0, attempt - 1)));
|
|
1145
|
+
}
|
|
914
1146
|
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
1147
|
+
function attemptLivenessTermination(agent, settings) {
|
|
1148
|
+
const context = agent.livenessTerminationContext;
|
|
1149
|
+
if (!context || agent.livenessTerminationStarted) return;
|
|
1150
|
+
|
|
1151
|
+
agent.livenessTerminationStarted = true;
|
|
1152
|
+
agent.livenessTerminationAttempts += 1;
|
|
1153
|
+
const attempt = agent.livenessTerminationAttempts;
|
|
1154
|
+
|
|
1155
|
+
Promise.resolve()
|
|
1156
|
+
.then(() => agent._killTask({ reason: context.reason, code: context.code }))
|
|
1157
|
+
.then((termination) => {
|
|
1158
|
+
if (termination?.forced === false) return;
|
|
1159
|
+
publishLivenessTerminationEvent(agent, context);
|
|
1160
|
+
})
|
|
1161
|
+
.catch((error) => {
|
|
1162
|
+
if (agent.livenessTerminationContext !== context) return;
|
|
1163
|
+
if (attempt >= MAX_LIVENESS_TERMINATION_ATTEMPTS) {
|
|
1164
|
+
exhaustLivenessTermination(agent, context, error);
|
|
918
1165
|
return;
|
|
919
1166
|
}
|
|
920
1167
|
|
|
921
|
-
|
|
1168
|
+
const delayMs = terminationRetryDelay(settings, attempt);
|
|
1169
|
+
agent.livenessTerminationRetryAt = Date.now() + delayMs;
|
|
1170
|
+
agent.livenessTerminationStarted = false;
|
|
1171
|
+
agent._publishLifecycle('AGENT_TERMINATION_RETRY', {
|
|
1172
|
+
taskId: context.taskId,
|
|
1173
|
+
reason: context.reason,
|
|
1174
|
+
code: context.code,
|
|
1175
|
+
error: error.message,
|
|
1176
|
+
attempt,
|
|
1177
|
+
nextAttempt: attempt + 1,
|
|
1178
|
+
maxAttempts: MAX_LIVENESS_TERMINATION_ATTEMPTS,
|
|
1179
|
+
delayMs,
|
|
1180
|
+
});
|
|
922
1181
|
agent._log(
|
|
923
|
-
`[${agent.id}]
|
|
924
|
-
`
|
|
925
|
-
`CPU=${analysis.cpuPercent}%, ctxSwitches=${analysis.ctxSwitchesDelta}`
|
|
1182
|
+
`[${agent.id}] Failed to terminate task (attempt ${attempt}/${MAX_LIVENESS_TERMINATION_ATTEMPTS}); ` +
|
|
1183
|
+
`retrying in ${delayMs}ms: ${error.message}`
|
|
926
1184
|
);
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
927
1187
|
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1188
|
+
function exhaustLivenessTermination(agent, context, terminationError) {
|
|
1189
|
+
if (agent.livenessCheckInterval) {
|
|
1190
|
+
clearInterval(agent.livenessCheckInterval);
|
|
1191
|
+
agent.livenessCheckInterval = null;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
const attempts = agent.livenessTerminationAttempts;
|
|
1195
|
+
publishLivenessTerminationEvent(agent, context);
|
|
1196
|
+
const error = new Error(
|
|
1197
|
+
`Failed to terminate isolated task ${context.taskId} after ${attempts} attempts; ` +
|
|
1198
|
+
`the provider task may still be running. Manual recovery is required before resume. ` +
|
|
1199
|
+
`Last error: ${terminationError.message}`
|
|
1200
|
+
);
|
|
1201
|
+
error.code = 'ISOLATED_TASK_TERMINATION_EXHAUSTED';
|
|
1202
|
+
error.taskId = context.taskId;
|
|
1203
|
+
error.permanent = true;
|
|
1204
|
+
error.restartExhausted = true;
|
|
1205
|
+
error.terminationExhausted = true;
|
|
1206
|
+
error.terminationAttempts = attempts;
|
|
1207
|
+
|
|
1208
|
+
agent.state = 'error';
|
|
1209
|
+
agent.cluster.failureInfo = {
|
|
1210
|
+
agentId: agent.id,
|
|
1211
|
+
taskId: context.taskId,
|
|
1212
|
+
iteration: agent.iteration,
|
|
1213
|
+
type: 'task_termination',
|
|
1214
|
+
reason: 'termination_unverified',
|
|
1215
|
+
error: error.message,
|
|
1216
|
+
attempts,
|
|
1217
|
+
timestamp: Date.now(),
|
|
1218
|
+
};
|
|
1219
|
+
agent._publishLifecycle('AGENT_TERMINATION_EXHAUSTED', {
|
|
1220
|
+
taskId: context.taskId,
|
|
1221
|
+
reason: context.reason,
|
|
1222
|
+
code: error.code,
|
|
1223
|
+
terminationCode: context.code,
|
|
1224
|
+
attempts,
|
|
1225
|
+
maxAttempts: MAX_LIVENESS_TERMINATION_ATTEMPTS,
|
|
1226
|
+
error: terminationError.message,
|
|
1227
|
+
requiresManualRecovery: true,
|
|
1228
|
+
});
|
|
1229
|
+
|
|
1230
|
+
if (typeof agent.currentTask?.failClosed === 'function') {
|
|
1231
|
+
agent.currentTask.failClosed(error);
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
agent._log(`[${agent.id}] ${error.message}`);
|
|
964
1236
|
}
|
|
965
1237
|
|
|
966
1238
|
/**
|
|
@@ -972,6 +1244,11 @@ function stopLivenessCheck(agent) {
|
|
|
972
1244
|
clearInterval(agent.livenessCheckInterval);
|
|
973
1245
|
agent.livenessCheckInterval = null;
|
|
974
1246
|
}
|
|
1247
|
+
agent.consecutiveStaleWarnings = 0;
|
|
1248
|
+
agent.livenessTerminationStarted = false;
|
|
1249
|
+
agent.livenessTerminationContext = null;
|
|
1250
|
+
agent.livenessTerminationAttempts = 0;
|
|
1251
|
+
agent.livenessTerminationRetryAt = 0;
|
|
975
1252
|
}
|
|
976
1253
|
|
|
977
1254
|
module.exports = {
|