@the-open-engine/zeroshot 6.6.0 → 6.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/bin/zeroshot-cluster-worker.js +16 -0
  2. package/cli/commands/providers.js +4 -1
  3. package/cli/index.js +153 -24
  4. package/docs/openengine-cluster-protocol/v1/legacy-worker.md +117 -0
  5. package/lib/agent-cli-provider/adapters/claude.d.ts.map +1 -1
  6. package/lib/agent-cli-provider/adapters/claude.js +23 -10
  7. package/lib/agent-cli-provider/adapters/claude.js.map +1 -1
  8. package/lib/agent-cli-provider/adapters/codex.d.ts.map +1 -1
  9. package/lib/agent-cli-provider/adapters/codex.js +4 -0
  10. package/lib/agent-cli-provider/adapters/codex.js.map +1 -1
  11. package/lib/agent-cli-provider/adapters/common.d.ts.map +1 -1
  12. package/lib/agent-cli-provider/adapters/common.js +2 -1
  13. package/lib/agent-cli-provider/adapters/common.js.map +1 -1
  14. package/lib/agent-cli-provider/contract-options.d.ts.map +1 -1
  15. package/lib/agent-cli-provider/contract-options.js +2 -1
  16. package/lib/agent-cli-provider/contract-options.js.map +1 -1
  17. package/lib/agent-cli-provider/index.d.ts +1 -1
  18. package/lib/agent-cli-provider/index.d.ts.map +1 -1
  19. package/lib/agent-cli-provider/index.js.map +1 -1
  20. package/lib/agent-cli-provider/provider-registry.d.ts +1 -1
  21. package/lib/agent-cli-provider/provider-registry.js +1 -1
  22. package/lib/agent-cli-provider/provider-registry.js.map +1 -1
  23. package/lib/agent-cli-provider/single-agent-runtime.js +6 -2
  24. package/lib/agent-cli-provider/single-agent-runtime.js.map +1 -1
  25. package/lib/agent-cli-provider/types.d.ts +3 -1
  26. package/lib/agent-cli-provider/types.d.ts.map +1 -1
  27. package/lib/agent-cli-provider/types.js.map +1 -1
  28. package/lib/cluster-worker/contracts.js +194 -0
  29. package/lib/cluster-worker/engine-adapter.js +300 -0
  30. package/lib/cluster-worker/executable.js +229 -0
  31. package/lib/cluster-worker/index.d.ts +227 -0
  32. package/lib/cluster-worker/index.js +294 -0
  33. package/lib/cluster-worker/object-utils.js +17 -0
  34. package/lib/cluster-worker/process-stdio.js +37 -0
  35. package/lib/cluster-worker/profiles.js +81 -0
  36. package/lib/cluster-worker/runtime-engine.js +50 -0
  37. package/lib/cluster-worker/runtime-support.js +298 -0
  38. package/lib/cluster-worker/state-machine.js +147 -0
  39. package/lib/cluster-worker/terminal-normalizer.js +95 -0
  40. package/lib/cluster-worker/worker-internals.js +20 -0
  41. package/lib/detached-startup.js +127 -11
  42. package/lib/process-liveness.js +26 -0
  43. package/lib/settings.js +1 -0
  44. package/lib/start-cluster.js +93 -18
  45. package/package.json +8 -2
  46. package/protocol/openengine-cluster/v1/worker.schema.json +1174 -0
  47. package/scripts/assert-release-published.js +65 -11
  48. package/scripts/run-lint-staged-no-stash.js +68 -0
  49. package/src/agent/agent-lifecycle.js +368 -91
  50. package/src/agent/agent-task-executor.js +384 -101
  51. package/src/agent-cli-provider/adapters/claude.ts +29 -11
  52. package/src/agent-cli-provider/adapters/codex.ts +4 -0
  53. package/src/agent-cli-provider/adapters/common.ts +2 -1
  54. package/src/agent-cli-provider/contract-options.ts +2 -1
  55. package/src/agent-cli-provider/index.ts +1 -0
  56. package/src/agent-cli-provider/provider-registry.ts +1 -1
  57. package/src/agent-cli-provider/single-agent-runtime.ts +8 -2
  58. package/src/agent-cli-provider/types.ts +3 -1
  59. package/src/agent-wrapper.js +9 -2
  60. package/src/config-validator.js +10 -11
  61. package/src/orchestrator.js +482 -67
  62. package/task-lib/attachable-watcher.js +10 -1
  63. package/task-lib/commands/kill.js +34 -9
  64. package/task-lib/commands/status.js +13 -3
  65. package/task-lib/process-termination.js +202 -0
  66. package/task-lib/runner.js +8 -20
  67. package/task-lib/store.js +28 -6
  68. package/task-lib/watcher.js +14 -2
@@ -7,7 +7,7 @@
7
7
  * - Message handling and routing
8
8
  * - Trigger action execution (execute_task, stop_cluster)
9
9
  * - Task execution with retry logic
10
- * - Liveness monitoring with multi-indicator stuck detection
10
+ * - Bounded task runtime and provider-output inactivity monitoring
11
11
  *
12
12
  * State machine: idle → evaluating → building_context → executing → idle
13
13
  */
@@ -17,11 +17,7 @@ const { executeHook } = require('./agent-hook-executor');
17
17
  const IsolationManager = require('../isolation-manager');
18
18
  const crypto = require('crypto');
19
19
  const { bufferMessage, scheduleDrain, drainBufferedMessages } = require('../message-buffer');
20
- const {
21
- analyzeProcessHealth,
22
- isPlatformSupported,
23
- STUCK_THRESHOLD,
24
- } = require('./agent-stuck-detector');
20
+ const { isPlatformSupported } = require('./agent-stuck-detector');
25
21
  const { normalizeProviderName } = require('../../lib/provider-names');
26
22
  const { loadSettings } = require('../../lib/settings');
27
23
  const { findPlatformMismatchReason } = require('./validation-platform');
@@ -36,6 +32,7 @@ class HookExecutionError extends Error {
36
32
  this.hookFailure = true;
37
33
  this.hookRetries = options?.hookRetries;
38
34
  this.originalHookError = options?.originalHookError;
35
+ this.taskId = options?.taskId || null;
39
36
  }
40
37
  }
41
38
 
@@ -217,21 +214,28 @@ async function stop(agent) {
217
214
 
218
215
  // Kill current task if any
219
216
  if (agent.currentTask) {
220
- agent._killTask();
217
+ await agent._killTask('Task stopped by cluster shutdown');
221
218
  }
222
219
 
223
220
  // Wait for in-flight execution to complete (up to 5 seconds)
224
221
  // This prevents write-after-close race conditions
225
222
  if (agent._currentExecution) {
223
+ let executionTimeout = null;
226
224
  try {
227
225
  await Promise.race([
228
226
  agent._currentExecution,
229
- new Promise((resolve) => setTimeout(resolve, 5000)),
227
+ new Promise((resolve) => {
228
+ executionTimeout = setTimeout(resolve, 5000);
229
+ }),
230
230
  ]);
231
231
  } catch {
232
232
  // Ignore errors from cancelled execution
233
+ } finally {
234
+ if (executionTimeout) {
235
+ clearTimeout(executionTimeout);
236
+ }
237
+ agent._currentExecution = null;
233
238
  }
234
- agent._currentExecution = null;
235
239
  }
236
240
 
237
241
  agent._log(`Agent ${agent.id} stopped`);
@@ -427,7 +431,7 @@ function publishTaskStarted(agent, triggeringMessage) {
427
431
 
428
432
  function attachResultMetadata(agent, result) {
429
433
  // Add task ID to result for debugging and hooks
430
- result.taskId = agent.currentTaskId;
434
+ result.taskId = result.taskId || agent.currentTaskId;
431
435
  result.agentId = agent.id;
432
436
  result.iteration = agent.iteration;
433
437
  }
@@ -467,6 +471,15 @@ function publishTokenUsage(agent, result) {
467
471
  });
468
472
  }
469
473
 
474
+ function clearTransientTaskState(agent) {
475
+ stopLivenessCheck(agent);
476
+ agent.currentTask = null;
477
+ agent.currentTaskId = null;
478
+ agent.processPid = null;
479
+ agent.lastOutputTime = null;
480
+ agent.taskStartedAt = null;
481
+ }
482
+
470
483
  async function executeOnCompleteHookWithRetry(agent, triggeringMessage, result) {
471
484
  // Execute onComplete hook WITH RETRY
472
485
  // Hook failure shouldn't retry the entire task - just the hook
@@ -510,7 +523,11 @@ ${'='.repeat(80)}`);
510
523
  `Hook execution failed after ${hookMaxRetries} attempts. ` +
511
524
  `Task completed successfully but hook could not publish result. ` +
512
525
  `Original error: ${hookError.message}`,
513
- { hookRetries: hookMaxRetries, originalHookError: hookError.message }
526
+ {
527
+ hookRetries: hookMaxRetries,
528
+ originalHookError: hookError.message,
529
+ taskId: result?.taskId || null,
530
+ }
514
531
  );
515
532
  }
516
533
  }
@@ -554,7 +571,10 @@ async function runTaskAttempt(agent, triggeringMessage) {
554
571
 
555
572
  // Check if task execution failed
556
573
  if (!result.success) {
557
- throw new Error(result.error || 'Task execution failed');
574
+ const error = new Error(result.error || 'Task execution failed');
575
+ error.code = result.code || result.errorType || null;
576
+ error.taskId = result.taskId || null;
577
+ throw error;
558
578
  }
559
579
 
560
580
  const fallbackReason = await maybeRetryValidatorInDocker(agent, result);
@@ -573,6 +593,7 @@ async function runTaskAttempt(agent, triggeringMessage) {
573
593
 
574
594
  publishTaskCompleted(agent, result);
575
595
  publishTokenUsage(agent, result);
596
+ clearTransientTaskState(agent);
576
597
  await executeOnCompleteHookWithRetry(agent, triggeringMessage, result);
577
598
  }
578
599
 
@@ -595,11 +616,12 @@ async function handleLockContention() {
595
616
  }
596
617
 
597
618
  async function handleFinalFailure(agent, triggeringMessage, error, maxRetries) {
619
+ const failureAttempts = error?.terminationAttempts ?? maxRetries;
598
620
  console.error(`
599
621
  ${'='.repeat(80)}`);
600
622
  console.error(`🔴🔴🔴 MAX RETRIES EXHAUSTED - AGENT: ${agent.id} 🔴🔴🔴`);
601
623
  console.error(`${'='.repeat(80)}`);
602
- console.error(`All ${maxRetries} attempts failed`);
624
+ console.error(`All ${failureAttempts} attempts failed`);
603
625
  console.error(`Final error: ${error.message}`);
604
626
  console.error(`Stack: ${error.stack}`);
605
627
  console.error(`${'='.repeat(80)}
@@ -613,7 +635,7 @@ ${'='.repeat(80)}`);
613
635
  ${'='.repeat(80)}`);
614
636
  console.error(`❌ VALIDATOR CRASHED - REJECTING (NOT AUTO-APPROVING)`);
615
637
  console.error(`${'='.repeat(80)}`);
616
- console.error(`Validator ${agent.id} crashed ${maxRetries} times`);
638
+ console.error(`Validator ${agent.id} crashed ${failureAttempts} times`);
617
639
  console.error(`Error: ${error.message}`);
618
640
  console.error(`REJECTING validation - broken code will NOT be merged`);
619
641
  console.error(`Investigation required before retry`);
@@ -627,16 +649,16 @@ ${'='.repeat(80)}`);
627
649
  topic: hook.config.topic,
628
650
  receiver: hook.config.receiver || 'broadcast',
629
651
  content: {
630
- text: `REJECTED: Validator crashed ${maxRetries} times - ${error.message}`,
652
+ text: `REJECTED: Validator crashed ${failureAttempts} times - ${error.message}`,
631
653
  data: {
632
654
  approved: false, // REJECT!
633
655
  crashedAfterRetries: true,
634
656
  errors: JSON.stringify([
635
- `VALIDATOR CRASHED ${maxRetries}x: ${error.message}`,
657
+ `VALIDATOR CRASHED ${failureAttempts}x: ${error.message}`,
636
658
  `Validation could not be performed - REJECTING to prevent broken code merge`,
637
659
  `Investigation required before retry`,
638
660
  ]),
639
- attempts: maxRetries,
661
+ attempts: failureAttempts,
640
662
  requiresInvestigation: true,
641
663
  },
642
664
  },
@@ -670,13 +692,32 @@ ${'='.repeat(80)}`);
670
692
  });
671
693
  }
672
694
 
695
+ if (error?.terminationExhausted) {
696
+ agent._publish({
697
+ topic: 'CLUSTER_FAILED',
698
+ receiver: 'broadcast',
699
+ content: {
700
+ text: `Cluster failed: task termination could not be verified for ${agent.id}`,
701
+ data: {
702
+ reason: 'task_termination_unverified',
703
+ agentId: agent.id,
704
+ role: agent.role,
705
+ taskId: error.taskId || agent.currentTaskId,
706
+ attempts: failureAttempts,
707
+ error: error.message,
708
+ },
709
+ },
710
+ });
711
+ }
712
+
673
713
  // Save failure info to cluster for resume capability
674
714
  agent.cluster.failureInfo = {
715
+ ...(error?.terminationExhausted ? agent.cluster.failureInfo : {}),
675
716
  agentId: agent.id,
676
- taskId: agent.currentTaskId,
717
+ taskId: error?.taskId || agent.currentTaskId,
677
718
  iteration: agent.iteration,
678
719
  error: error.message,
679
- attempts: maxRetries,
720
+ attempts: failureAttempts,
680
721
  timestamp: Date.now(),
681
722
  };
682
723
 
@@ -685,18 +726,20 @@ ${'='.repeat(80)}`);
685
726
  topic: 'AGENT_ERROR',
686
727
  receiver: 'broadcast',
687
728
  content: {
688
- text: `Task execution failed after ${maxRetries} attempts: ${error.message}`,
729
+ text: `Task execution failed after ${failureAttempts} attempts: ${error.message}`,
689
730
  data: {
690
731
  error: error.message,
691
732
  stack: error.stack,
692
733
  hookFailure: error?.hookFailure === true,
734
+ restartExhausted: error?.restartExhausted === true,
735
+ terminationExhausted: error?.terminationExhausted === true,
693
736
  hookRetries: error?.hookRetries ?? undefined,
694
737
  originalHookError: error?.originalHookError ?? undefined,
695
738
  agent: agent.id,
696
739
  role: agent.role,
697
740
  iteration: agent.iteration,
698
- taskId: agent.currentTaskId,
699
- attempts: maxRetries,
741
+ taskId: error?.taskId || agent.currentTaskId,
742
+ attempts: failureAttempts,
700
743
  hookFailureContext: error.message.includes('Hook uses result')
701
744
  ? {
702
745
  taskId: agent.currentTaskId || 'UNKNOWN',
@@ -723,7 +766,9 @@ ${'='.repeat(80)}`);
723
766
  orchestrator: agent.orchestrator,
724
767
  });
725
768
 
726
- agent.state = 'idle';
769
+ if (!error?.terminationExhausted) {
770
+ agent.state = 'idle';
771
+ }
727
772
  }
728
773
 
729
774
  async function scheduleRetry(agent, error, attempt, maxRetries, _baseDelay) {
@@ -805,6 +850,97 @@ function maybeExtendMaxRetries({
805
850
  return { maxRetries, sigtermRetryGranted, noMessagesRetryGranted };
806
851
  }
807
852
 
853
+ const RECOVERABLE_STUCK_TASK_CODES = new Set(['PROVIDER_INACTIVITY_TIMEOUT', 'AGENT_TASK_TIMEOUT']);
854
+
855
+ function readRestartHistory(agent) {
856
+ const lifecycle = agent.messageBus.query({
857
+ cluster_id: agent.cluster.id,
858
+ topic: 'AGENT_LIFECYCLE',
859
+ sender: agent.id,
860
+ });
861
+ let lastCompletedIndex = -1;
862
+ let totalRestarts = 0;
863
+
864
+ lifecycle.forEach((message, index) => {
865
+ const event = message.content?.data?.event;
866
+ if (event === 'TASK_COMPLETED') {
867
+ lastCompletedIndex = index;
868
+ } else if (event === 'AGENT_RESTART_ATTEMPT') {
869
+ totalRestarts += 1;
870
+ }
871
+ });
872
+
873
+ const restartsSinceCompletion = lifecycle
874
+ .slice(lastCompletedIndex + 1)
875
+ .filter((message) => message.content?.data?.event === 'AGENT_RESTART_ATTEMPT').length;
876
+
877
+ return { restartsSinceCompletion, totalRestarts };
878
+ }
879
+
880
+ function resolveRestartBudget(agent, settings) {
881
+ const history = readRestartHistory(agent);
882
+ const maxRestartAttempts = settings.maxRestartAttempts ?? 3;
883
+ const maxTotalRestarts = settings.maxTotalRestarts ?? 10;
884
+ const allowed =
885
+ history.restartsSinceCompletion < maxRestartAttempts &&
886
+ history.totalRestarts < maxTotalRestarts;
887
+
888
+ return {
889
+ ...history,
890
+ maxRestartAttempts,
891
+ maxTotalRestarts,
892
+ allowed,
893
+ };
894
+ }
895
+
896
+ async function handleRecoverableStuckTaskFailure({
897
+ agent,
898
+ triggeringMessage,
899
+ error,
900
+ attempt,
901
+ maxRetries,
902
+ baseDelay,
903
+ settings,
904
+ }) {
905
+ if (!RECOVERABLE_STUCK_TASK_CODES.has(error.code)) {
906
+ return { handled: false, maxRetries };
907
+ }
908
+
909
+ logTaskAttemptFailure(agent, attempt, maxRetries, error);
910
+ const budget = resolveRestartBudget(agent, settings);
911
+ if (!budget.allowed) {
912
+ agent._publishLifecycle('AGENT_RESTART_EXHAUSTED', {
913
+ taskId: error.taskId,
914
+ reason: error.message,
915
+ code: error.code,
916
+ attempts: attempt,
917
+ restartsSinceCompletion: budget.restartsSinceCompletion,
918
+ totalRestarts: budget.totalRestarts,
919
+ maxRestartAttempts: budget.maxRestartAttempts,
920
+ maxTotalRestarts: budget.maxTotalRestarts,
921
+ });
922
+ error.restartExhausted = true;
923
+ await handleFinalFailure(agent, triggeringMessage, error, attempt);
924
+ return { handled: true, shouldStop: true, maxRetries };
925
+ }
926
+
927
+ const nextRestartAttempt = budget.restartsSinceCompletion + 1;
928
+ const nextTotalRestart = budget.totalRestarts + 1;
929
+ agent._publishLifecycle('AGENT_RESTART_ATTEMPT', {
930
+ taskId: error.taskId,
931
+ reason: error.message,
932
+ code: error.code,
933
+ attempt: nextRestartAttempt,
934
+ totalRestartAttempt: nextTotalRestart,
935
+ maxRestartAttempts: budget.maxRestartAttempts,
936
+ maxTotalRestarts: budget.maxTotalRestarts,
937
+ });
938
+
939
+ const extendedMaxRetries = Math.max(maxRetries, attempt + 1);
940
+ await scheduleRetry(agent, error, attempt, extendedMaxRetries, baseDelay);
941
+ return { handled: true, shouldStop: false, maxRetries: extendedMaxRetries };
942
+ }
943
+
808
944
  /**
809
945
  * Execute claude-zeroshots with built context
810
946
  * Default: uses settings.maxRetries (default 3) for exponential backoff retries.
@@ -849,6 +985,30 @@ async function executeTask(agent, triggeringMessage) {
849
985
  await handleFinalFailure(agent, triggeringMessage, error, 1);
850
986
  return;
851
987
  }
988
+ agent._publishLifecycle('TASK_FAILED', {
989
+ iteration: agent.iteration,
990
+ taskId: error.taskId || agent.currentTaskId,
991
+ error: error.message,
992
+ code: error.code || null,
993
+ attempt,
994
+ });
995
+ clearTransientTaskState(agent);
996
+ const stuckTaskResult = await handleRecoverableStuckTaskFailure({
997
+ agent,
998
+ triggeringMessage,
999
+ error,
1000
+ attempt,
1001
+ maxRetries,
1002
+ baseDelay,
1003
+ settings,
1004
+ });
1005
+ if (stuckTaskResult.handled) {
1006
+ maxRetries = stuckTaskResult.maxRetries;
1007
+ if (stuckTaskResult.shouldStop) {
1008
+ return;
1009
+ }
1010
+ continue;
1011
+ }
852
1012
  const updated = maybeExtendMaxRetries({
853
1013
  error,
854
1014
  attempt,
@@ -878,89 +1038,201 @@ function startLivenessCheck(agent) {
878
1038
  clearInterval(agent.livenessCheckInterval);
879
1039
  }
880
1040
 
881
- // Check if platform supports /proc filesystem (Linux only)
882
- if (!isPlatformSupported()) {
883
- agent._log(
884
- `[${agent.id}] Liveness check disabled: /proc filesystem not available (non-Linux platform)`
885
- );
886
- return;
887
- }
1041
+ const settings = loadSettings();
1042
+ const warningsBeforeKill = Math.max(1, settings.staleWarningsBeforeKill ?? 2);
1043
+ const staleDuration = Math.max(1, agent.staleDuration);
1044
+ const configuredTimeout = agent.timeout > 0 ? agent.timeout : null;
1045
+ const shortestLimit = configuredTimeout
1046
+ ? Math.min(staleDuration, configuredTimeout)
1047
+ : staleDuration;
1048
+ const checkIntervalMs = Math.min(60 * 1000, Math.max(10, Math.floor(shortestLimit / 4)));
1049
+
1050
+ agent.consecutiveStaleWarnings = 0;
1051
+ agent.livenessTerminationStarted = false;
1052
+ agent.livenessTerminationContext = null;
1053
+ agent.livenessTerminationAttempts = 0;
1054
+ agent.livenessTerminationRetryAt = 0;
1055
+
1056
+ agent.livenessCheckInterval = setInterval(() => {
1057
+ const hasRecoverableTask =
1058
+ Boolean(agent.currentTask) || Boolean(agent.isolation?.enabled && agent.currentTaskId);
1059
+ if (!hasRecoverableTask || agent.livenessTerminationStarted) {
1060
+ return;
1061
+ }
1062
+
1063
+ const now = Date.now();
1064
+ if (agent.livenessTerminationContext) {
1065
+ if (now >= agent.livenessTerminationRetryAt) {
1066
+ attemptLivenessTermination(agent, settings);
1067
+ }
1068
+ return;
1069
+ }
888
1070
 
889
- // Check every 60 seconds (gives time for multi-indicator analysis)
890
- const CHECK_INTERVAL_MS = 60 * 1000;
891
- const ANALYSIS_SAMPLE_MS = 5000; // Sample CPU/context switches over 5 seconds
1071
+ const taskStartedAt = agent.taskStartedAt || agent.lastOutputTime || now;
1072
+ const lastOutputTime = agent.lastOutputTime || taskStartedAt;
1073
+ const taskRuntime = now - taskStartedAt;
1074
+ const timeSinceLastOutput = now - lastOutputTime;
892
1075
 
893
- agent.livenessCheckInterval = setInterval(async () => {
894
- // Skip if no task running or no PID tracked
895
- if (!agent.currentTask || !agent.processPid) {
1076
+ if (configuredTimeout && taskRuntime >= configuredTimeout) {
1077
+ const reason = `Task timed out after ${configuredTimeout}ms`;
1078
+ beginLivenessTermination(agent, settings, reason, 'AGENT_TASK_TIMEOUT', {
1079
+ taskId: agent.currentTaskId,
1080
+ taskRuntime,
1081
+ timeout: configuredTimeout,
1082
+ });
896
1083
  return;
897
1084
  }
898
1085
 
899
- // Skip if output is recent (process is clearly active)
900
- if (agent.lastOutputTime) {
901
- const timeSinceLastOutput = Date.now() - agent.lastOutputTime;
902
- if (timeSinceLastOutput < agent.staleDuration) {
903
- return; // Output is recent, definitely not stuck
904
- }
1086
+ if (timeSinceLastOutput < staleDuration) {
1087
+ agent.consecutiveStaleWarnings = 0;
1088
+ return;
905
1089
  }
906
1090
 
907
- // Output is stale - run multi-indicator analysis to confirm
908
- agent._log(
909
- `[${agent.id}] Output stale for ${Math.round((Date.now() - (agent.lastOutputTime || 0)) / 1000)}s, running multi-indicator analysis...`
910
- );
1091
+ agent.consecutiveStaleWarnings += 1;
1092
+ agent._publishLifecycle('AGENT_STALE_WARNING', {
1093
+ taskId: agent.currentTaskId,
1094
+ timeSinceLastOutput,
1095
+ staleDuration,
1096
+ lastOutputTime,
1097
+ consecutiveWarnings: agent.consecutiveStaleWarnings,
1098
+ warningsBeforeKill,
1099
+ processDiagnosticsAvailable: isPlatformSupported(),
1100
+ analysis: `Provider produced no output for ${timeSinceLastOutput}ms`,
1101
+ });
911
1102
 
912
- try {
913
- const analysis = await analyzeProcessHealth(agent.processPid, ANALYSIS_SAMPLE_MS);
1103
+ if (agent.consecutiveStaleWarnings < warningsBeforeKill) {
1104
+ return;
1105
+ }
1106
+
1107
+ const reason = `Provider produced no output for ${timeSinceLastOutput}ms`;
1108
+ beginLivenessTermination(agent, settings, reason, 'PROVIDER_INACTIVITY_TIMEOUT', {
1109
+ taskId: agent.currentTaskId,
1110
+ timeSinceLastOutput,
1111
+ staleDuration,
1112
+ consecutiveWarnings: agent.consecutiveStaleWarnings,
1113
+ });
1114
+ }, checkIntervalMs);
1115
+ }
1116
+
1117
+ const MAX_LIVENESS_TERMINATION_ATTEMPTS = 3;
1118
+
1119
+ function beginLivenessTermination(agent, settings, reason, code, eventData) {
1120
+ agent.livenessTerminationContext = {
1121
+ taskId: agent.currentTaskId,
1122
+ reason,
1123
+ code,
1124
+ eventData,
1125
+ eventPublished: false,
1126
+ };
1127
+ agent.livenessTerminationAttempts = 0;
1128
+ agent.livenessTerminationRetryAt = 0;
1129
+ attemptLivenessTermination(agent, settings);
1130
+ }
1131
+
1132
+ function publishLivenessTerminationEvent(agent, context) {
1133
+ if (context.eventPublished) return;
1134
+ context.eventPublished = true;
1135
+ agent._publishLifecycle(
1136
+ context.code === 'AGENT_TASK_TIMEOUT' ? 'AGENT_TASK_TIMEOUT' : 'AGENT_INACTIVITY_TIMEOUT',
1137
+ context.eventData
1138
+ );
1139
+ }
1140
+
1141
+ function terminationRetryDelay(settings, attempt) {
1142
+ const baseDelay = Math.max(0, settings.backoffBaseMs ?? 2000);
1143
+ const maxDelay = Math.max(0, settings.backoffMaxMs ?? 30000);
1144
+ return Math.min(maxDelay, baseDelay * Math.pow(2, Math.max(0, attempt - 1)));
1145
+ }
914
1146
 
915
- // Process died during analysis
916
- if (analysis.isLikelyStuck === null) {
917
- agent._log(`[${agent.id}] Process analysis inconclusive: ${analysis.reason}`);
1147
+ function attemptLivenessTermination(agent, settings) {
1148
+ const context = agent.livenessTerminationContext;
1149
+ if (!context || agent.livenessTerminationStarted) return;
1150
+
1151
+ agent.livenessTerminationStarted = true;
1152
+ agent.livenessTerminationAttempts += 1;
1153
+ const attempt = agent.livenessTerminationAttempts;
1154
+
1155
+ Promise.resolve()
1156
+ .then(() => agent._killTask({ reason: context.reason, code: context.code }))
1157
+ .then((termination) => {
1158
+ if (termination?.forced === false) return;
1159
+ publishLivenessTerminationEvent(agent, context);
1160
+ })
1161
+ .catch((error) => {
1162
+ if (agent.livenessTerminationContext !== context) return;
1163
+ if (attempt >= MAX_LIVENESS_TERMINATION_ATTEMPTS) {
1164
+ exhaustLivenessTermination(agent, context, error);
918
1165
  return;
919
1166
  }
920
1167
 
921
- // Log analysis details for debugging
1168
+ const delayMs = terminationRetryDelay(settings, attempt);
1169
+ agent.livenessTerminationRetryAt = Date.now() + delayMs;
1170
+ agent.livenessTerminationStarted = false;
1171
+ agent._publishLifecycle('AGENT_TERMINATION_RETRY', {
1172
+ taskId: context.taskId,
1173
+ reason: context.reason,
1174
+ code: context.code,
1175
+ error: error.message,
1176
+ attempt,
1177
+ nextAttempt: attempt + 1,
1178
+ maxAttempts: MAX_LIVENESS_TERMINATION_ATTEMPTS,
1179
+ delayMs,
1180
+ });
922
1181
  agent._log(
923
- `[${agent.id}] Analysis: score=${analysis.stuckScore}/${STUCK_THRESHOLD}, ` +
924
- `state=${analysis.state}, wchan=${analysis.wchan}, ` +
925
- `CPU=${analysis.cpuPercent}%, ctxSwitches=${analysis.ctxSwitchesDelta}`
1182
+ `[${agent.id}] Failed to terminate task (attempt ${attempt}/${MAX_LIVENESS_TERMINATION_ATTEMPTS}); ` +
1183
+ `retrying in ${delayMs}ms: ${error.message}`
926
1184
  );
1185
+ });
1186
+ }
927
1187
 
928
- if (analysis.isLikelyStuck) {
929
- agent._log(`⚠️ Agent ${agent.id}: CONFIRMED STUCK (confidence: ${analysis.confidence})`);
930
- agent._log(` ${analysis.analysis}`);
931
-
932
- // CHANGED: Stale detection is informational only - never kills tasks
933
- // Publish stale detection event with full analysis (for logging/monitoring)
934
- agent._publishLifecycle('AGENT_STALE_WARNING', {
935
- timeSinceLastOutput: Date.now() - (agent.lastOutputTime || 0),
936
- staleDuration: agent.staleDuration,
937
- lastOutputTime: agent.lastOutputTime,
938
- // Multi-indicator analysis results
939
- stuckScore: analysis.stuckScore,
940
- confidence: analysis.confidence,
941
- processState: analysis.state,
942
- wchan: analysis.wchan,
943
- cpuPercent: analysis.cpuPercent,
944
- ctxSwitchesDelta: analysis.ctxSwitchesDelta,
945
- indicators: analysis.indicators,
946
- analysis: analysis.analysis,
947
- });
948
-
949
- // Keep monitoring - do NOT stop the agent
950
- // User can manually intervene with 'zeroshot resume' if needed
951
- // stopLivenessCheck(agent); // REMOVED - keep monitoring
952
- } else {
953
- agent._log(
954
- `[${agent.id}] Process appears WORKING despite stale output (score: ${analysis.stuckScore})`
955
- );
956
- agent._log(` ${analysis.analysis}`);
957
- // Don't flag as stuck - process is legitimately working
958
- }
959
- } catch (err) {
960
- agent._log(`[${agent.id}] Error during stuck analysis: ${err.message}`);
961
- // Don't flag as stuck on analysis error
962
- }
963
- }, CHECK_INTERVAL_MS);
1188
+ function exhaustLivenessTermination(agent, context, terminationError) {
1189
+ if (agent.livenessCheckInterval) {
1190
+ clearInterval(agent.livenessCheckInterval);
1191
+ agent.livenessCheckInterval = null;
1192
+ }
1193
+
1194
+ const attempts = agent.livenessTerminationAttempts;
1195
+ publishLivenessTerminationEvent(agent, context);
1196
+ const error = new Error(
1197
+ `Failed to terminate isolated task ${context.taskId} after ${attempts} attempts; ` +
1198
+ `the provider task may still be running. Manual recovery is required before resume. ` +
1199
+ `Last error: ${terminationError.message}`
1200
+ );
1201
+ error.code = 'ISOLATED_TASK_TERMINATION_EXHAUSTED';
1202
+ error.taskId = context.taskId;
1203
+ error.permanent = true;
1204
+ error.restartExhausted = true;
1205
+ error.terminationExhausted = true;
1206
+ error.terminationAttempts = attempts;
1207
+
1208
+ agent.state = 'error';
1209
+ agent.cluster.failureInfo = {
1210
+ agentId: agent.id,
1211
+ taskId: context.taskId,
1212
+ iteration: agent.iteration,
1213
+ type: 'task_termination',
1214
+ reason: 'termination_unverified',
1215
+ error: error.message,
1216
+ attempts,
1217
+ timestamp: Date.now(),
1218
+ };
1219
+ agent._publishLifecycle('AGENT_TERMINATION_EXHAUSTED', {
1220
+ taskId: context.taskId,
1221
+ reason: context.reason,
1222
+ code: error.code,
1223
+ terminationCode: context.code,
1224
+ attempts,
1225
+ maxAttempts: MAX_LIVENESS_TERMINATION_ATTEMPTS,
1226
+ error: terminationError.message,
1227
+ requiresManualRecovery: true,
1228
+ });
1229
+
1230
+ if (typeof agent.currentTask?.failClosed === 'function') {
1231
+ agent.currentTask.failClosed(error);
1232
+ return;
1233
+ }
1234
+
1235
+ agent._log(`[${agent.id}] ${error.message}`);
964
1236
  }
965
1237
 
966
1238
  /**
@@ -972,6 +1244,11 @@ function stopLivenessCheck(agent) {
972
1244
  clearInterval(agent.livenessCheckInterval);
973
1245
  agent.livenessCheckInterval = null;
974
1246
  }
1247
+ agent.consecutiveStaleWarnings = 0;
1248
+ agent.livenessTerminationStarted = false;
1249
+ agent.livenessTerminationContext = null;
1250
+ agent.livenessTerminationAttempts = 0;
1251
+ agent.livenessTerminationRetryAt = 0;
975
1252
  }
976
1253
 
977
1254
  module.exports = {