@the-open-engine/zeroshot 6.6.0 → 6.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/zeroshot-cluster-worker.js +16 -0
- package/cli/index.js +135 -12
- package/docs/openengine-cluster-protocol/v1/legacy-worker.md +117 -0
- package/lib/cluster-worker/contracts.js +194 -0
- package/lib/cluster-worker/engine-adapter.js +300 -0
- package/lib/cluster-worker/executable.js +229 -0
- package/lib/cluster-worker/index.d.ts +227 -0
- package/lib/cluster-worker/index.js +294 -0
- package/lib/cluster-worker/object-utils.js +17 -0
- package/lib/cluster-worker/process-stdio.js +37 -0
- package/lib/cluster-worker/profiles.js +81 -0
- package/lib/cluster-worker/runtime-engine.js +50 -0
- package/lib/cluster-worker/runtime-support.js +298 -0
- package/lib/cluster-worker/state-machine.js +147 -0
- package/lib/cluster-worker/terminal-normalizer.js +95 -0
- package/lib/cluster-worker/worker-internals.js +20 -0
- package/lib/detached-startup.js +127 -11
- package/lib/process-liveness.js +26 -0
- package/lib/start-cluster.js +93 -18
- package/package.json +8 -2
- package/protocol/openengine-cluster/v1/worker.schema.json +1174 -0
- package/scripts/assert-release-published.js +65 -11
- package/src/orchestrator.js +464 -61
package/src/orchestrator.js
CHANGED
|
@@ -48,6 +48,7 @@ const TemplateResolver = require('./template-resolver');
|
|
|
48
48
|
const { loadSettings } = require('../lib/settings');
|
|
49
49
|
const { normalizeProviderName } = require('../lib/provider-names');
|
|
50
50
|
const { resolveRunPlan } = require('../lib/run-plan');
|
|
51
|
+
const { isProcessRunning } = require('../lib/process-liveness');
|
|
51
52
|
const { getProvider } = require('./providers');
|
|
52
53
|
const StateSnapshotter = require('./state-snapshotter');
|
|
53
54
|
const { resolveClusterRequiredQualityGates } = require('./quality-gates');
|
|
@@ -581,7 +582,9 @@ class Orchestrator {
|
|
|
581
582
|
isolationManager,
|
|
582
583
|
containerId: isolation?.containerId || null,
|
|
583
584
|
});
|
|
584
|
-
this.
|
|
585
|
+
if (clusterContext.state === 'running' && this._isProcessRunning(clusterContext.pid)) {
|
|
586
|
+
this._startSnapshotter(clusterContext);
|
|
587
|
+
}
|
|
585
588
|
}
|
|
586
589
|
this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
|
|
587
590
|
|
|
@@ -626,8 +629,13 @@ class Orchestrator {
|
|
|
626
629
|
id: clusterId,
|
|
627
630
|
quiet: this.quiet,
|
|
628
631
|
modelOverride: clusterData.modelOverride || null,
|
|
632
|
+
testMode: Boolean(this.taskRunner),
|
|
629
633
|
};
|
|
630
634
|
|
|
635
|
+
if (this.taskRunner) {
|
|
636
|
+
agentOptions.taskRunner = this.taskRunner;
|
|
637
|
+
}
|
|
638
|
+
|
|
631
639
|
if (isolation?.enabled && isolationManager) {
|
|
632
640
|
agentOptions.isolation = {
|
|
633
641
|
enabled: true,
|
|
@@ -758,21 +766,13 @@ class Orchestrator {
|
|
|
758
766
|
if (inactiveStates.includes(cluster.state)) continue;
|
|
759
767
|
|
|
760
768
|
// Check if process is still running (zombie detection)
|
|
761
|
-
if (cluster.pid) {
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
state: cluster.state,
|
|
769
|
-
createdAt: cluster.createdAt,
|
|
770
|
-
pid: cluster.pid,
|
|
771
|
-
});
|
|
772
|
-
} catch {
|
|
773
|
-
// Process doesn't exist - cluster is a zombie (stale entry)
|
|
774
|
-
// Don't include in active list
|
|
775
|
-
}
|
|
769
|
+
if (cluster.pid && this._isProcessRunning(cluster.pid)) {
|
|
770
|
+
activeClusters.push({
|
|
771
|
+
id: clusterId,
|
|
772
|
+
state: cluster.state,
|
|
773
|
+
createdAt: cluster.createdAt,
|
|
774
|
+
pid: cluster.pid,
|
|
775
|
+
});
|
|
776
776
|
}
|
|
777
777
|
}
|
|
778
778
|
|
|
@@ -1126,11 +1126,7 @@ class Orchestrator {
|
|
|
1126
1126
|
// Resolve input (issue/file/text) and reject duplicate active runs on the same issue
|
|
1127
1127
|
// BEFORE allocating any resources. A duplicate-run rejection must have zero side
|
|
1128
1128
|
// effects: no ledger db file, no worktree/container, no clusters.json entry.
|
|
1129
|
-
|
|
1130
|
-
input,
|
|
1131
|
-
options,
|
|
1132
|
-
clusterId
|
|
1133
|
-
);
|
|
1129
|
+
let { inputData, issueProviderId } = await this._resolveClusterInput(input, options, clusterId);
|
|
1134
1130
|
|
|
1135
1131
|
// Create ledger and message bus with persistent storage
|
|
1136
1132
|
const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
|
|
@@ -1220,6 +1216,34 @@ class Orchestrator {
|
|
|
1220
1216
|
});
|
|
1221
1217
|
|
|
1222
1218
|
try {
|
|
1219
|
+
if (typeof options.prepareIsolatedInput === 'function') {
|
|
1220
|
+
const dockerWorkspace =
|
|
1221
|
+
isolationManager?.isolatedDirs?.get(clusterId)?.path || options.cwd || process.cwd();
|
|
1222
|
+
const isolation = worktreeInfo
|
|
1223
|
+
? Object.freeze({
|
|
1224
|
+
kind: 'worktree',
|
|
1225
|
+
hostRoot: worktreeInfo.path,
|
|
1226
|
+
runtimeRoot: worktreeInfo.path,
|
|
1227
|
+
})
|
|
1228
|
+
: containerId && dockerWorkspace
|
|
1229
|
+
? Object.freeze({
|
|
1230
|
+
kind: 'docker',
|
|
1231
|
+
hostRoot: dockerWorkspace,
|
|
1232
|
+
runtimeRoot: '/workspace',
|
|
1233
|
+
})
|
|
1234
|
+
: null;
|
|
1235
|
+
if (!isolation) {
|
|
1236
|
+
throw new Error('Prepared input requires an allocated worktree or Docker workspace');
|
|
1237
|
+
}
|
|
1238
|
+
const preparedText = await options.prepareIsolatedInput(
|
|
1239
|
+
Object.freeze({ clusterId, isolation })
|
|
1240
|
+
);
|
|
1241
|
+
if (typeof preparedText !== 'string' || preparedText.trim().length === 0) {
|
|
1242
|
+
throw new Error('Prepared isolated input must be a nonempty string');
|
|
1243
|
+
}
|
|
1244
|
+
inputData = InputHelpers.createTextInput(preparedText);
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1223
1247
|
// Input (issue/file/text) was already resolved and duplicate-checked in
|
|
1224
1248
|
// _resolveClusterInput() before any resource was allocated (see above).
|
|
1225
1249
|
commandProofs = mergeCommandProofs(
|
|
@@ -2369,8 +2393,17 @@ class Orchestrator {
|
|
|
2369
2393
|
}
|
|
2370
2394
|
|
|
2371
2395
|
if (cluster.state === 'running') {
|
|
2372
|
-
|
|
2373
|
-
|
|
2396
|
+
if (this._isProcessRunning(cluster.pid)) {
|
|
2397
|
+
throw new Error(
|
|
2398
|
+
`Cluster ${clusterId} is still running. Use 'zeroshot stop' first if you want to restart it.`
|
|
2399
|
+
);
|
|
2400
|
+
}
|
|
2401
|
+
// state says running but the recorded PID is dead: this is the same
|
|
2402
|
+
// zombie condition getStatus()/listClusters() already detect. Recover
|
|
2403
|
+
// it here instead of forcing a manual `stop` for a process that no
|
|
2404
|
+
// longer exists.
|
|
2405
|
+
this._log(
|
|
2406
|
+
`[Orchestrator] Cluster ${clusterId} was marked running with dead PID ${cluster.pid}; recovering as a zombie instead of rejecting resume.`
|
|
2374
2407
|
);
|
|
2375
2408
|
}
|
|
2376
2409
|
|
|
@@ -2388,19 +2421,28 @@ class Orchestrator {
|
|
|
2388
2421
|
}
|
|
2389
2422
|
|
|
2390
2423
|
const failureInfo = this._resolveFailureInfo(cluster, clusterId);
|
|
2424
|
+
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
2425
|
+
const cleanResumePlan = failureInfo ? null : this._buildCleanResumePlan(clusterId, cluster);
|
|
2426
|
+
|
|
2427
|
+
if (cleanResumePlan?.kind === 'failure') {
|
|
2428
|
+
return this._failResumeReconstruction(clusterId, cluster, cleanResumePlan);
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
if (failureInfo) {
|
|
2432
|
+
this._requireFailedAgent(clusterId, cluster, failureInfo);
|
|
2433
|
+
}
|
|
2391
2434
|
|
|
2392
2435
|
await this._ensureIsolationForResume(clusterId, cluster);
|
|
2393
2436
|
this._ensureWorktreeForResume(clusterId, cluster);
|
|
2394
2437
|
this._startSnapshotter(cluster);
|
|
2438
|
+
this._clearTransientAgentState(cluster);
|
|
2395
2439
|
await this._restartClusterAgents(cluster);
|
|
2396
2440
|
|
|
2397
|
-
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
2398
|
-
|
|
2399
2441
|
if (failureInfo) {
|
|
2400
2442
|
return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
|
|
2401
2443
|
}
|
|
2402
2444
|
|
|
2403
|
-
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
|
|
2445
|
+
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt, cleanResumePlan);
|
|
2404
2446
|
}
|
|
2405
2447
|
|
|
2406
2448
|
_validateGuidanceAgentArgs(clusterId, agentId, text) {
|
|
@@ -2596,7 +2638,7 @@ class Orchestrator {
|
|
|
2596
2638
|
|
|
2597
2639
|
_resolveFailureInfo(cluster, clusterId) {
|
|
2598
2640
|
if (cluster.failureInfo) {
|
|
2599
|
-
return cluster.failureInfo;
|
|
2641
|
+
return cluster.failureInfo.agentId ? cluster.failureInfo : null;
|
|
2600
2642
|
}
|
|
2601
2643
|
|
|
2602
2644
|
const errors = cluster.messageBus.query({
|
|
@@ -2622,6 +2664,15 @@ class Orchestrator {
|
|
|
2622
2664
|
return failureInfo;
|
|
2623
2665
|
}
|
|
2624
2666
|
|
|
2667
|
+
_requireFailedAgent(clusterId, cluster, failureInfo) {
|
|
2668
|
+
const failedAgent = cluster.agents.find((agent) => agent.id === failureInfo.agentId);
|
|
2669
|
+
if (!failedAgent) {
|
|
2670
|
+
throw new Error(
|
|
2671
|
+
`Cannot resume cluster ${clusterId}: Failed agent '${failureInfo.agentId}' is not present in the restored configuration.`
|
|
2672
|
+
);
|
|
2673
|
+
}
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2625
2676
|
_checkContainerExists(containerId) {
|
|
2626
2677
|
const { spawn } = require('child_process');
|
|
2627
2678
|
const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
|
|
@@ -2715,6 +2766,22 @@ class Orchestrator {
|
|
|
2715
2766
|
.reverse();
|
|
2716
2767
|
}
|
|
2717
2768
|
|
|
2769
|
+
_findLastDurableWorkflowTrigger(cluster, clusterId) {
|
|
2770
|
+
let latest = null;
|
|
2771
|
+
|
|
2772
|
+
for (const topic of WORKFLOW_TRIGGERS) {
|
|
2773
|
+
const candidate = cluster.messageBus.findLast({
|
|
2774
|
+
cluster_id: clusterId,
|
|
2775
|
+
topic,
|
|
2776
|
+
});
|
|
2777
|
+
if (candidate && (!latest || candidate.timestamp > latest.timestamp)) {
|
|
2778
|
+
latest = candidate;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
return latest;
|
|
2783
|
+
}
|
|
2784
|
+
|
|
2718
2785
|
_buildResumeContext(recentMessages, prompt, options) {
|
|
2719
2786
|
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
2720
2787
|
let context = `${options.header}\n\n`;
|
|
@@ -2723,6 +2790,13 @@ class Orchestrator {
|
|
|
2723
2790
|
context += `Previous error: ${options.error}\n\n`;
|
|
2724
2791
|
}
|
|
2725
2792
|
|
|
2793
|
+
if (options.durableMessages?.length > 0) {
|
|
2794
|
+
context += `## Durable Workflow Context\n\n`;
|
|
2795
|
+
for (const msg of options.durableMessages) {
|
|
2796
|
+
context += `[${msg.topic}] [${msg.sender}]\n${msg.content?.text || ''}\n\n`;
|
|
2797
|
+
}
|
|
2798
|
+
}
|
|
2799
|
+
|
|
2726
2800
|
context += `## Recent Context\n\n`;
|
|
2727
2801
|
for (const msg of recentMessages.slice(-10)) {
|
|
2728
2802
|
if (options.topics.includes(msg.topic)) {
|
|
@@ -2745,7 +2819,8 @@ class Orchestrator {
|
|
|
2745
2819
|
}
|
|
2746
2820
|
|
|
2747
2821
|
_selectAgentsToResume(cluster, lastTrigger) {
|
|
2748
|
-
const
|
|
2822
|
+
const matching = [];
|
|
2823
|
+
const eligible = [];
|
|
2749
2824
|
|
|
2750
2825
|
for (const agent of cluster.agents) {
|
|
2751
2826
|
if (!agent.config.triggers) continue;
|
|
@@ -2755,15 +2830,365 @@ class Orchestrator {
|
|
|
2755
2830
|
);
|
|
2756
2831
|
if (!matchingTrigger) continue;
|
|
2757
2832
|
|
|
2833
|
+
const candidate = { agent, message: lastTrigger, trigger: matchingTrigger };
|
|
2834
|
+
matching.push(candidate);
|
|
2835
|
+
|
|
2758
2836
|
if (matchingTrigger.logic?.script) {
|
|
2759
2837
|
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
2760
2838
|
if (!shouldTrigger) continue;
|
|
2761
2839
|
}
|
|
2762
2840
|
|
|
2763
|
-
|
|
2841
|
+
eligible.push(candidate);
|
|
2842
|
+
}
|
|
2843
|
+
|
|
2844
|
+
return { matching, eligible };
|
|
2845
|
+
}
|
|
2846
|
+
|
|
2847
|
+
_findDurableTriggerBefore(cluster, clusterId, topic, timestamp) {
|
|
2848
|
+
if (!topic || topic === 'AGENT_RESUME') {
|
|
2849
|
+
return null;
|
|
2850
|
+
}
|
|
2851
|
+
|
|
2852
|
+
return cluster.messageBus.findLast({
|
|
2853
|
+
cluster_id: clusterId,
|
|
2854
|
+
topic,
|
|
2855
|
+
until: timestamp - 1,
|
|
2856
|
+
});
|
|
2857
|
+
}
|
|
2858
|
+
|
|
2859
|
+
_collectDurableResumeContext(cluster, clusterId, messages = []) {
|
|
2860
|
+
const durableMessages = [
|
|
2861
|
+
cluster.messageBus.findLast({ cluster_id: clusterId, topic: 'ISSUE_OPENED' }),
|
|
2862
|
+
cluster.messageBus.findLast({ cluster_id: clusterId, topic: 'PLAN_READY' }),
|
|
2863
|
+
cluster.messageBus.findLast({ cluster_id: clusterId, topic: 'IMPLEMENTATION_READY' }),
|
|
2864
|
+
...messages,
|
|
2865
|
+
].filter(Boolean);
|
|
2866
|
+
const uniqueById = new Map(durableMessages.map((message) => [message.id, message]));
|
|
2867
|
+
return [...uniqueById.values()].sort((left, right) => left.timestamp - right.timestamp);
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
_findInterruptedResumeTargets(clusterId, cluster) {
|
|
2871
|
+
const targets = [];
|
|
2872
|
+
const invalid = [];
|
|
2873
|
+
const issueMessage = cluster.messageBus.findLast({
|
|
2874
|
+
cluster_id: clusterId,
|
|
2875
|
+
topic: 'ISSUE_OPENED',
|
|
2876
|
+
});
|
|
2877
|
+
|
|
2878
|
+
for (const agent of cluster.agents) {
|
|
2879
|
+
if (agent.state !== 'executing_task') {
|
|
2880
|
+
continue;
|
|
2881
|
+
}
|
|
2882
|
+
|
|
2883
|
+
const lifecycle = cluster.messageBus.query({
|
|
2884
|
+
cluster_id: clusterId,
|
|
2885
|
+
topic: 'AGENT_LIFECYCLE',
|
|
2886
|
+
sender: agent.id,
|
|
2887
|
+
});
|
|
2888
|
+
const startedIndex = lifecycle.findLastIndex(
|
|
2889
|
+
(message) => message.content?.data?.event === 'TASK_STARTED'
|
|
2890
|
+
);
|
|
2891
|
+
const lastStarted = startedIndex >= 0 ? lifecycle[startedIndex] : null;
|
|
2892
|
+
const lifecycleAfterStart = startedIndex >= 0 ? lifecycle.slice(startedIndex + 1) : [];
|
|
2893
|
+
const completedAfterStart = lifecycleAfterStart.some(
|
|
2894
|
+
(message) => message.content?.data?.event === 'TASK_COMPLETED'
|
|
2895
|
+
);
|
|
2896
|
+
const lastTaskIdAssigned = [...lifecycleAfterStart]
|
|
2897
|
+
.reverse()
|
|
2898
|
+
.find((message) => message.content?.data?.event === 'TASK_ID_ASSIGNED');
|
|
2899
|
+
const assignedTaskId = lastTaskIdAssigned?.content?.data?.taskId || null;
|
|
2900
|
+
const startedIteration = lastStarted?.content?.data?.iteration;
|
|
2901
|
+
const triggeredBy = lastStarted?.content?.data?.triggeredBy;
|
|
2902
|
+
const triggeringMessage = lastStarted
|
|
2903
|
+
? this._findDurableTriggerBefore(cluster, clusterId, triggeredBy, lastStarted.timestamp)
|
|
2904
|
+
: null;
|
|
2905
|
+
|
|
2906
|
+
if (
|
|
2907
|
+
!agent.currentTaskId ||
|
|
2908
|
+
!lastStarted ||
|
|
2909
|
+
completedAfterStart ||
|
|
2910
|
+
startedIteration !== agent.iteration ||
|
|
2911
|
+
!assignedTaskId ||
|
|
2912
|
+
assignedTaskId !== agent.currentTaskId ||
|
|
2913
|
+
!issueMessage
|
|
2914
|
+
) {
|
|
2915
|
+
invalid.push({
|
|
2916
|
+
agentId: agent.id,
|
|
2917
|
+
reason: !issueMessage ? 'missing_durable_context' : 'ambiguous_task_identity',
|
|
2918
|
+
state: agent.state,
|
|
2919
|
+
iteration: agent.iteration,
|
|
2920
|
+
currentTaskId: agent.currentTaskId || null,
|
|
2921
|
+
assignedTaskId,
|
|
2922
|
+
lastStartedIteration: startedIteration ?? null,
|
|
2923
|
+
completedAfterStart: Boolean(completedAfterStart),
|
|
2924
|
+
});
|
|
2925
|
+
continue;
|
|
2926
|
+
}
|
|
2927
|
+
|
|
2928
|
+
targets.push({
|
|
2929
|
+
agent,
|
|
2930
|
+
message: triggeringMessage || issueMessage,
|
|
2931
|
+
lifecycleMessage: lastStarted,
|
|
2932
|
+
issueMessage,
|
|
2933
|
+
trigger: null,
|
|
2934
|
+
reason: 'interrupted_task',
|
|
2935
|
+
});
|
|
2764
2936
|
}
|
|
2765
2937
|
|
|
2766
|
-
|
|
2938
|
+
const interruptedRoles = new Set(targets.map(({ agent }) => agent.role));
|
|
2939
|
+
if (interruptedRoles.size > 1) {
|
|
2940
|
+
invalid.push({
|
|
2941
|
+
reason: 'multiple_interrupted_phases',
|
|
2942
|
+
agents: targets.map(({ agent }) => agent.id),
|
|
2943
|
+
});
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
return { targets, invalid };
|
|
2947
|
+
}
|
|
2948
|
+
|
|
2949
|
+
_findPartialValidationTargets(clusterId, cluster, lastTrigger) {
|
|
2950
|
+
if (
|
|
2951
|
+
!lastTrigger ||
|
|
2952
|
+
!['IMPLEMENTATION_READY', 'VALIDATION_RESULT'].includes(lastTrigger.topic)
|
|
2953
|
+
) {
|
|
2954
|
+
return null;
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
const implementationReady = cluster.messageBus.findLast({
|
|
2958
|
+
cluster_id: clusterId,
|
|
2959
|
+
topic: 'IMPLEMENTATION_READY',
|
|
2960
|
+
until: lastTrigger.timestamp,
|
|
2961
|
+
});
|
|
2962
|
+
if (!implementationReady) {
|
|
2963
|
+
return null;
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
const validators = cluster.agents.filter((agent) => agent.role === 'validator');
|
|
2967
|
+
if (validators.length === 0) {
|
|
2968
|
+
return null;
|
|
2969
|
+
}
|
|
2970
|
+
|
|
2971
|
+
const validatorIds = new Set(validators.map((agent) => agent.id));
|
|
2972
|
+
const validationResults = cluster.messageBus
|
|
2973
|
+
.query({
|
|
2974
|
+
cluster_id: clusterId,
|
|
2975
|
+
topic: 'VALIDATION_RESULT',
|
|
2976
|
+
since: implementationReady.timestamp,
|
|
2977
|
+
})
|
|
2978
|
+
.filter(
|
|
2979
|
+
(message) =>
|
|
2980
|
+
message.timestamp > implementationReady.timestamp && validatorIds.has(message.sender)
|
|
2981
|
+
);
|
|
2982
|
+
const responded = new Set(validationResults.map((message) => message.sender));
|
|
2983
|
+
const missing = validators.filter((agent) => !responded.has(agent.id));
|
|
2984
|
+
if (missing.length === 0) {
|
|
2985
|
+
return { kind: 'complete', validationResults, implementationReady };
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
const missingIds = new Set(missing.map((agent) => agent.id));
|
|
2989
|
+
const selection = this._selectAgentsToResume(cluster, implementationReady);
|
|
2990
|
+
const matching = selection.matching.filter(({ agent }) => missingIds.has(agent.id));
|
|
2991
|
+
const eligible = selection.eligible.filter(({ agent }) => missingIds.has(agent.id));
|
|
2992
|
+
|
|
2993
|
+
if (eligible.length !== missing.length) {
|
|
2994
|
+
return {
|
|
2995
|
+
kind: 'failure',
|
|
2996
|
+
missingAgentIds: missing.map((agent) => agent.id),
|
|
2997
|
+
matchingAgentIds: matching.map(({ agent }) => agent.id),
|
|
2998
|
+
implementationReady,
|
|
2999
|
+
validationResults,
|
|
3000
|
+
};
|
|
3001
|
+
}
|
|
3002
|
+
|
|
3003
|
+
return {
|
|
3004
|
+
kind: 'agents',
|
|
3005
|
+
agentsToResume: eligible.map((target) => ({
|
|
3006
|
+
...target,
|
|
3007
|
+
reason: 'missing_validation_result',
|
|
3008
|
+
})),
|
|
3009
|
+
implementationReady,
|
|
3010
|
+
validationResults,
|
|
3011
|
+
};
|
|
3012
|
+
}
|
|
3013
|
+
|
|
3014
|
+
_buildCleanResumePlan(clusterId, cluster) {
|
|
3015
|
+
const interrupted = this._findInterruptedResumeTargets(clusterId, cluster);
|
|
3016
|
+
if (interrupted.invalid.length > 0) {
|
|
3017
|
+
const missingContext = interrupted.invalid.some(
|
|
3018
|
+
(entry) => entry.reason === 'missing_durable_context'
|
|
3019
|
+
);
|
|
3020
|
+
return {
|
|
3021
|
+
kind: 'failure',
|
|
3022
|
+
error: missingContext
|
|
3023
|
+
? `Cannot resume cluster ${clusterId}: interrupted work has no durable workflow context. ` +
|
|
3024
|
+
`No agents were started and the preserved cluster remains stopped. ` +
|
|
3025
|
+
`Inspect it with 'zeroshot export ${clusterId}' before repairing its persisted state; do not launch a duplicate run.`
|
|
3026
|
+
: `Cannot resume cluster ${clusterId}: persisted interrupted-task state is ambiguous. ` +
|
|
3027
|
+
`No agents were started; inspect failureInfo for reconstruction details.`,
|
|
3028
|
+
details: {
|
|
3029
|
+
reason: missingContext ? 'missing_workflow_history' : 'ambiguous_interrupted_task',
|
|
3030
|
+
interrupted: interrupted.invalid,
|
|
3031
|
+
},
|
|
3032
|
+
};
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
const lastTrigger = this._findLastDurableWorkflowTrigger(cluster, clusterId);
|
|
3036
|
+
const partialValidation = this._findPartialValidationTargets(clusterId, cluster, lastTrigger);
|
|
3037
|
+
if (partialValidation?.kind === 'failure') {
|
|
3038
|
+
return {
|
|
3039
|
+
kind: 'failure',
|
|
3040
|
+
error:
|
|
3041
|
+
`Cannot resume cluster ${clusterId}: validators ${partialValidation.missingAgentIds.join(
|
|
3042
|
+
', '
|
|
3043
|
+
)} are missing results, but not every missing validator has an eligible IMPLEMENTATION_READY handler. ` +
|
|
3044
|
+
`No agents were started.`,
|
|
3045
|
+
details: {
|
|
3046
|
+
reason: 'partial_validation_handlers_ineligible',
|
|
3047
|
+
missingAgentIds: partialValidation.missingAgentIds,
|
|
3048
|
+
matchingAgentIds: partialValidation.matchingAgentIds,
|
|
3049
|
+
},
|
|
3050
|
+
};
|
|
3051
|
+
}
|
|
3052
|
+
|
|
3053
|
+
if (interrupted.targets.length > 0) {
|
|
3054
|
+
const interruptedValidators = interrupted.targets.filter(
|
|
3055
|
+
({ agent }) => agent.role === 'validator'
|
|
3056
|
+
);
|
|
3057
|
+
if (interruptedValidators.length > 0 && partialValidation?.kind !== 'agents') {
|
|
3058
|
+
return {
|
|
3059
|
+
kind: 'failure',
|
|
3060
|
+
error:
|
|
3061
|
+
`Cannot resume cluster ${clusterId}: interrupted validator state does not match an active validation cycle. ` +
|
|
3062
|
+
`No agents were started.`,
|
|
3063
|
+
details: {
|
|
3064
|
+
reason: 'ambiguous_validation_cycle',
|
|
3065
|
+
interruptedAgentIds: interruptedValidators.map(({ agent }) => agent.id),
|
|
3066
|
+
},
|
|
3067
|
+
};
|
|
3068
|
+
}
|
|
3069
|
+
|
|
3070
|
+
const targets =
|
|
3071
|
+
interruptedValidators.length > 0 ? partialValidation.agentsToResume : interrupted.targets;
|
|
3072
|
+
return {
|
|
3073
|
+
kind: 'agents',
|
|
3074
|
+
reason: interruptedValidators.length > 0 ? 'partial_validation' : 'interrupted_tasks',
|
|
3075
|
+
agentsToResume: targets,
|
|
3076
|
+
lastTrigger,
|
|
3077
|
+
durableMessages: this._collectDurableResumeContext(
|
|
3078
|
+
cluster,
|
|
3079
|
+
clusterId,
|
|
3080
|
+
targets.map(({ message }) => message)
|
|
3081
|
+
),
|
|
3082
|
+
};
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
if (partialValidation?.kind === 'agents') {
|
|
3086
|
+
return {
|
|
3087
|
+
kind: 'agents',
|
|
3088
|
+
reason: 'partial_validation',
|
|
3089
|
+
agentsToResume: partialValidation.agentsToResume,
|
|
3090
|
+
lastTrigger,
|
|
3091
|
+
durableMessages: this._collectDurableResumeContext(
|
|
3092
|
+
cluster,
|
|
3093
|
+
clusterId,
|
|
3094
|
+
partialValidation.validationResults
|
|
3095
|
+
),
|
|
3096
|
+
};
|
|
3097
|
+
}
|
|
3098
|
+
|
|
3099
|
+
if (!lastTrigger) {
|
|
3100
|
+
const issueMessage = cluster.messageBus.findLast({
|
|
3101
|
+
cluster_id: clusterId,
|
|
3102
|
+
topic: 'ISSUE_OPENED',
|
|
3103
|
+
});
|
|
3104
|
+
if (issueMessage) {
|
|
3105
|
+
return {
|
|
3106
|
+
kind: 'bootstrap',
|
|
3107
|
+
reason: 'missing_workflow_trigger',
|
|
3108
|
+
issueMessage,
|
|
3109
|
+
agentsToResume: [],
|
|
3110
|
+
lastTrigger: null,
|
|
3111
|
+
durableMessages: [issueMessage],
|
|
3112
|
+
};
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
return {
|
|
3116
|
+
kind: 'failure',
|
|
3117
|
+
error:
|
|
3118
|
+
`Cannot resume cluster ${clusterId}: no workflow trigger or ISSUE_OPENED message exists. ` +
|
|
3119
|
+
`No agents were started and the preserved cluster remains stopped. ` +
|
|
3120
|
+
`Inspect it with 'zeroshot export ${clusterId}' before repairing its persisted state; do not launch a duplicate run.`,
|
|
3121
|
+
details: { reason: 'missing_workflow_history' },
|
|
3122
|
+
};
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
const selection = this._selectAgentsToResume(cluster, lastTrigger);
|
|
3126
|
+
if (selection.eligible.length > 0) {
|
|
3127
|
+
return {
|
|
3128
|
+
kind: 'agents',
|
|
3129
|
+
reason: 'eligible_handlers',
|
|
3130
|
+
agentsToResume: selection.eligible,
|
|
3131
|
+
lastTrigger,
|
|
3132
|
+
durableMessages: this._collectDurableResumeContext(
|
|
3133
|
+
cluster,
|
|
3134
|
+
clusterId,
|
|
3135
|
+
selection.eligible.map(({ message }) => message)
|
|
3136
|
+
),
|
|
3137
|
+
};
|
|
3138
|
+
}
|
|
3139
|
+
|
|
3140
|
+
const matchingAgentIds = selection.matching.map(({ agent }) => agent.id);
|
|
3141
|
+
if (matchingAgentIds.length > 0) {
|
|
3142
|
+
return {
|
|
3143
|
+
kind: 'failure',
|
|
3144
|
+
error:
|
|
3145
|
+
`Cannot resume cluster ${clusterId}: trigger ${lastTrigger.topic} is registered by ` +
|
|
3146
|
+
`${matchingAgentIds.join(', ')}, but no handler is currently eligible and no interrupted task can be reconstructed. ` +
|
|
3147
|
+
`No agents were started.`,
|
|
3148
|
+
details: {
|
|
3149
|
+
reason: 'registered_handlers_ineligible',
|
|
3150
|
+
trigger: lastTrigger.topic,
|
|
3151
|
+
matchingAgentIds,
|
|
3152
|
+
},
|
|
3153
|
+
};
|
|
3154
|
+
}
|
|
3155
|
+
|
|
3156
|
+
return {
|
|
3157
|
+
kind: 'failure',
|
|
3158
|
+
error:
|
|
3159
|
+
`Cannot resume cluster ${clusterId}: no restored agent registers trigger ${lastTrigger.topic}. ` +
|
|
3160
|
+
`No agents were started; check the persisted agent configuration.`,
|
|
3161
|
+
details: {
|
|
3162
|
+
reason: 'unhandled_trigger',
|
|
3163
|
+
trigger: lastTrigger.topic,
|
|
3164
|
+
},
|
|
3165
|
+
};
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
async _failResumeReconstruction(clusterId, cluster, plan) {
|
|
3169
|
+
cluster.state = 'stopped';
|
|
3170
|
+
cluster.pid = null;
|
|
3171
|
+
cluster.failureInfo = {
|
|
3172
|
+
type: 'resume_reconstruction',
|
|
3173
|
+
error: plan.error,
|
|
3174
|
+
...plan.details,
|
|
3175
|
+
timestamp: Date.now(),
|
|
3176
|
+
};
|
|
3177
|
+
await this._saveClusters();
|
|
3178
|
+
|
|
3179
|
+
const error = new Error(plan.error);
|
|
3180
|
+
error.code = 'RESUME_RECONSTRUCTION_FAILED';
|
|
3181
|
+
throw error;
|
|
3182
|
+
}
|
|
3183
|
+
|
|
3184
|
+
_clearTransientAgentState(cluster) {
|
|
3185
|
+
for (const agent of cluster.agents) {
|
|
3186
|
+
agent.currentTask = null;
|
|
3187
|
+
agent.currentTaskId = null;
|
|
3188
|
+
agent.processPid = null;
|
|
3189
|
+
agent._currentExecution = null;
|
|
3190
|
+
agent.state = 'idle';
|
|
3191
|
+
}
|
|
2767
3192
|
}
|
|
2768
3193
|
|
|
2769
3194
|
_resumeAgents(agentsToResume, context) {
|
|
@@ -2829,15 +3254,16 @@ class Orchestrator {
|
|
|
2829
3254
|
};
|
|
2830
3255
|
}
|
|
2831
3256
|
|
|
2832
|
-
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
|
|
3257
|
+
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt, plan) {
|
|
2833
3258
|
this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
|
|
2834
3259
|
|
|
2835
3260
|
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
2836
3261
|
header: 'Resuming cluster from previous session.',
|
|
2837
3262
|
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
|
|
3263
|
+
durableMessages: plan.durableMessages,
|
|
2838
3264
|
});
|
|
2839
3265
|
|
|
2840
|
-
const lastTrigger =
|
|
3266
|
+
const lastTrigger = plan.lastTrigger;
|
|
2841
3267
|
if (lastTrigger) {
|
|
2842
3268
|
this._log(
|
|
2843
3269
|
`[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
|
|
@@ -2846,31 +3272,16 @@ class Orchestrator {
|
|
|
2846
3272
|
this._log(`[Orchestrator] No workflow triggers found in ledger`);
|
|
2847
3273
|
}
|
|
2848
3274
|
|
|
2849
|
-
const agentsToResume =
|
|
2850
|
-
if (
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
`[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
|
|
2854
|
-
);
|
|
2855
|
-
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
2856
|
-
|
|
2857
|
-
if (!this._republishIssue(cluster, clusterId, recentMessages)) {
|
|
2858
|
-
throw new Error(
|
|
2859
|
-
`Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
|
|
2860
|
-
`The cluster may not have started properly. Try: zeroshot run <issue> instead.`
|
|
2861
|
-
);
|
|
2862
|
-
}
|
|
2863
|
-
} else {
|
|
2864
|
-
throw new Error(
|
|
2865
|
-
`Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
|
|
2866
|
-
`Check agent trigger configurations.`
|
|
2867
|
-
);
|
|
2868
|
-
}
|
|
3275
|
+
const agentsToResume = plan.agentsToResume;
|
|
3276
|
+
if (plan.kind === 'bootstrap') {
|
|
3277
|
+
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
3278
|
+
this._republishIssue(cluster, clusterId, [plan.issueMessage]);
|
|
2869
3279
|
} else {
|
|
2870
3280
|
this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
|
|
2871
3281
|
this._resumeAgents(agentsToResume, context);
|
|
2872
3282
|
}
|
|
2873
3283
|
|
|
3284
|
+
cluster.failureInfo = null;
|
|
2874
3285
|
await this._saveClusters();
|
|
2875
3286
|
this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
|
|
2876
3287
|
|
|
@@ -3632,15 +4043,7 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
3632
4043
|
* @private
|
|
3633
4044
|
*/
|
|
3634
4045
|
_isProcessRunning(pid) {
|
|
3635
|
-
|
|
3636
|
-
try {
|
|
3637
|
-
// Signal 0 doesn't kill, just checks if process exists
|
|
3638
|
-
process.kill(pid, 0);
|
|
3639
|
-
return true;
|
|
3640
|
-
} catch (e) {
|
|
3641
|
-
// ESRCH = No such process, EPERM = process exists but no permission
|
|
3642
|
-
return e.code === 'EPERM';
|
|
3643
|
-
}
|
|
4046
|
+
return isProcessRunning(pid);
|
|
3644
4047
|
}
|
|
3645
4048
|
|
|
3646
4049
|
/**
|