@covibes/zeroshot 5.2.1 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +174 -189
- package/README.md +226 -195
- package/cli/commands/providers.js +149 -0
- package/cli/index.js +3145 -2366
- package/cli/lib/first-run.js +40 -3
- package/cli/message-formatters-normal.js +28 -6
- package/cluster-templates/base-templates/debug-workflow.json +24 -78
- package/cluster-templates/base-templates/full-workflow.json +99 -316
- package/cluster-templates/base-templates/single-worker.json +23 -15
- package/cluster-templates/base-templates/worker-validator.json +105 -36
- package/cluster-templates/conductor-bootstrap.json +9 -7
- package/lib/docker-config.js +14 -1
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +10 -7
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-detection.js +59 -0
- package/lib/provider-names.js +57 -0
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +298 -15
- package/lib/stream-json-parser.js +4 -238
- package/package.json +27 -6
- package/scripts/setup-merge-queue.sh +170 -0
- package/scripts/validate-templates.js +100 -0
- package/src/agent/agent-config.js +140 -63
- package/src/agent/agent-context-builder.js +336 -165
- package/src/agent/agent-hook-executor.js +337 -67
- package/src/agent/agent-lifecycle.js +386 -287
- package/src/agent/agent-stuck-detector.js +7 -7
- package/src/agent/agent-task-executor.js +944 -683
- package/src/agent/output-extraction.js +217 -0
- package/src/agent/output-reformatter.js +175 -0
- package/src/agent/schema-utils.js +146 -0
- package/src/agent-wrapper.js +112 -31
- package/src/agents/git-pusher-template.js +285 -0
- package/src/claude-task-runner.js +145 -44
- package/src/config-router.js +13 -13
- package/src/config-validator.js +1049 -563
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +499 -320
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +273 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +179 -0
- package/src/issue-providers/gitlab-provider.js +241 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +239 -0
- package/src/ledger.js +50 -11
- package/src/lib/safe-exec.js +88 -0
- package/src/orchestrator.js +1348 -757
- package/src/preflight.js +306 -149
- package/src/process-metrics.js +98 -56
- package/src/providers/anthropic/cli-builder.js +73 -0
- package/src/providers/anthropic/index.js +204 -0
- package/src/providers/anthropic/models.js +23 -0
- package/src/providers/anthropic/output-parser.js +177 -0
- package/src/providers/base-provider.js +251 -0
- package/src/providers/capabilities.js +60 -0
- package/src/providers/google/cli-builder.js +55 -0
- package/src/providers/google/index.js +116 -0
- package/src/providers/google/models.js +24 -0
- package/src/providers/google/output-parser.js +101 -0
- package/src/providers/index.js +91 -0
- package/src/providers/openai/cli-builder.js +133 -0
- package/src/providers/openai/index.js +136 -0
- package/src/providers/openai/models.js +21 -0
- package/src/providers/openai/output-parser.js +143 -0
- package/src/providers/opencode/cli-builder.js +42 -0
- package/src/providers/opencode/index.js +103 -0
- package/src/providers/opencode/models.js +55 -0
- package/src/providers/opencode/output-parser.js +122 -0
- package/src/schemas/sub-cluster.js +68 -39
- package/src/status-footer.js +94 -48
- package/src/sub-cluster-wrapper.js +92 -36
- package/src/task-runner.js +8 -6
- package/src/template-resolver.js +12 -9
- package/src/tui/data-poller.js +123 -99
- package/src/tui/formatters.js +4 -3
- package/src/tui/keybindings.js +259 -318
- package/src/tui/layout.js +20 -3
- package/src/tui/renderer.js +11 -21
- package/task-lib/attachable-watcher.js +150 -111
- package/task-lib/claude-recovery.js +156 -0
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/list.js +3 -3
- package/task-lib/commands/logs.js +231 -189
- package/task-lib/commands/resume.js +3 -2
- package/task-lib/commands/run.js +12 -3
- package/task-lib/commands/schedules.js +111 -61
- package/task-lib/config.js +0 -2
- package/task-lib/runner.js +128 -50
- package/task-lib/scheduler.js +3 -3
- package/task-lib/store.js +464 -152
- package/task-lib/tui/formatters.js +94 -45
- package/task-lib/tui/renderer.js +13 -3
- package/task-lib/tui.js +73 -32
- package/task-lib/watcher.js +157 -100
- package/src/agents/git-pusher-agent.json +0 -20
- package/src/github.js +0 -103
package/src/orchestrator.js
CHANGED
|
@@ -13,15 +13,51 @@ const fs = require('fs');
|
|
|
13
13
|
const path = require('path');
|
|
14
14
|
const os = require('os');
|
|
15
15
|
const lockfile = require('proper-lockfile');
|
|
16
|
+
|
|
17
|
+
// Stale lock timeout in ms - if lock file is older than this, delete it
|
|
18
|
+
const LOCK_STALE_MS = 5000;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Remove lock file if it's stale (older than LOCK_STALE_MS)
|
|
22
|
+
* Handles crashes that leave orphaned lock files
|
|
23
|
+
*/
|
|
24
|
+
function cleanStaleLock(lockPath) {
|
|
25
|
+
try {
|
|
26
|
+
if (fs.existsSync(lockPath)) {
|
|
27
|
+
const age = Date.now() - fs.statSync(lockPath).mtimeMs;
|
|
28
|
+
if (age > LOCK_STALE_MS) {
|
|
29
|
+
fs.unlinkSync(lockPath);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
} catch {
|
|
33
|
+
// Ignore - another process may have cleaned it
|
|
34
|
+
}
|
|
35
|
+
}
|
|
16
36
|
const AgentWrapper = require('./agent-wrapper');
|
|
17
37
|
const SubClusterWrapper = require('./sub-cluster-wrapper');
|
|
18
38
|
const MessageBus = require('./message-bus');
|
|
19
39
|
const Ledger = require('./ledger');
|
|
20
|
-
const
|
|
40
|
+
const InputHelpers = require('./input-helpers');
|
|
41
|
+
const { detectProvider } = require('./issue-providers');
|
|
21
42
|
const IsolationManager = require('./isolation-manager');
|
|
22
43
|
const { generateName } = require('./name-generator');
|
|
23
44
|
const configValidator = require('./config-validator');
|
|
24
45
|
const TemplateResolver = require('./template-resolver');
|
|
46
|
+
const { loadSettings } = require('../lib/settings');
|
|
47
|
+
const { normalizeProviderName } = require('../lib/provider-names');
|
|
48
|
+
const crypto = require('crypto');
|
|
49
|
+
|
|
50
|
+
function applyModelOverride(agentConfig, modelOverride) {
|
|
51
|
+
if (!modelOverride) return;
|
|
52
|
+
|
|
53
|
+
agentConfig.model = modelOverride;
|
|
54
|
+
if (agentConfig.modelRules) {
|
|
55
|
+
delete agentConfig.modelRules;
|
|
56
|
+
}
|
|
57
|
+
if (agentConfig.modelConfig) {
|
|
58
|
+
delete agentConfig.modelConfig;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
25
61
|
|
|
26
62
|
/**
|
|
27
63
|
* Operation Chain Schema
|
|
@@ -66,10 +102,23 @@ class Orchestrator {
|
|
|
66
102
|
// Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
|
|
67
103
|
this.closed = false;
|
|
68
104
|
|
|
69
|
-
//
|
|
105
|
+
// Track if clusters are loaded (for lazy loading pattern)
|
|
106
|
+
this._clustersLoaded = options.skipLoad === true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Factory method for async initialization
|
|
111
|
+
* Use this instead of `new Orchestrator()` for proper async cluster loading
|
|
112
|
+
* @param {Object} options - Same options as constructor
|
|
113
|
+
* @returns {Promise<Orchestrator>}
|
|
114
|
+
*/
|
|
115
|
+
static async create(options = {}) {
|
|
116
|
+
const instance = new Orchestrator({ ...options, skipLoad: true });
|
|
70
117
|
if (options.skipLoad !== true) {
|
|
71
|
-
|
|
118
|
+
await instance._loadClusters();
|
|
119
|
+
instance._clustersLoaded = true;
|
|
72
120
|
}
|
|
121
|
+
return instance;
|
|
73
122
|
}
|
|
74
123
|
|
|
75
124
|
/**
|
|
@@ -82,12 +131,24 @@ class Orchestrator {
|
|
|
82
131
|
}
|
|
83
132
|
}
|
|
84
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Get input source type for metadata
|
|
136
|
+
* @param {Object} input - Input object
|
|
137
|
+
* @returns {string} Source type: 'github', 'file', or 'text'
|
|
138
|
+
* @private
|
|
139
|
+
*/
|
|
140
|
+
_getInputSource(input) {
|
|
141
|
+
if (input.issue) return 'github';
|
|
142
|
+
if (input.file) return 'file';
|
|
143
|
+
return 'text';
|
|
144
|
+
}
|
|
145
|
+
|
|
85
146
|
/**
|
|
86
147
|
* Load clusters from persistent storage
|
|
87
148
|
* Uses file locking for consistent reads
|
|
88
149
|
* @private
|
|
89
150
|
*/
|
|
90
|
-
_loadClusters() {
|
|
151
|
+
async _loadClusters() {
|
|
91
152
|
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
92
153
|
this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
|
|
93
154
|
|
|
@@ -100,30 +161,20 @@ class Orchestrator {
|
|
|
100
161
|
let release;
|
|
101
162
|
|
|
102
163
|
try {
|
|
103
|
-
//
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const waitMs = retryDelayMs + Math.random() * retryDelayMs;
|
|
118
|
-
const start = Date.now();
|
|
119
|
-
while (Date.now() - start < waitMs) {
|
|
120
|
-
/* spin wait */
|
|
121
|
-
}
|
|
122
|
-
continue;
|
|
123
|
-
}
|
|
124
|
-
throw lockErr;
|
|
125
|
-
}
|
|
126
|
-
}
|
|
164
|
+
// Clean stale locks from crashed processes
|
|
165
|
+
cleanStaleLock(lockfilePath);
|
|
166
|
+
|
|
167
|
+
// Acquire lock with async API (proper retries without CPU spin-wait)
|
|
168
|
+
release = await lockfile.lock(clustersFile, {
|
|
169
|
+
lockfilePath,
|
|
170
|
+
stale: LOCK_STALE_MS,
|
|
171
|
+
retries: {
|
|
172
|
+
retries: 20,
|
|
173
|
+
minTimeout: 100,
|
|
174
|
+
maxTimeout: 200,
|
|
175
|
+
randomize: true,
|
|
176
|
+
},
|
|
177
|
+
});
|
|
127
178
|
|
|
128
179
|
const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
|
|
129
180
|
const clusterIds = Object.keys(data);
|
|
@@ -138,7 +189,9 @@ class Orchestrator {
|
|
|
138
189
|
// Skip clusters whose .db file doesn't exist (orphaned registry entries)
|
|
139
190
|
const dbPath = path.join(this.storageDir, `${clusterId}.db`);
|
|
140
191
|
if (!fs.existsSync(dbPath)) {
|
|
141
|
-
console.warn(
|
|
192
|
+
console.warn(
|
|
193
|
+
`[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
|
|
194
|
+
);
|
|
142
195
|
clustersToRemove.push(clusterId);
|
|
143
196
|
continue;
|
|
144
197
|
}
|
|
@@ -152,8 +205,12 @@ class Orchestrator {
|
|
|
152
205
|
const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
|
|
153
206
|
if (messageCount === 0) {
|
|
154
207
|
console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
|
|
155
|
-
console.warn(
|
|
156
|
-
|
|
208
|
+
console.warn(
|
|
209
|
+
`[Orchestrator] This likely occurred from SIGINT during initialization.`
|
|
210
|
+
);
|
|
211
|
+
console.warn(
|
|
212
|
+
`[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
|
|
213
|
+
);
|
|
157
214
|
corruptedClusters.push(clusterId);
|
|
158
215
|
// Mark cluster as corrupted for visibility in status/list commands
|
|
159
216
|
cluster.state = 'corrupted';
|
|
@@ -168,12 +225,16 @@ class Orchestrator {
|
|
|
168
225
|
delete data[clusterId];
|
|
169
226
|
}
|
|
170
227
|
fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
|
|
171
|
-
this._log(
|
|
228
|
+
this._log(
|
|
229
|
+
`[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
|
|
230
|
+
);
|
|
172
231
|
}
|
|
173
232
|
|
|
174
233
|
// Log summary of corrupted clusters
|
|
175
234
|
if (corruptedClusters.length > 0) {
|
|
176
|
-
console.warn(
|
|
235
|
+
console.warn(
|
|
236
|
+
`\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
|
|
237
|
+
);
|
|
177
238
|
for (const clusterId of corruptedClusters) {
|
|
178
239
|
console.warn(` - ${clusterId}`);
|
|
179
240
|
}
|
|
@@ -186,7 +247,7 @@ class Orchestrator {
|
|
|
186
247
|
console.error(error.stack);
|
|
187
248
|
} finally {
|
|
188
249
|
if (release) {
|
|
189
|
-
release();
|
|
250
|
+
await release();
|
|
190
251
|
}
|
|
191
252
|
}
|
|
192
253
|
}
|
|
@@ -207,8 +268,37 @@ class Orchestrator {
|
|
|
207
268
|
const messageBus = new MessageBus(ledger);
|
|
208
269
|
|
|
209
270
|
// Restore isolation manager FIRST if cluster was running in isolation mode
|
|
271
|
+
const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
|
|
272
|
+
|
|
273
|
+
// Reconstruct agent metadata from config (processes are ephemeral)
|
|
274
|
+
// CRITICAL: Pass isolation context to agents if cluster was running in isolation
|
|
275
|
+
const agents = this._rebuildClusterAgents(
|
|
276
|
+
clusterId,
|
|
277
|
+
clusterData,
|
|
278
|
+
messageBus,
|
|
279
|
+
isolation,
|
|
280
|
+
isolationManager
|
|
281
|
+
);
|
|
282
|
+
|
|
283
|
+
const cluster = {
|
|
284
|
+
...clusterData,
|
|
285
|
+
ledger,
|
|
286
|
+
messageBus,
|
|
287
|
+
agents,
|
|
288
|
+
isolation,
|
|
289
|
+
autoPr: clusterData.autoPr || false,
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
this.clusters.set(clusterId, cluster);
|
|
293
|
+
this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
|
|
294
|
+
|
|
295
|
+
return cluster;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
_restoreClusterIsolation(clusterId, clusterData) {
|
|
210
299
|
let isolation = clusterData.isolation || null;
|
|
211
300
|
let isolationManager = null;
|
|
301
|
+
|
|
212
302
|
if (isolation?.enabled && isolation.containerId) {
|
|
213
303
|
isolationManager = new IsolationManager({ image: isolation.image });
|
|
214
304
|
// Restore the container mapping so cleanup works
|
|
@@ -229,74 +319,84 @@ class Orchestrator {
|
|
|
229
319
|
);
|
|
230
320
|
}
|
|
231
321
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
const agents = [];
|
|
322
|
+
return { isolation, isolationManager };
|
|
323
|
+
}
|
|
235
324
|
|
|
236
|
-
|
|
237
|
-
// This fixes clusters saved before the cwd injection bug was fixed
|
|
325
|
+
_resolveAgentCwd(clusterData) {
|
|
238
326
|
const worktreePath = clusterData.worktree?.path;
|
|
239
327
|
const isolationWorkDir = clusterData.isolation?.workDir;
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
if (clusterData.config?.agents) {
|
|
243
|
-
for (const agentConfig of clusterData.config.agents) {
|
|
244
|
-
// Fix agents that were saved without cwd (pre-bugfix clusters)
|
|
245
|
-
if (!agentConfig.cwd && agentCwd) {
|
|
246
|
-
agentConfig.cwd = agentCwd;
|
|
247
|
-
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
248
|
-
}
|
|
328
|
+
return worktreePath || isolationWorkDir || null;
|
|
329
|
+
}
|
|
249
330
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
331
|
+
_buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
|
|
332
|
+
const agentOptions = {
|
|
333
|
+
id: clusterId,
|
|
334
|
+
quiet: this.quiet,
|
|
335
|
+
modelOverride: clusterData.modelOverride || null,
|
|
336
|
+
};
|
|
254
337
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
}
|
|
338
|
+
if (isolation?.enabled && isolationManager) {
|
|
339
|
+
agentOptions.isolation = {
|
|
340
|
+
enabled: true,
|
|
341
|
+
manager: isolationManager,
|
|
342
|
+
clusterId,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
263
345
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
if (agentConfig.type === 'subcluster') {
|
|
267
|
-
agent = new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
268
|
-
} else {
|
|
269
|
-
agent = new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
270
|
-
}
|
|
346
|
+
return agentOptions;
|
|
347
|
+
}
|
|
271
348
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
agent.state = savedState.state || 'idle';
|
|
277
|
-
agent.iteration = savedState.iteration || 0;
|
|
278
|
-
agent.currentTask = savedState.currentTask || false;
|
|
279
|
-
agent.currentTaskId = savedState.currentTaskId || null;
|
|
280
|
-
agent.processPid = savedState.processPid || null;
|
|
281
|
-
}
|
|
282
|
-
}
|
|
349
|
+
_instantiateAgent(agentConfig, messageBus, clusterId, agentOptions) {
|
|
350
|
+
if (agentConfig.type === 'subcluster') {
|
|
351
|
+
return new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
352
|
+
}
|
|
283
353
|
|
|
284
|
-
|
|
285
|
-
|
|
354
|
+
return new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
_restoreAgentState(agent, agentConfig, clusterData) {
|
|
358
|
+
if (!clusterData.agentStates) return;
|
|
359
|
+
|
|
360
|
+
const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
|
|
361
|
+
if (!savedState) return;
|
|
362
|
+
|
|
363
|
+
agent.state = savedState.state || 'idle';
|
|
364
|
+
agent.iteration = savedState.iteration || 0;
|
|
365
|
+
agent.currentTask = savedState.currentTask || false;
|
|
366
|
+
agent.currentTaskId = savedState.currentTaskId || null;
|
|
367
|
+
agent.processPid = savedState.processPid || null;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
_rebuildClusterAgents(clusterId, clusterData, messageBus, isolation, isolationManager) {
|
|
371
|
+
const agents = [];
|
|
372
|
+
const agentCwd = this._resolveAgentCwd(clusterData);
|
|
373
|
+
|
|
374
|
+
if (!clusterData.config?.agents) {
|
|
375
|
+
return agents;
|
|
286
376
|
}
|
|
287
377
|
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
isolation,
|
|
294
|
-
};
|
|
378
|
+
for (const agentConfig of clusterData.config.agents) {
|
|
379
|
+
if (!agentConfig.cwd && agentCwd) {
|
|
380
|
+
agentConfig.cwd = agentCwd;
|
|
381
|
+
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
382
|
+
}
|
|
295
383
|
|
|
296
|
-
|
|
297
|
-
|
|
384
|
+
if (clusterData.modelOverride) {
|
|
385
|
+
applyModelOverride(agentConfig, clusterData.modelOverride);
|
|
386
|
+
}
|
|
298
387
|
|
|
299
|
-
|
|
388
|
+
const agentOptions = this._buildAgentOptions(
|
|
389
|
+
clusterId,
|
|
390
|
+
clusterData,
|
|
391
|
+
isolation,
|
|
392
|
+
isolationManager
|
|
393
|
+
);
|
|
394
|
+
const agent = this._instantiateAgent(agentConfig, messageBus, clusterId, agentOptions);
|
|
395
|
+
this._restoreAgentState(agent, agentConfig, clusterData);
|
|
396
|
+
agents.push(agent);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return agents;
|
|
300
400
|
}
|
|
301
401
|
|
|
302
402
|
/**
|
|
@@ -316,7 +416,7 @@ class Orchestrator {
|
|
|
316
416
|
* Uses file locking to prevent race conditions with other processes
|
|
317
417
|
* @private
|
|
318
418
|
*/
|
|
319
|
-
_saveClusters() {
|
|
419
|
+
async _saveClusters() {
|
|
320
420
|
// Skip saving if orchestrator is closed (prevents race conditions during cleanup)
|
|
321
421
|
if (this.closed) {
|
|
322
422
|
return;
|
|
@@ -327,30 +427,20 @@ class Orchestrator {
|
|
|
327
427
|
let release;
|
|
328
428
|
|
|
329
429
|
try {
|
|
330
|
-
//
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
const waitMs = retryDelayMs + Math.random() * retryDelayMs * 2;
|
|
345
|
-
const start = Date.now();
|
|
346
|
-
while (Date.now() - start < waitMs) {
|
|
347
|
-
/* spin wait */
|
|
348
|
-
}
|
|
349
|
-
continue;
|
|
350
|
-
}
|
|
351
|
-
throw lockErr;
|
|
352
|
-
}
|
|
353
|
-
}
|
|
430
|
+
// Clean stale locks from crashed processes
|
|
431
|
+
cleanStaleLock(lockfilePath);
|
|
432
|
+
|
|
433
|
+
// Acquire exclusive lock with async API (proper retries without CPU spin-wait)
|
|
434
|
+
release = await lockfile.lock(clustersFile, {
|
|
435
|
+
lockfilePath,
|
|
436
|
+
stale: LOCK_STALE_MS,
|
|
437
|
+
retries: {
|
|
438
|
+
retries: 50,
|
|
439
|
+
minTimeout: 100,
|
|
440
|
+
maxTimeout: 300,
|
|
441
|
+
randomize: true,
|
|
442
|
+
},
|
|
443
|
+
});
|
|
354
444
|
|
|
355
445
|
// Read existing clusters from file (other processes may have added clusters)
|
|
356
446
|
let existingClusters = {};
|
|
@@ -391,6 +481,10 @@ class Orchestrator {
|
|
|
391
481
|
pid: cluster.state === 'running' ? cluster.pid : null,
|
|
392
482
|
// Persist failure info for resume capability
|
|
393
483
|
failureInfo: cluster.failureInfo || null,
|
|
484
|
+
// Persist PR mode for completion agent selection
|
|
485
|
+
autoPr: cluster.autoPr || false,
|
|
486
|
+
// Persist model override for consistent agent spawning on resume
|
|
487
|
+
modelOverride: cluster.modelOverride || null,
|
|
394
488
|
// Persist isolation info (excluding manager instance which can't be serialized)
|
|
395
489
|
// CRITICAL: workDir is required for resume() to recreate container with same workspace
|
|
396
490
|
isolation: cluster.isolation
|
|
@@ -423,7 +517,7 @@ class Orchestrator {
|
|
|
423
517
|
} finally {
|
|
424
518
|
// Always release lock
|
|
425
519
|
if (release) {
|
|
426
|
-
release();
|
|
520
|
+
await release();
|
|
427
521
|
}
|
|
428
522
|
}
|
|
429
523
|
}
|
|
@@ -445,11 +539,14 @@ class Orchestrator {
|
|
|
445
539
|
try {
|
|
446
540
|
if (!fs.existsSync(clustersFile)) return;
|
|
447
541
|
|
|
542
|
+
// Clean stale locks from crashed processes
|
|
543
|
+
cleanStaleLock(lockfilePath);
|
|
544
|
+
|
|
448
545
|
// Try to acquire lock once (polling is best-effort, will retry on next cycle)
|
|
449
546
|
try {
|
|
450
547
|
release = lockfile.lockSync(clustersFile, {
|
|
451
548
|
lockfilePath,
|
|
452
|
-
stale:
|
|
549
|
+
stale: LOCK_STALE_MS,
|
|
453
550
|
});
|
|
454
551
|
} catch (lockErr) {
|
|
455
552
|
// Lock busy - skip this poll cycle, try again next interval
|
|
@@ -541,7 +638,11 @@ class Orchestrator {
|
|
|
541
638
|
isolation: options.isolation || false,
|
|
542
639
|
isolationImage: options.isolationImage,
|
|
543
640
|
worktree: options.worktree || false,
|
|
544
|
-
autoPr: process.env.ZEROSHOT_PR === '1',
|
|
641
|
+
autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
|
|
642
|
+
modelOverride: options.modelOverride, // Model override for all agents
|
|
643
|
+
clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
|
|
644
|
+
settings: options.settings, // User settings for issue provider detection
|
|
645
|
+
forceProvider: options.forceProvider, // Force specific issue provider
|
|
545
646
|
});
|
|
546
647
|
}
|
|
547
648
|
|
|
@@ -550,8 +651,15 @@ class Orchestrator {
|
|
|
550
651
|
* @private
|
|
551
652
|
*/
|
|
552
653
|
async _startInternal(config, input = {}, options = {}) {
|
|
553
|
-
//
|
|
554
|
-
|
|
654
|
+
// Generate a unique cluster ID for this process call.
|
|
655
|
+
// IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
|
|
656
|
+
// - test harnesses may set it globally (breaking multi-start tests)
|
|
657
|
+
// - callers may start multiple clusters in one process
|
|
658
|
+
// Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
|
|
659
|
+
const clusterId = this._generateUniqueClusterId(
|
|
660
|
+
options.clusterId || null,
|
|
661
|
+
config?.dbPath || null
|
|
662
|
+
);
|
|
555
663
|
|
|
556
664
|
// Create ledger and message bus with persistent storage
|
|
557
665
|
const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
|
|
@@ -559,46 +667,11 @@ class Orchestrator {
|
|
|
559
667
|
const messageBus = new MessageBus(ledger);
|
|
560
668
|
|
|
561
669
|
// Handle isolation mode (Docker container OR git worktree)
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
// Check Docker availability
|
|
568
|
-
if (!IsolationManager.isDockerAvailable()) {
|
|
569
|
-
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
570
|
-
}
|
|
571
|
-
|
|
572
|
-
// Ensure image exists (auto-build if missing)
|
|
573
|
-
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
574
|
-
await IsolationManager.ensureImage(image);
|
|
575
|
-
|
|
576
|
-
isolationManager = new IsolationManager({ image });
|
|
577
|
-
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
578
|
-
|
|
579
|
-
// Create container with workspace mounted
|
|
580
|
-
// CRITICAL: Use options.cwd (git repo root) instead of process.cwd()
|
|
581
|
-
const workDir = options.cwd || process.cwd();
|
|
582
|
-
containerId = await isolationManager.createContainer(clusterId, {
|
|
583
|
-
workDir,
|
|
584
|
-
image,
|
|
585
|
-
// Mount configuration (CLI overrides)
|
|
586
|
-
noMounts: options.noMounts,
|
|
587
|
-
mounts: options.mounts,
|
|
588
|
-
containerHome: options.containerHome,
|
|
589
|
-
});
|
|
590
|
-
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
591
|
-
} else if (options.worktree) {
|
|
592
|
-
// Worktree isolation: lightweight git-based isolation (no Docker required)
|
|
593
|
-
const workDir = options.cwd || process.cwd();
|
|
594
|
-
|
|
595
|
-
isolationManager = new IsolationManager({});
|
|
596
|
-
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
|
|
597
|
-
|
|
598
|
-
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
599
|
-
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
600
|
-
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
601
|
-
}
|
|
670
|
+
const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
|
|
671
|
+
options,
|
|
672
|
+
config,
|
|
673
|
+
clusterId
|
|
674
|
+
);
|
|
602
675
|
|
|
603
676
|
// Build cluster object
|
|
604
677
|
// CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
|
|
@@ -621,6 +694,13 @@ class Orchestrator {
|
|
|
621
694
|
// Initialization completion tracking (for safe SIGINT handling)
|
|
622
695
|
initCompletePromise,
|
|
623
696
|
_resolveInitComplete: resolveInitComplete,
|
|
697
|
+
autoPr: options.autoPr || false,
|
|
698
|
+
// Model override for all agents (applied to dynamically added agents)
|
|
699
|
+
modelOverride: options.modelOverride || null,
|
|
700
|
+
// Issue provider tracking (where issue was fetched from)
|
|
701
|
+
issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
|
|
702
|
+
// Git platform tracking (where PR/MR will be created - independent of issue provider)
|
|
703
|
+
gitPlatform: null, // Set when --pr mode is active
|
|
624
704
|
// Isolation state (only if enabled)
|
|
625
705
|
// CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
|
|
626
706
|
isolation: options.isolation
|
|
@@ -648,122 +728,67 @@ class Orchestrator {
|
|
|
648
728
|
this.clusters.set(clusterId, cluster);
|
|
649
729
|
|
|
650
730
|
try {
|
|
651
|
-
// Fetch input (
|
|
731
|
+
// Fetch input (issue from provider, file, or text)
|
|
652
732
|
let inputData;
|
|
653
733
|
if (input.issue) {
|
|
654
|
-
|
|
734
|
+
// Detect provider and fetch issue
|
|
735
|
+
const ProviderClass = detectProvider(
|
|
736
|
+
input.issue,
|
|
737
|
+
options.settings || {},
|
|
738
|
+
options.forceProvider
|
|
739
|
+
);
|
|
740
|
+
if (!ProviderClass) {
|
|
741
|
+
throw new Error(`No issue provider matched input: ${input.issue}`);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
const provider = new ProviderClass();
|
|
745
|
+
inputData = await provider.fetchIssue(input.issue, options.settings || {});
|
|
746
|
+
|
|
747
|
+
// Store issue provider for logging/debugging and cross-provider workflows
|
|
748
|
+
cluster.issueProvider = ProviderClass.id;
|
|
749
|
+
|
|
655
750
|
// Log clickable issue link
|
|
656
751
|
if (inputData.url) {
|
|
657
|
-
this._log(`[Orchestrator] Issue: ${inputData.url}`);
|
|
752
|
+
this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
|
|
658
753
|
}
|
|
754
|
+
} else if (input.file) {
|
|
755
|
+
inputData = InputHelpers.createFileInput(input.file);
|
|
756
|
+
this._log(`[Orchestrator] File: ${input.file}`);
|
|
659
757
|
} else if (input.text) {
|
|
660
|
-
inputData =
|
|
758
|
+
inputData = InputHelpers.createTextInput(input.text);
|
|
661
759
|
} else {
|
|
662
|
-
throw new Error('Either issue or text input is required');
|
|
760
|
+
throw new Error('Either issue, file, or text input is required');
|
|
663
761
|
}
|
|
664
762
|
|
|
665
|
-
//
|
|
763
|
+
// Detect git platform for --pr mode (independent of issue provider)
|
|
666
764
|
if (options.autoPr) {
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
// Load and configure git-pusher agent (use fs.readFileSync to avoid require cache)
|
|
671
|
-
const gitPusherPath = path.join(__dirname, 'agents', 'git-pusher-agent.json');
|
|
672
|
-
const gitPusherConfig = JSON.parse(fs.readFileSync(gitPusherPath, 'utf8'));
|
|
673
|
-
|
|
674
|
-
// Inject issue context placeholders
|
|
675
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
|
|
676
|
-
/\{\{issue_number\}\}/g,
|
|
677
|
-
inputData.number || 'unknown'
|
|
678
|
-
);
|
|
679
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
|
|
680
|
-
/\{\{issue_title\}\}/g,
|
|
681
|
-
inputData.title || 'Implementation'
|
|
682
|
-
);
|
|
765
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
766
|
+
const gitContext = detectGitContext(options.cwd);
|
|
767
|
+
cluster.gitPlatform = gitContext?.provider || null;
|
|
683
768
|
|
|
684
|
-
|
|
685
|
-
|
|
769
|
+
if (cluster.gitPlatform) {
|
|
770
|
+
this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
|
|
771
|
+
}
|
|
686
772
|
}
|
|
687
773
|
|
|
774
|
+
// Inject git-pusher agent if --pr is set (replaces completion-detector)
|
|
775
|
+
this._applyAutoPrConfig(config, inputData, options);
|
|
776
|
+
|
|
688
777
|
// Inject workers instruction if --workers explicitly provided and > 1
|
|
689
|
-
|
|
690
|
-
if (workersCount > 1) {
|
|
691
|
-
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
692
|
-
if (workerAgent) {
|
|
693
|
-
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
694
|
-
|
|
695
|
-
if (!workerAgent.prompt) {
|
|
696
|
-
workerAgent.prompt = instruction;
|
|
697
|
-
} else if (typeof workerAgent.prompt === 'string') {
|
|
698
|
-
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
699
|
-
} else if (workerAgent.prompt.system) {
|
|
700
|
-
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
701
|
-
}
|
|
702
|
-
this._log(
|
|
703
|
-
`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`
|
|
704
|
-
);
|
|
705
|
-
}
|
|
706
|
-
}
|
|
778
|
+
this._applyWorkerInstruction(config);
|
|
707
779
|
|
|
708
780
|
// Initialize agents with optional mock injection
|
|
709
781
|
// Check agent type: regular agent or subcluster
|
|
710
782
|
// CRITICAL: Inject cwd into each agent config for proper working directory
|
|
711
783
|
// In worktree mode, agents run in the worktree path (not original cwd)
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
testMode: options.testMode || !!this.taskRunner, // Enable testMode if taskRunner provided
|
|
721
|
-
quiet: this.quiet,
|
|
722
|
-
};
|
|
723
|
-
|
|
724
|
-
// Inject mock spawn function if provided (legacy mockExecutor API)
|
|
725
|
-
if (options.mockExecutor) {
|
|
726
|
-
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
// TaskRunner DI - new pattern for mocking task execution
|
|
730
|
-
// Creates a mockSpawnFn wrapper that delegates to the taskRunner
|
|
731
|
-
if (this.taskRunner) {
|
|
732
|
-
// CRITICAL: agent is a closure variable capturing the AgentWrapper instance
|
|
733
|
-
// We cannot access agent._selectModel() here because agent doesn't exist yet
|
|
734
|
-
// Solution: Pass a factory function that will be called when agent is available
|
|
735
|
-
agentOptions.taskRunner = this.taskRunner;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
// Pass isolation context if enabled
|
|
739
|
-
if (cluster.isolation) {
|
|
740
|
-
agentOptions.isolation = {
|
|
741
|
-
enabled: true,
|
|
742
|
-
manager: isolationManager,
|
|
743
|
-
clusterId,
|
|
744
|
-
};
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
// Pass worktree context if enabled (lightweight isolation without Docker)
|
|
748
|
-
if (cluster.worktree) {
|
|
749
|
-
agentOptions.worktree = {
|
|
750
|
-
enabled: true,
|
|
751
|
-
path: cluster.worktree.path,
|
|
752
|
-
branch: cluster.worktree.branch,
|
|
753
|
-
repoRoot: cluster.worktree.repoRoot,
|
|
754
|
-
};
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
// Create agent or subcluster wrapper based on type
|
|
758
|
-
let agent;
|
|
759
|
-
if (agentConfig.type === 'subcluster') {
|
|
760
|
-
agent = new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
761
|
-
} else {
|
|
762
|
-
agent = new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
cluster.agents.push(agent);
|
|
766
|
-
}
|
|
784
|
+
this._initializeClusterAgents({
|
|
785
|
+
config,
|
|
786
|
+
cluster,
|
|
787
|
+
messageBus,
|
|
788
|
+
options,
|
|
789
|
+
isolationManager,
|
|
790
|
+
clusterId,
|
|
791
|
+
});
|
|
767
792
|
|
|
768
793
|
// ========================================================================
|
|
769
794
|
// CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
|
|
@@ -777,167 +802,17 @@ class Orchestrator {
|
|
|
777
802
|
//
|
|
778
803
|
// ORDER:
|
|
779
804
|
// 1. Register subscriptions (lines below)
|
|
780
|
-
// 2. Start agents
|
|
781
|
-
// 3. Publish ISSUE_OPENED
|
|
805
|
+
// 2. Start agents
|
|
806
|
+
// 3. Publish ISSUE_OPENED
|
|
782
807
|
//
|
|
783
808
|
// DO NOT move subscriptions after agent.start() - this will reintroduce
|
|
784
809
|
// the race condition fixed in issue #31.
|
|
785
810
|
// ========================================================================
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
handler(message);
|
|
792
|
-
}
|
|
793
|
-
});
|
|
794
|
-
};
|
|
795
|
-
|
|
796
|
-
// Watch for CLUSTER_COMPLETE message to auto-stop
|
|
797
|
-
subscribeToClusterTopic('CLUSTER_COMPLETE', (message) => {
|
|
798
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
799
|
-
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
800
|
-
this._log(`${'='.repeat(80)}`);
|
|
801
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
802
|
-
this._log(`Initiated by: ${message.sender}`);
|
|
803
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
804
|
-
|
|
805
|
-
// Auto-stop cluster
|
|
806
|
-
this.stop(clusterId).catch((err) => {
|
|
807
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
808
|
-
});
|
|
809
|
-
});
|
|
810
|
-
|
|
811
|
-
// Watch for CLUSTER_FAILED message to auto-stop
|
|
812
|
-
subscribeToClusterTopic('CLUSTER_FAILED', (message) => {
|
|
813
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
814
|
-
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
815
|
-
this._log(`${'='.repeat(80)}`);
|
|
816
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
817
|
-
this._log(`Agent: ${message.sender}`);
|
|
818
|
-
if (message.content?.text) {
|
|
819
|
-
this._log(`Details: ${message.content.text}`);
|
|
820
|
-
}
|
|
821
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
822
|
-
|
|
823
|
-
// Auto-stop cluster
|
|
824
|
-
this.stop(clusterId).catch((err) => {
|
|
825
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
826
|
-
});
|
|
827
|
-
});
|
|
828
|
-
|
|
829
|
-
// Watch for AGENT_ERROR - if critical agent fails, stop cluster
|
|
830
|
-
subscribeToClusterTopic('AGENT_ERROR', (message) => {
|
|
831
|
-
const agentRole = message.content?.data?.role;
|
|
832
|
-
const attempts = message.content?.data?.attempts || 1;
|
|
833
|
-
|
|
834
|
-
// Save cluster state to persist failureInfo
|
|
835
|
-
this._saveClusters();
|
|
836
|
-
|
|
837
|
-
// Only stop cluster if non-validator agent exhausted retries
|
|
838
|
-
if (agentRole === 'implementation' && attempts >= 3) {
|
|
839
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
840
|
-
this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
|
|
841
|
-
this._log(`${'='.repeat(80)}`);
|
|
842
|
-
this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
|
|
843
|
-
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
844
|
-
this._log(`Stopping cluster - worker cannot continue`);
|
|
845
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
846
|
-
|
|
847
|
-
// Auto-stop cluster
|
|
848
|
-
this.stop(clusterId).catch((err) => {
|
|
849
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
850
|
-
});
|
|
851
|
-
}
|
|
852
|
-
});
|
|
853
|
-
|
|
854
|
-
// Persist agent state changes for accurate status display
|
|
855
|
-
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
856
|
-
const event = message.content?.data?.event;
|
|
857
|
-
// Save on key state transitions that affect status display
|
|
858
|
-
if (
|
|
859
|
-
['TASK_STARTED', 'TASK_COMPLETED', 'PROCESS_SPAWNED', 'TASK_ID_ASSIGNED', 'STARTED'].includes(
|
|
860
|
-
event
|
|
861
|
-
)
|
|
862
|
-
) {
|
|
863
|
-
this._saveClusters();
|
|
864
|
-
}
|
|
865
|
-
});
|
|
866
|
-
|
|
867
|
-
// Watch for stale agent detection (informational only)
|
|
868
|
-
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
869
|
-
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
870
|
-
|
|
871
|
-
const agentId = message.content?.data?.agent;
|
|
872
|
-
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
873
|
-
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
874
|
-
|
|
875
|
-
this._log(
|
|
876
|
-
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
877
|
-
);
|
|
878
|
-
this._log(` Analysis: ${analysis}`);
|
|
879
|
-
this._log(` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`);
|
|
880
|
-
});
|
|
881
|
-
|
|
882
|
-
// Watch for CLUSTER_OPERATIONS - dynamic agent spawn/removal/update
|
|
883
|
-
subscribeToClusterTopic('CLUSTER_OPERATIONS', (message) => {
|
|
884
|
-
let operations = message.content?.data?.operations;
|
|
885
|
-
|
|
886
|
-
// Parse operations if they came as a JSON string
|
|
887
|
-
if (typeof operations === 'string') {
|
|
888
|
-
try {
|
|
889
|
-
operations = JSON.parse(operations);
|
|
890
|
-
} catch (e) {
|
|
891
|
-
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
892
|
-
return;
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
if (!operations || !Array.isArray(operations)) {
|
|
897
|
-
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
898
|
-
return;
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
902
|
-
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
903
|
-
this._log(`${'='.repeat(80)}`);
|
|
904
|
-
if (message.content?.data?.reasoning) {
|
|
905
|
-
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
906
|
-
}
|
|
907
|
-
this._log(`Operations: ${operations.length}`);
|
|
908
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
909
|
-
|
|
910
|
-
// Execute operation chain
|
|
911
|
-
this._handleOperations(clusterId, operations, message.sender, {
|
|
912
|
-
isolationManager,
|
|
913
|
-
containerId,
|
|
914
|
-
}).catch((err) => {
|
|
915
|
-
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
916
|
-
// Publish failure message
|
|
917
|
-
messageBus.publish({
|
|
918
|
-
cluster_id: clusterId,
|
|
919
|
-
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
920
|
-
sender: 'orchestrator',
|
|
921
|
-
content: {
|
|
922
|
-
text: `Operation chain failed: ${err.message}`,
|
|
923
|
-
data: {
|
|
924
|
-
error: err.message,
|
|
925
|
-
operations: operations,
|
|
926
|
-
},
|
|
927
|
-
},
|
|
928
|
-
});
|
|
929
|
-
|
|
930
|
-
// CRITICAL: Stop cluster on operation failure
|
|
931
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
932
|
-
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
933
|
-
this._log(`${'='.repeat(80)}`);
|
|
934
|
-
this._log(`Error: ${err.message}`);
|
|
935
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
936
|
-
|
|
937
|
-
this.stop(clusterId).catch((stopErr) => {
|
|
938
|
-
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
939
|
-
});
|
|
940
|
-
});
|
|
811
|
+
this._registerClusterSubscriptions({
|
|
812
|
+
messageBus,
|
|
813
|
+
clusterId,
|
|
814
|
+
isolationManager,
|
|
815
|
+
containerId,
|
|
941
816
|
});
|
|
942
817
|
|
|
943
818
|
// Start all agents
|
|
@@ -961,7 +836,7 @@ class Orchestrator {
|
|
|
961
836
|
},
|
|
962
837
|
},
|
|
963
838
|
metadata: {
|
|
964
|
-
source: input
|
|
839
|
+
source: this._getInputSource(input),
|
|
965
840
|
},
|
|
966
841
|
});
|
|
967
842
|
|
|
@@ -989,7 +864,7 @@ class Orchestrator {
|
|
|
989
864
|
// ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
|
|
990
865
|
|
|
991
866
|
// Save cluster to disk
|
|
992
|
-
this._saveClusters();
|
|
867
|
+
await this._saveClusters();
|
|
993
868
|
|
|
994
869
|
return {
|
|
995
870
|
id: clusterId,
|
|
@@ -1009,10 +884,485 @@ class Orchestrator {
|
|
|
1009
884
|
}
|
|
1010
885
|
}
|
|
1011
886
|
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
887
|
+
_initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
|
|
888
|
+
const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
|
|
889
|
+
|
|
890
|
+
for (const agentConfig of config.agents) {
|
|
891
|
+
if (!agentConfig.cwd) {
|
|
892
|
+
agentConfig.cwd = agentCwd;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
if (options.modelOverride) {
|
|
896
|
+
applyModelOverride(agentConfig, options.modelOverride);
|
|
897
|
+
this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
const agentOptions = {
|
|
901
|
+
testMode: options.testMode || !!this.taskRunner,
|
|
902
|
+
quiet: this.quiet,
|
|
903
|
+
modelOverride: options.modelOverride || null,
|
|
904
|
+
};
|
|
905
|
+
|
|
906
|
+
if (options.mockExecutor) {
|
|
907
|
+
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
if (this.taskRunner) {
|
|
911
|
+
agentOptions.taskRunner = this.taskRunner;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
if (cluster.isolation) {
|
|
915
|
+
agentOptions.isolation = {
|
|
916
|
+
enabled: true,
|
|
917
|
+
manager: isolationManager,
|
|
918
|
+
clusterId,
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
if (cluster.worktree) {
|
|
923
|
+
agentOptions.worktree = {
|
|
924
|
+
enabled: true,
|
|
925
|
+
path: cluster.worktree.path,
|
|
926
|
+
branch: cluster.worktree.branch,
|
|
927
|
+
repoRoot: cluster.worktree.repoRoot,
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
const agent =
|
|
932
|
+
agentConfig.type === 'subcluster'
|
|
933
|
+
? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
|
|
934
|
+
: new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
935
|
+
|
|
936
|
+
cluster.agents.push(agent);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
_subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
|
|
941
|
+
messageBus.subscribe((message) => {
|
|
942
|
+
if (message.topic === topic && message.cluster_id === clusterId) {
|
|
943
|
+
handler(message);
|
|
944
|
+
}
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
_registerClusterCompletionHandlers(messageBus, clusterId) {
|
|
949
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
|
|
950
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
951
|
+
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
952
|
+
this._log(`${'='.repeat(80)}`);
|
|
953
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
954
|
+
this._log(`Initiated by: ${message.sender}`);
|
|
955
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
956
|
+
|
|
957
|
+
this.stop(clusterId).catch((err) => {
|
|
958
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
959
|
+
});
|
|
960
|
+
});
|
|
961
|
+
|
|
962
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
|
|
963
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
964
|
+
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
965
|
+
this._log(`${'='.repeat(80)}`);
|
|
966
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
967
|
+
this._log(`Agent: ${message.sender}`);
|
|
968
|
+
if (message.content?.text) {
|
|
969
|
+
this._log(`Details: ${message.content.text}`);
|
|
970
|
+
}
|
|
971
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
972
|
+
|
|
973
|
+
this.stop(clusterId).catch((err) => {
|
|
974
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
975
|
+
});
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
_registerAgentErrorHandler(messageBus, clusterId) {
|
|
980
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
|
|
981
|
+
const agentRole = message.content?.data?.role;
|
|
982
|
+
const attempts = message.content?.data?.attempts || 1;
|
|
983
|
+
|
|
984
|
+
await this._saveClusters();
|
|
985
|
+
|
|
986
|
+
if (agentRole === 'implementation' && attempts >= 3) {
|
|
987
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
988
|
+
this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
|
|
989
|
+
this._log(`${'='.repeat(80)}`);
|
|
990
|
+
this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
|
|
991
|
+
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
992
|
+
this._log(`Stopping cluster - worker cannot continue`);
|
|
993
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
994
|
+
|
|
995
|
+
this.stop(clusterId).catch((err) => {
|
|
996
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
_registerAgentLifecycleHandlers(messageBus, clusterId) {
|
|
1003
|
+
messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
|
|
1004
|
+
const event = message.content?.data?.event;
|
|
1005
|
+
if (
|
|
1006
|
+
[
|
|
1007
|
+
'TASK_STARTED',
|
|
1008
|
+
'TASK_COMPLETED',
|
|
1009
|
+
'PROCESS_SPAWNED',
|
|
1010
|
+
'TASK_ID_ASSIGNED',
|
|
1011
|
+
'STARTED',
|
|
1012
|
+
].includes(event)
|
|
1013
|
+
) {
|
|
1014
|
+
await this._saveClusters();
|
|
1015
|
+
}
|
|
1016
|
+
});
|
|
1017
|
+
|
|
1018
|
+
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
1019
|
+
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
1020
|
+
|
|
1021
|
+
const agentId = message.content?.data?.agent;
|
|
1022
|
+
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
1023
|
+
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
1024
|
+
|
|
1025
|
+
this._log(
|
|
1026
|
+
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
1027
|
+
);
|
|
1028
|
+
this._log(` Analysis: ${analysis}`);
|
|
1029
|
+
this._log(
|
|
1030
|
+
` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
|
|
1031
|
+
);
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
_registerConductorWatchdog(messageBus, clusterId) {
|
|
1036
|
+
const timeoutMs = 30000;
|
|
1037
|
+
let watchdogTimer = null;
|
|
1038
|
+
let completedAt = null;
|
|
1039
|
+
|
|
1040
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
|
|
1041
|
+
const event = message.content?.data?.event;
|
|
1042
|
+
const role = message.content?.data?.role;
|
|
1043
|
+
|
|
1044
|
+
if (event === 'TASK_COMPLETED' && role === 'conductor') {
|
|
1045
|
+
completedAt = Date.now();
|
|
1046
|
+
this._log(
|
|
1047
|
+
`⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
|
|
1048
|
+
);
|
|
1049
|
+
|
|
1050
|
+
watchdogTimer = setTimeout(() => {
|
|
1051
|
+
const clusterOps = messageBus.query({
|
|
1052
|
+
cluster_id: clusterId,
|
|
1053
|
+
topic: 'CLUSTER_OPERATIONS',
|
|
1054
|
+
limit: 1,
|
|
1055
|
+
});
|
|
1056
|
+
if (clusterOps.length === 0) {
|
|
1057
|
+
console.error(`\n${'='.repeat(80)}`);
|
|
1058
|
+
console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
|
|
1059
|
+
console.error(`${'='.repeat(80)}`);
|
|
1060
|
+
console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
|
|
1061
|
+
console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
|
|
1062
|
+
console.error(
|
|
1063
|
+
`Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
|
|
1064
|
+
);
|
|
1065
|
+
console.error(`${'='.repeat(80)}\n`);
|
|
1066
|
+
|
|
1067
|
+
messageBus.publish({
|
|
1068
|
+
cluster_id: clusterId,
|
|
1069
|
+
topic: 'CLUSTER_FAILED',
|
|
1070
|
+
sender: 'orchestrator',
|
|
1071
|
+
content: {
|
|
1072
|
+
text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
|
|
1073
|
+
data: {
|
|
1074
|
+
reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
|
|
1075
|
+
conductorCompletedAt: completedAt,
|
|
1076
|
+
timeoutMs: timeoutMs,
|
|
1077
|
+
},
|
|
1078
|
+
},
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
}, timeoutMs);
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
return {
|
|
1086
|
+
clear: () => {
|
|
1087
|
+
if (!watchdogTimer) {
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
1090
|
+
clearTimeout(watchdogTimer);
|
|
1091
|
+
watchdogTimer = null;
|
|
1092
|
+
const elapsed = completedAt ? Date.now() - completedAt : 0;
|
|
1093
|
+
this._log(
|
|
1094
|
+
`✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
|
|
1095
|
+
);
|
|
1096
|
+
},
|
|
1097
|
+
};
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
_registerClusterOperationsHandler(
|
|
1101
|
+
messageBus,
|
|
1102
|
+
clusterId,
|
|
1103
|
+
isolationManager,
|
|
1104
|
+
containerId,
|
|
1105
|
+
watchdog
|
|
1106
|
+
) {
|
|
1107
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
|
|
1108
|
+
watchdog?.clear?.();
|
|
1109
|
+
|
|
1110
|
+
let operations = message.content?.data?.operations;
|
|
1111
|
+
if (typeof operations === 'string') {
|
|
1112
|
+
try {
|
|
1113
|
+
operations = JSON.parse(operations);
|
|
1114
|
+
} catch (e) {
|
|
1115
|
+
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
if (!operations || !Array.isArray(operations)) {
|
|
1121
|
+
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1126
|
+
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
1127
|
+
this._log(`${'='.repeat(80)}`);
|
|
1128
|
+
if (message.content?.data?.reasoning) {
|
|
1129
|
+
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
1130
|
+
}
|
|
1131
|
+
this._log(`Operations: ${operations.length}`);
|
|
1132
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1133
|
+
|
|
1134
|
+
this._handleOperations(clusterId, operations, message.sender, {
|
|
1135
|
+
isolationManager,
|
|
1136
|
+
containerId,
|
|
1137
|
+
}).catch((err) => {
|
|
1138
|
+
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
1139
|
+
messageBus.publish({
|
|
1140
|
+
cluster_id: clusterId,
|
|
1141
|
+
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
1142
|
+
sender: 'orchestrator',
|
|
1143
|
+
content: {
|
|
1144
|
+
text: `Operation chain failed: ${err.message}`,
|
|
1145
|
+
data: {
|
|
1146
|
+
error: err.message,
|
|
1147
|
+
operations: operations,
|
|
1148
|
+
},
|
|
1149
|
+
},
|
|
1150
|
+
});
|
|
1151
|
+
|
|
1152
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1153
|
+
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
1154
|
+
this._log(`${'='.repeat(80)}`);
|
|
1155
|
+
this._log(`Error: ${err.message}`);
|
|
1156
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1157
|
+
|
|
1158
|
+
this.stop(clusterId).catch((stopErr) => {
|
|
1159
|
+
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
1160
|
+
});
|
|
1161
|
+
});
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
_registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
|
|
1166
|
+
this._registerClusterCompletionHandlers(messageBus, clusterId);
|
|
1167
|
+
this._registerAgentErrorHandler(messageBus, clusterId);
|
|
1168
|
+
this._registerAgentLifecycleHandlers(messageBus, clusterId);
|
|
1169
|
+
|
|
1170
|
+
const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
|
|
1171
|
+
this._registerClusterOperationsHandler(
|
|
1172
|
+
messageBus,
|
|
1173
|
+
clusterId,
|
|
1174
|
+
isolationManager,
|
|
1175
|
+
containerId,
|
|
1176
|
+
watchdog
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
async _initializeIsolation(options, config, clusterId) {
|
|
1181
|
+
let isolationManager = null;
|
|
1182
|
+
let containerId = null;
|
|
1183
|
+
let worktreeInfo = null;
|
|
1184
|
+
|
|
1185
|
+
if (options.isolation) {
|
|
1186
|
+
if (!IsolationManager.isDockerAvailable()) {
|
|
1187
|
+
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
1191
|
+
await IsolationManager.ensureImage(image);
|
|
1192
|
+
|
|
1193
|
+
isolationManager = new IsolationManager({ image });
|
|
1194
|
+
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
1195
|
+
|
|
1196
|
+
const workDir = options.cwd || process.cwd();
|
|
1197
|
+
const providerName = normalizeProviderName(
|
|
1198
|
+
config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
|
|
1199
|
+
);
|
|
1200
|
+
containerId = await isolationManager.createContainer(clusterId, {
|
|
1201
|
+
workDir,
|
|
1202
|
+
image,
|
|
1203
|
+
noMounts: options.noMounts,
|
|
1204
|
+
mounts: options.mounts,
|
|
1205
|
+
containerHome: options.containerHome,
|
|
1206
|
+
provider: providerName,
|
|
1207
|
+
});
|
|
1208
|
+
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
1209
|
+
} else if (options.worktree) {
|
|
1210
|
+
const workDir = options.cwd || process.cwd();
|
|
1211
|
+
|
|
1212
|
+
isolationManager = new IsolationManager({});
|
|
1213
|
+
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
|
|
1214
|
+
|
|
1215
|
+
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
1216
|
+
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
1217
|
+
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
return { isolationManager, containerId, worktreeInfo };
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
_applyAutoPrConfig(config, inputData, options) {
|
|
1224
|
+
if (!options.autoPr) {
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
|
|
1229
|
+
|
|
1230
|
+
// Detect git platform (independent of issue provider)
|
|
1231
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
1232
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
1233
|
+
|
|
1234
|
+
let platform;
|
|
1235
|
+
try {
|
|
1236
|
+
// ALWAYS use git context, regardless of issue provider
|
|
1237
|
+
// This allows Jira issues in GitHub repos, etc.
|
|
1238
|
+
platform = getPlatformForPR(options.cwd);
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
throw new Error(`--pr mode failed: ${error.message}`);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// Check for repo mismatch (issue URL repo vs current git remote)
|
|
1244
|
+
// This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
|
|
1245
|
+
// from a different repo directory
|
|
1246
|
+
let skipCloseIssue = false;
|
|
1247
|
+
const gitContext = detectGitContext(options.cwd);
|
|
1248
|
+
if (inputData.url && gitContext) {
|
|
1249
|
+
const issueHost = this._extractHostFromUrl(inputData.url);
|
|
1250
|
+
const gitHost = gitContext.host;
|
|
1251
|
+
|
|
1252
|
+
// Compare hosts - if different platforms, warn and don't close issue
|
|
1253
|
+
if (issueHost && gitHost && issueHost !== gitHost) {
|
|
1254
|
+
this._log(
|
|
1255
|
+
`[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
|
|
1256
|
+
);
|
|
1257
|
+
this._log(
|
|
1258
|
+
`[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
|
|
1259
|
+
);
|
|
1260
|
+
this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
|
|
1261
|
+
skipCloseIssue = true;
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// Generate platform-specific git-pusher agent from template
|
|
1266
|
+
const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
|
|
1267
|
+
|
|
1268
|
+
if (!isPlatformSupported(platform)) {
|
|
1269
|
+
throw new Error(
|
|
1270
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
1271
|
+
);
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
1275
|
+
|
|
1276
|
+
// Template replacement for issue context
|
|
1277
|
+
const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
|
|
1278
|
+
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
1279
|
+
.replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
|
|
1280
|
+
.replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
|
|
1281
|
+
.replace(/Closes #\{\{issue_number\}\}/g, issueRef);
|
|
1282
|
+
|
|
1283
|
+
config.agents.push(gitPusherConfig);
|
|
1284
|
+
this._log(
|
|
1285
|
+
`[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/**
|
|
1290
|
+
* Extract hostname from a URL
|
|
1291
|
+
* @private
|
|
1292
|
+
*/
|
|
1293
|
+
_extractHostFromUrl(url) {
|
|
1294
|
+
try {
|
|
1295
|
+
const match = url.match(/https?:\/\/([^/]+)/);
|
|
1296
|
+
return match ? match[1] : null;
|
|
1297
|
+
} catch {
|
|
1298
|
+
return null;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
_applyWorkerInstruction(config) {
|
|
1303
|
+
const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
|
|
1304
|
+
if (workersCount <= 1) {
|
|
1305
|
+
return;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
1309
|
+
if (!workerAgent) {
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
1314
|
+
|
|
1315
|
+
if (!workerAgent.prompt) {
|
|
1316
|
+
workerAgent.prompt = instruction;
|
|
1317
|
+
} else if (typeof workerAgent.prompt === 'string') {
|
|
1318
|
+
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
1319
|
+
} else if (workerAgent.prompt.system) {
|
|
1320
|
+
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/**
|
|
1327
|
+
* Generate a unique cluster ID, safe for concurrent starts in-process.
|
|
1328
|
+
* If an explicit ID is provided, uses it as a base and suffixes on collision.
|
|
1329
|
+
* @private
|
|
1330
|
+
*/
|
|
1331
|
+
_generateUniqueClusterId(explicitId, explicitDbPath) {
|
|
1332
|
+
const baseId = explicitId || generateName('cluster');
|
|
1333
|
+
const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
|
|
1334
|
+
|
|
1335
|
+
// Fast path: base is unused.
|
|
1336
|
+
if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
|
|
1337
|
+
return baseId;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
// Collision: suffix with random bytes to avoid race conditions under concurrency.
|
|
1341
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1342
|
+
const suffix = crypto.randomBytes(3).toString('hex');
|
|
1343
|
+
const candidateId = `${baseId}-${suffix}`;
|
|
1344
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1345
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1346
|
+
return candidateId;
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
// Last resort: new generated name (should never happen).
|
|
1351
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1352
|
+
const candidateId = generateName('cluster');
|
|
1353
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1354
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1355
|
+
return candidateId;
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
throw new Error('Failed to generate unique cluster ID after many attempts');
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
/**
|
|
1363
|
+
* Stop a cluster
|
|
1364
|
+
* @param {String} clusterId - Cluster ID
|
|
1365
|
+
*/
|
|
1016
1366
|
async stop(clusterId) {
|
|
1017
1367
|
const cluster = this.clusters.get(clusterId);
|
|
1018
1368
|
if (!cluster) {
|
|
@@ -1040,7 +1390,9 @@ class Orchestrator {
|
|
|
1040
1390
|
// Clean up isolation container if enabled
|
|
1041
1391
|
// CRITICAL: Preserve workspace for resume capability - only delete on kill()
|
|
1042
1392
|
if (cluster.isolation?.manager) {
|
|
1043
|
-
this._log(
|
|
1393
|
+
this._log(
|
|
1394
|
+
`[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
|
|
1395
|
+
);
|
|
1044
1396
|
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
|
|
1045
1397
|
this._log(`[Orchestrator] Container stopped, workspace preserved`);
|
|
1046
1398
|
}
|
|
@@ -1058,7 +1410,7 @@ class Orchestrator {
|
|
|
1058
1410
|
this._log(`Cluster ${clusterId} stopped`);
|
|
1059
1411
|
|
|
1060
1412
|
// Save updated state
|
|
1061
|
-
this._saveClusters();
|
|
1413
|
+
await this._saveClusters();
|
|
1062
1414
|
}
|
|
1063
1415
|
|
|
1064
1416
|
/**
|
|
@@ -1080,7 +1432,9 @@ class Orchestrator {
|
|
|
1080
1432
|
|
|
1081
1433
|
// Force remove isolation container AND workspace (full cleanup, no resume)
|
|
1082
1434
|
if (cluster.isolation?.manager) {
|
|
1083
|
-
this._log(
|
|
1435
|
+
this._log(
|
|
1436
|
+
`[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
|
|
1437
|
+
);
|
|
1084
1438
|
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
|
|
1085
1439
|
this._log(`[Orchestrator] Container and workspace removed`);
|
|
1086
1440
|
}
|
|
@@ -1104,7 +1458,7 @@ class Orchestrator {
|
|
|
1104
1458
|
this._log(`Cluster ${clusterId} killed`);
|
|
1105
1459
|
|
|
1106
1460
|
// Save updated state (will be marked as 'killed' in file)
|
|
1107
|
-
this._saveClusters();
|
|
1461
|
+
await this._saveClusters();
|
|
1108
1462
|
|
|
1109
1463
|
// Now remove from memory after persisting
|
|
1110
1464
|
this.clusters.delete(clusterId);
|
|
@@ -1175,271 +1529,303 @@ class Orchestrator {
|
|
|
1175
1529
|
);
|
|
1176
1530
|
}
|
|
1177
1531
|
|
|
1178
|
-
|
|
1179
|
-
let failureInfo = cluster.failureInfo;
|
|
1532
|
+
const failureInfo = this._resolveFailureInfo(cluster, clusterId);
|
|
1180
1533
|
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
cluster_id: clusterId,
|
|
1185
|
-
topic: 'AGENT_ERROR',
|
|
1186
|
-
limit: 10,
|
|
1187
|
-
});
|
|
1534
|
+
await this._ensureIsolationForResume(clusterId, cluster);
|
|
1535
|
+
this._ensureWorktreeForResume(clusterId, cluster);
|
|
1536
|
+
await this._restartClusterAgents(cluster);
|
|
1188
1537
|
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
agentId: firstError.sender,
|
|
1194
|
-
taskId: firstError.content?.data?.taskId || null,
|
|
1195
|
-
iteration: firstError.content?.data?.iteration || 0,
|
|
1196
|
-
error: firstError.content?.data?.error || firstError.content?.text,
|
|
1197
|
-
timestamp: firstError.timestamp,
|
|
1198
|
-
};
|
|
1199
|
-
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
1200
|
-
}
|
|
1538
|
+
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
1539
|
+
|
|
1540
|
+
if (failureInfo) {
|
|
1541
|
+
return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
|
|
1201
1542
|
}
|
|
1202
1543
|
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
const { spawn } = require('child_process');
|
|
1206
|
-
const oldContainerId = cluster.isolation.containerId;
|
|
1544
|
+
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
|
|
1545
|
+
}
|
|
1207
1546
|
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
const containerExists = await new Promise((resolve) => {
|
|
1213
|
-
checkContainer.on('close', (code) => resolve(code === 0));
|
|
1214
|
-
});
|
|
1547
|
+
_resolveFailureInfo(cluster, clusterId) {
|
|
1548
|
+
if (cluster.failureInfo) {
|
|
1549
|
+
return cluster.failureInfo;
|
|
1550
|
+
}
|
|
1215
1551
|
|
|
1216
|
-
|
|
1217
|
-
|
|
1552
|
+
const errors = cluster.messageBus.query({
|
|
1553
|
+
cluster_id: clusterId,
|
|
1554
|
+
topic: 'AGENT_ERROR',
|
|
1555
|
+
limit: 10,
|
|
1556
|
+
order: 'desc',
|
|
1557
|
+
});
|
|
1218
1558
|
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
if (!workDir) {
|
|
1223
|
-
throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
|
|
1224
|
-
}
|
|
1559
|
+
if (errors.length === 0) {
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1225
1562
|
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1563
|
+
const firstError = errors[0];
|
|
1564
|
+
const failureInfo = {
|
|
1565
|
+
agentId: firstError.sender,
|
|
1566
|
+
taskId: firstError.content?.data?.taskId || null,
|
|
1567
|
+
iteration: firstError.content?.data?.iteration || 0,
|
|
1568
|
+
error: firstError.content?.data?.error || firstError.content?.text,
|
|
1569
|
+
timestamp: firstError.timestamp,
|
|
1570
|
+
};
|
|
1571
|
+
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
1572
|
+
return failureInfo;
|
|
1573
|
+
}
|
|
1234
1574
|
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1575
|
+
_checkContainerExists(containerId) {
|
|
1576
|
+
const { spawn } = require('child_process');
|
|
1577
|
+
const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
|
|
1578
|
+
return new Promise((resolve) => {
|
|
1579
|
+
checkContainer.on('close', (code) => resolve(code === 0));
|
|
1580
|
+
});
|
|
1581
|
+
}
|
|
1240
1582
|
|
|
1241
|
-
|
|
1583
|
+
async _ensureIsolationForResume(clusterId, cluster) {
|
|
1584
|
+
if (!cluster.isolation?.enabled) {
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1242
1587
|
|
|
1243
|
-
|
|
1244
|
-
|
|
1588
|
+
const oldContainerId = cluster.isolation.containerId;
|
|
1589
|
+
const containerExists = await this._checkContainerExists(oldContainerId);
|
|
1590
|
+
if (containerExists) {
|
|
1591
|
+
this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1245
1594
|
|
|
1246
|
-
|
|
1247
|
-
for (const agent of cluster.agents) {
|
|
1248
|
-
if (agent.isolation?.enabled) {
|
|
1249
|
-
agent.isolation.containerId = newContainerId;
|
|
1250
|
-
agent.isolation.manager = cluster.isolation.manager;
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1595
|
+
this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
|
|
1253
1596
|
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
}
|
|
1597
|
+
const workDir = cluster.isolation.workDir;
|
|
1598
|
+
if (!workDir) {
|
|
1599
|
+
throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
|
|
1258
1600
|
}
|
|
1259
1601
|
|
|
1260
|
-
|
|
1261
|
-
if (
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1602
|
+
const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
|
|
1603
|
+
if (!fs.existsSync(isolatedPath)) {
|
|
1604
|
+
throw new Error(
|
|
1605
|
+
`Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
|
|
1606
|
+
`Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
|
|
1607
|
+
);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
const providerName = normalizeProviderName(
|
|
1611
|
+
cluster.config?.forceProvider ||
|
|
1612
|
+
cluster.config?.defaultProvider ||
|
|
1613
|
+
loadSettings().defaultProvider ||
|
|
1614
|
+
'claude'
|
|
1615
|
+
);
|
|
1616
|
+
const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
|
|
1617
|
+
workDir,
|
|
1618
|
+
image: cluster.isolation.image,
|
|
1619
|
+
reuseExistingWorkspace: true,
|
|
1620
|
+
provider: providerName,
|
|
1621
|
+
});
|
|
1622
|
+
|
|
1623
|
+
this._log(`[Orchestrator] New container created: ${newContainerId}`);
|
|
1624
|
+
cluster.isolation.containerId = newContainerId;
|
|
1625
|
+
|
|
1626
|
+
for (const agent of cluster.agents) {
|
|
1627
|
+
if (agent.isolation?.enabled) {
|
|
1628
|
+
agent.isolation.containerId = newContainerId;
|
|
1629
|
+
agent.isolation.manager = cluster.isolation.manager;
|
|
1268
1630
|
}
|
|
1631
|
+
}
|
|
1269
1632
|
|
|
1270
|
-
|
|
1271
|
-
|
|
1633
|
+
this._log(`[Orchestrator] All agents updated with new container ID`);
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
_ensureWorktreeForResume(clusterId, cluster) {
|
|
1637
|
+
if (!cluster.worktree?.enabled) {
|
|
1638
|
+
return;
|
|
1272
1639
|
}
|
|
1273
1640
|
|
|
1274
|
-
|
|
1641
|
+
const worktreePath = cluster.worktree.path;
|
|
1642
|
+
if (!fs.existsSync(worktreePath)) {
|
|
1643
|
+
throw new Error(
|
|
1644
|
+
`Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
|
|
1645
|
+
`Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
|
|
1650
|
+
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
async _restartClusterAgents(cluster) {
|
|
1275
1654
|
cluster.state = 'running';
|
|
1276
1655
|
for (const agent of cluster.agents) {
|
|
1277
1656
|
if (!agent.running) {
|
|
1278
1657
|
await agent.start();
|
|
1279
1658
|
}
|
|
1280
1659
|
}
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
_loadRecentMessages(cluster, clusterId, limit) {
|
|
1663
|
+
return cluster.messageBus
|
|
1664
|
+
.query({
|
|
1665
|
+
cluster_id: clusterId,
|
|
1666
|
+
limit,
|
|
1667
|
+
order: 'desc',
|
|
1668
|
+
})
|
|
1669
|
+
.reverse();
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
_buildResumeContext(recentMessages, prompt, options) {
|
|
1673
|
+
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
1674
|
+
let context = `${options.header}\n\n`;
|
|
1675
|
+
|
|
1676
|
+
if (options.error) {
|
|
1677
|
+
context += `Previous error: ${options.error}\n\n`;
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
context += `## Recent Context\n\n`;
|
|
1681
|
+
for (const msg of recentMessages.slice(-10)) {
|
|
1682
|
+
if (options.topics.includes(msg.topic)) {
|
|
1683
|
+
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
|
|
1688
|
+
return context;
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
_triggerMatchesTopic(triggerTopic, topic) {
|
|
1692
|
+
if (triggerTopic === topic) return true;
|
|
1693
|
+
if (triggerTopic === '*') return true;
|
|
1694
|
+
if (triggerTopic.endsWith('*')) {
|
|
1695
|
+
const prefix = triggerTopic.slice(0, -1);
|
|
1696
|
+
return topic.startsWith(prefix);
|
|
1697
|
+
}
|
|
1698
|
+
return false;
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
_selectAgentsToResume(cluster, lastTrigger) {
|
|
1702
|
+
const agentsToResume = [];
|
|
1703
|
+
|
|
1704
|
+
for (const agent of cluster.agents) {
|
|
1705
|
+
if (!agent.config.triggers) continue;
|
|
1706
|
+
|
|
1707
|
+
const matchingTrigger = agent.config.triggers.find((trigger) =>
|
|
1708
|
+
this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
|
|
1709
|
+
);
|
|
1710
|
+
if (!matchingTrigger) continue;
|
|
1711
|
+
|
|
1712
|
+
if (matchingTrigger.logic?.script) {
|
|
1713
|
+
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
1714
|
+
if (!shouldTrigger) continue;
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
return agentsToResume;
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
_resumeAgents(agentsToResume, context) {
|
|
1724
|
+
for (const { agent, message } of agentsToResume) {
|
|
1725
|
+
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
1726
|
+
agent.resume(context).catch((err) => {
|
|
1727
|
+
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
1728
|
+
});
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
_republishIssue(cluster, clusterId, recentMessages) {
|
|
1733
|
+
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
1734
|
+
if (!issueMessage) {
|
|
1735
|
+
return false;
|
|
1736
|
+
}
|
|
1281
1737
|
|
|
1282
|
-
|
|
1283
|
-
const recentMessages = cluster.messageBus.query({
|
|
1738
|
+
cluster.messageBus.publish({
|
|
1284
1739
|
cluster_id: clusterId,
|
|
1285
|
-
|
|
1740
|
+
topic: 'ISSUE_OPENED',
|
|
1741
|
+
sender: 'system',
|
|
1742
|
+
receiver: 'broadcast',
|
|
1743
|
+
content: issueMessage.content,
|
|
1744
|
+
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
1286
1745
|
});
|
|
1287
1746
|
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
const { agentId, iteration, error } = failureInfo;
|
|
1291
|
-
this._log(
|
|
1292
|
-
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
1293
|
-
);
|
|
1294
|
-
this._log(`[Orchestrator] Previous error: ${error}`);
|
|
1295
|
-
|
|
1296
|
-
// Find the failed agent
|
|
1297
|
-
const failedAgent = cluster.agents.find((a) => a.id === agentId);
|
|
1298
|
-
if (!failedAgent) {
|
|
1299
|
-
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
// Build failure resume context
|
|
1303
|
-
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
1304
|
-
let context = `You are resuming from a previous failed attempt.\n\n`;
|
|
1305
|
-
context += `Previous error: ${error}\n\n`;
|
|
1306
|
-
context += `## Recent Context\n\n`;
|
|
1747
|
+
return true;
|
|
1748
|
+
}
|
|
1307
1749
|
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1750
|
+
async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
|
|
1751
|
+
const { agentId, iteration, error } = failureInfo;
|
|
1752
|
+
this._log(
|
|
1753
|
+
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
1754
|
+
);
|
|
1755
|
+
this._log(`[Orchestrator] Previous error: ${error}`);
|
|
1313
1756
|
|
|
1314
|
-
|
|
1757
|
+
const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
|
|
1758
|
+
if (!failedAgent) {
|
|
1759
|
+
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
1760
|
+
}
|
|
1315
1761
|
|
|
1316
|
-
|
|
1317
|
-
|
|
1762
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1763
|
+
header: 'You are resuming from a previous failed attempt.',
|
|
1764
|
+
error,
|
|
1765
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
|
|
1766
|
+
});
|
|
1318
1767
|
|
|
1319
|
-
|
|
1320
|
-
|
|
1768
|
+
cluster.failureInfo = null;
|
|
1769
|
+
await this._saveClusters();
|
|
1321
1770
|
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
});
|
|
1771
|
+
failedAgent.resume(context).catch((err) => {
|
|
1772
|
+
console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
|
|
1773
|
+
});
|
|
1326
1774
|
|
|
1327
|
-
|
|
1775
|
+
this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
|
|
1328
1776
|
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1777
|
+
return {
|
|
1778
|
+
id: clusterId,
|
|
1779
|
+
state: cluster.state,
|
|
1780
|
+
resumeType: 'failure',
|
|
1781
|
+
resumedAgent: agentId,
|
|
1782
|
+
previousError: error,
|
|
1783
|
+
};
|
|
1784
|
+
}
|
|
1337
1785
|
|
|
1338
|
-
|
|
1786
|
+
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
|
|
1339
1787
|
this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
|
|
1340
1788
|
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
for (const msg of recentMessages.slice(-10)) {
|
|
1347
|
-
if (
|
|
1348
|
-
msg.topic === 'AGENT_OUTPUT' ||
|
|
1349
|
-
msg.topic === 'VALIDATION_RESULT' ||
|
|
1350
|
-
msg.topic === 'ISSUE_OPENED'
|
|
1351
|
-
) {
|
|
1352
|
-
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1353
|
-
}
|
|
1354
|
-
}
|
|
1355
|
-
|
|
1356
|
-
context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
|
|
1789
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1790
|
+
header: 'Resuming cluster from previous session.',
|
|
1791
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
|
|
1792
|
+
});
|
|
1357
1793
|
|
|
1358
|
-
// Find the LAST workflow trigger - not arbitrary last 5 messages
|
|
1359
|
-
// This is the message that indicates current workflow state
|
|
1360
1794
|
const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
|
|
1361
|
-
const agentsToResume = [];
|
|
1362
|
-
|
|
1363
1795
|
if (lastTrigger) {
|
|
1364
1796
|
this._log(
|
|
1365
1797
|
`[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
|
|
1366
1798
|
);
|
|
1367
|
-
|
|
1368
|
-
for (const agent of cluster.agents) {
|
|
1369
|
-
if (!agent.config.triggers) continue;
|
|
1370
|
-
|
|
1371
|
-
const matchingTrigger = agent.config.triggers.find((trigger) => {
|
|
1372
|
-
// Exact match
|
|
1373
|
-
if (trigger.topic === lastTrigger.topic) return true;
|
|
1374
|
-
// Wildcard match
|
|
1375
|
-
if (trigger.topic === '*') return true;
|
|
1376
|
-
// Prefix match (e.g., "VALIDATION_*")
|
|
1377
|
-
if (trigger.topic.endsWith('*')) {
|
|
1378
|
-
const prefix = trigger.topic.slice(0, -1);
|
|
1379
|
-
return lastTrigger.topic.startsWith(prefix);
|
|
1380
|
-
}
|
|
1381
|
-
return false;
|
|
1382
|
-
});
|
|
1383
|
-
|
|
1384
|
-
if (matchingTrigger) {
|
|
1385
|
-
// Evaluate logic script if present
|
|
1386
|
-
if (matchingTrigger.logic?.script) {
|
|
1387
|
-
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
1388
|
-
if (!shouldTrigger) continue;
|
|
1389
|
-
}
|
|
1390
|
-
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
1391
|
-
}
|
|
1392
|
-
}
|
|
1393
1799
|
} else {
|
|
1394
1800
|
this._log(`[Orchestrator] No workflow triggers found in ledger`);
|
|
1395
1801
|
}
|
|
1396
1802
|
|
|
1803
|
+
const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
|
|
1397
1804
|
if (agentsToResume.length === 0) {
|
|
1398
1805
|
if (!lastTrigger) {
|
|
1399
|
-
// No workflow activity - cluster never really started
|
|
1400
1806
|
this._log(
|
|
1401
1807
|
`[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
|
|
1402
1808
|
);
|
|
1403
1809
|
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
1404
1810
|
|
|
1405
|
-
|
|
1406
|
-
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
1407
|
-
if (issueMessage) {
|
|
1408
|
-
cluster.messageBus.publish({
|
|
1409
|
-
cluster_id: clusterId,
|
|
1410
|
-
topic: 'ISSUE_OPENED',
|
|
1411
|
-
sender: 'system',
|
|
1412
|
-
receiver: 'broadcast',
|
|
1413
|
-
content: issueMessage.content,
|
|
1414
|
-
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
1415
|
-
});
|
|
1416
|
-
} else {
|
|
1811
|
+
if (!this._republishIssue(cluster, clusterId, recentMessages)) {
|
|
1417
1812
|
throw new Error(
|
|
1418
1813
|
`Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
|
|
1419
1814
|
`The cluster may not have started properly. Try: zeroshot run <issue> instead.`
|
|
1420
1815
|
);
|
|
1421
1816
|
}
|
|
1422
1817
|
} else {
|
|
1423
|
-
// Had a trigger but no agents matched - something is wrong with agent configs
|
|
1424
1818
|
throw new Error(
|
|
1425
1819
|
`Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
|
|
1426
1820
|
`Check agent trigger configurations.`
|
|
1427
1821
|
);
|
|
1428
1822
|
}
|
|
1429
1823
|
} else {
|
|
1430
|
-
// Resume agents that should run based on ledger state
|
|
1431
1824
|
this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
|
|
1432
|
-
|
|
1433
|
-
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
1434
|
-
agent.resume(context).catch((err) => {
|
|
1435
|
-
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
1436
|
-
});
|
|
1437
|
-
}
|
|
1825
|
+
this._resumeAgents(agentsToResume, context);
|
|
1438
1826
|
}
|
|
1439
1827
|
|
|
1440
|
-
|
|
1441
|
-
this._saveClusters();
|
|
1442
|
-
|
|
1828
|
+
await this._saveClusters();
|
|
1443
1829
|
this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
|
|
1444
1830
|
|
|
1445
1831
|
return {
|
|
@@ -1505,10 +1891,13 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1505
1891
|
`.trim();
|
|
1506
1892
|
|
|
1507
1893
|
// Get recent context from ledger
|
|
1508
|
-
const recentMessages = cluster.messageBus
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1894
|
+
const recentMessages = cluster.messageBus
|
|
1895
|
+
.query({
|
|
1896
|
+
cluster_id: cluster.id,
|
|
1897
|
+
limit: 10,
|
|
1898
|
+
order: 'desc',
|
|
1899
|
+
})
|
|
1900
|
+
.reverse();
|
|
1512
1901
|
|
|
1513
1902
|
const contextText = recentMessages
|
|
1514
1903
|
.map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
|
|
@@ -1555,7 +1944,63 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1555
1944
|
this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
|
|
1556
1945
|
|
|
1557
1946
|
// Phase 1: Pre-validate operation structure
|
|
1947
|
+
const validationErrors = this._validateOperationChain(operations);
|
|
1948
|
+
|
|
1949
|
+
if (validationErrors.length > 0) {
|
|
1950
|
+
const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
|
|
1951
|
+
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
1952
|
+
throw new Error(errorMsg);
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
// Phase 2: Build mock cluster config with proposed agents
|
|
1956
|
+
// Collect all agents that would exist after operations complete
|
|
1957
|
+
const existingAgentConfigs = cluster.config.agents || [];
|
|
1958
|
+
const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
|
|
1959
|
+
|
|
1960
|
+
// Phase 3: Validate proposed cluster config
|
|
1961
|
+
const validation = this._validateProposedConfig(
|
|
1962
|
+
clusterId,
|
|
1963
|
+
cluster,
|
|
1964
|
+
proposedAgentConfigs,
|
|
1965
|
+
operations
|
|
1966
|
+
);
|
|
1967
|
+
|
|
1968
|
+
// Log warnings but proceed
|
|
1969
|
+
if (validation.warnings.length > 0) {
|
|
1970
|
+
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
1971
|
+
for (const warning of validation.warnings) {
|
|
1972
|
+
this._log(` - ${warning}`);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
// Phase 4: Execute validated operations
|
|
1977
|
+
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
1978
|
+
|
|
1979
|
+
await this._executeOperations(cluster, operations, sender, context);
|
|
1980
|
+
|
|
1981
|
+
this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
|
|
1982
|
+
|
|
1983
|
+
// Publish success notification
|
|
1984
|
+
cluster.messageBus.publish({
|
|
1985
|
+
cluster_id: clusterId,
|
|
1986
|
+
topic: 'CLUSTER_OPERATIONS_SUCCESS',
|
|
1987
|
+
sender: 'orchestrator',
|
|
1988
|
+
content: {
|
|
1989
|
+
text: `Executed ${operations.length} operation(s)`,
|
|
1990
|
+
data: {
|
|
1991
|
+
operationCount: operations.length,
|
|
1992
|
+
agentCount: cluster.agents.length,
|
|
1993
|
+
},
|
|
1994
|
+
},
|
|
1995
|
+
});
|
|
1996
|
+
|
|
1997
|
+
// Save updated cluster state to disk
|
|
1998
|
+
await this._saveClusters();
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
_validateOperationChain(operations) {
|
|
1558
2002
|
const validationErrors = [];
|
|
2003
|
+
|
|
1559
2004
|
for (let i = 0; i < operations.length; i++) {
|
|
1560
2005
|
const op = operations[i];
|
|
1561
2006
|
if (!op.action) {
|
|
@@ -1569,21 +2014,15 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1569
2014
|
}
|
|
1570
2015
|
}
|
|
1571
2016
|
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
1575
|
-
throw new Error(errorMsg);
|
|
1576
|
-
}
|
|
2017
|
+
return validationErrors;
|
|
2018
|
+
}
|
|
1577
2019
|
|
|
1578
|
-
|
|
1579
|
-
// Collect all agents that would exist after operations complete
|
|
1580
|
-
const existingAgentConfigs = cluster.config.agents || [];
|
|
2020
|
+
_buildProposedAgentConfigs(existingAgentConfigs, operations) {
|
|
1581
2021
|
const proposedAgentConfigs = [...existingAgentConfigs];
|
|
1582
2022
|
|
|
1583
2023
|
for (const op of operations) {
|
|
1584
2024
|
if (op.action === 'add_agents' && op.agents) {
|
|
1585
2025
|
for (const agentConfig of op.agents) {
|
|
1586
|
-
// Check for duplicate before adding
|
|
1587
2026
|
const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
|
|
1588
2027
|
if (existingIdx === -1) {
|
|
1589
2028
|
proposedAgentConfigs.push(agentConfig);
|
|
@@ -1604,7 +2043,10 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1604
2043
|
}
|
|
1605
2044
|
}
|
|
1606
2045
|
|
|
1607
|
-
|
|
2046
|
+
return proposedAgentConfigs;
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
_validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
|
|
1608
2050
|
const mockConfig = { agents: proposedAgentConfigs };
|
|
1609
2051
|
const validation = configValidator.validateConfig(mockConfig);
|
|
1610
2052
|
|
|
@@ -1630,62 +2072,35 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1630
2072
|
throw new Error(errorMsg);
|
|
1631
2073
|
}
|
|
1632
2074
|
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
1636
|
-
for (const warning of validation.warnings) {
|
|
1637
|
-
this._log(` - ${warning}`);
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
// Phase 4: Execute validated operations
|
|
1642
|
-
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
2075
|
+
return validation;
|
|
2076
|
+
}
|
|
1643
2077
|
|
|
2078
|
+
async _executeOperations(cluster, operations, sender, context) {
|
|
1644
2079
|
for (let i = 0; i < operations.length; i++) {
|
|
1645
2080
|
const op = operations[i];
|
|
1646
2081
|
this._log(` [${i + 1}/${operations.length}] ${op.action}`);
|
|
1647
|
-
|
|
1648
|
-
switch (op.action) {
|
|
1649
|
-
case 'add_agents':
|
|
1650
|
-
await this._opAddAgents(cluster, op, context);
|
|
1651
|
-
break;
|
|
1652
|
-
|
|
1653
|
-
case 'remove_agents':
|
|
1654
|
-
await this._opRemoveAgents(cluster, op);
|
|
1655
|
-
break;
|
|
1656
|
-
|
|
1657
|
-
case 'update_agent':
|
|
1658
|
-
this._opUpdateAgent(cluster, op);
|
|
1659
|
-
break;
|
|
1660
|
-
|
|
1661
|
-
case 'publish':
|
|
1662
|
-
this._opPublish(cluster, op, sender);
|
|
1663
|
-
break;
|
|
1664
|
-
|
|
1665
|
-
case 'load_config':
|
|
1666
|
-
await this._opLoadConfig(cluster, op, context);
|
|
1667
|
-
break;
|
|
1668
|
-
}
|
|
2082
|
+
await this._executeOperation(cluster, op, sender, context);
|
|
1669
2083
|
}
|
|
2084
|
+
}
|
|
1670
2085
|
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
2086
|
+
async _executeOperation(cluster, op, sender, context) {
|
|
2087
|
+
switch (op.action) {
|
|
2088
|
+
case 'add_agents':
|
|
2089
|
+
await this._opAddAgents(cluster, op, context);
|
|
2090
|
+
break;
|
|
2091
|
+
case 'remove_agents':
|
|
2092
|
+
await this._opRemoveAgents(cluster, op);
|
|
2093
|
+
break;
|
|
2094
|
+
case 'update_agent':
|
|
2095
|
+
this._opUpdateAgent(cluster, op);
|
|
2096
|
+
break;
|
|
2097
|
+
case 'publish':
|
|
2098
|
+
this._opPublish(cluster, op, sender);
|
|
2099
|
+
break;
|
|
2100
|
+
case 'load_config':
|
|
2101
|
+
await this._opLoadConfig(cluster, op, context);
|
|
2102
|
+
break;
|
|
2103
|
+
}
|
|
1689
2104
|
}
|
|
1690
2105
|
|
|
1691
2106
|
/**
|
|
@@ -1709,6 +2124,12 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1709
2124
|
agentConfig.cwd = agentCwd;
|
|
1710
2125
|
this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
|
|
1711
2126
|
}
|
|
2127
|
+
|
|
2128
|
+
// Apply model override if set (for consistency with initial agents)
|
|
2129
|
+
if (cluster.modelOverride) {
|
|
2130
|
+
applyModelOverride(agentConfig, cluster.modelOverride);
|
|
2131
|
+
this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
|
|
2132
|
+
}
|
|
1712
2133
|
// Validate agent config has required fields
|
|
1713
2134
|
if (!agentConfig.id) {
|
|
1714
2135
|
throw new Error('Agent config missing required field: id');
|
|
@@ -1731,6 +2152,7 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1731
2152
|
const agentOptions = {
|
|
1732
2153
|
testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
|
|
1733
2154
|
quiet: this.quiet,
|
|
2155
|
+
modelOverride: cluster.modelOverride || null,
|
|
1734
2156
|
};
|
|
1735
2157
|
|
|
1736
2158
|
// TaskRunner DI - propagate to dynamically spawned agents
|
|
@@ -1905,6 +2327,111 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1905
2327
|
await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
|
|
1906
2328
|
|
|
1907
2329
|
this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
|
|
2330
|
+
|
|
2331
|
+
// Inject completion agent (templates don't include one - orchestrator controls termination)
|
|
2332
|
+
await this._injectCompletionAgent(cluster, context);
|
|
2333
|
+
}
|
|
2334
|
+
|
|
2335
|
+
/**
|
|
2336
|
+
* Inject appropriate completion agent based on mode
|
|
2337
|
+
* Templates define work, orchestrator controls termination strategy
|
|
2338
|
+
* @private
|
|
2339
|
+
*/
|
|
2340
|
+
async _injectCompletionAgent(cluster, context) {
|
|
2341
|
+
// Skip if completion agent already exists
|
|
2342
|
+
const hasCompletionAgent = cluster.agents.some(
|
|
2343
|
+
(a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
|
|
2344
|
+
);
|
|
2345
|
+
if (hasCompletionAgent) {
|
|
2346
|
+
return;
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
|
|
2350
|
+
|
|
2351
|
+
if (isPrMode) {
|
|
2352
|
+
// Detect platform from stored cluster metadata OR git context
|
|
2353
|
+
let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
|
|
2354
|
+
|
|
2355
|
+
// Fallback to git context detection if cluster metadata missing
|
|
2356
|
+
if (!platform) {
|
|
2357
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
2358
|
+
try {
|
|
2359
|
+
platform = getPlatformForPR(cluster.cwd || process.cwd());
|
|
2360
|
+
} catch (error) {
|
|
2361
|
+
throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
// Generate platform-specific git-pusher agent from template
|
|
2366
|
+
const {
|
|
2367
|
+
generateGitPusherAgent,
|
|
2368
|
+
isPlatformSupported,
|
|
2369
|
+
} = require('./agents/git-pusher-template');
|
|
2370
|
+
|
|
2371
|
+
if (!isPlatformSupported(platform)) {
|
|
2372
|
+
throw new Error(
|
|
2373
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
2374
|
+
);
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
2378
|
+
|
|
2379
|
+
// Get issue context from ledger
|
|
2380
|
+
const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
|
|
2381
|
+
const issueNumber = issueMsg?.content?.data?.number || 'unknown';
|
|
2382
|
+
const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
|
|
2383
|
+
|
|
2384
|
+
// Inject issue context into prompt
|
|
2385
|
+
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
2386
|
+
.replace(/\{\{issue_number\}\}/g, issueNumber)
|
|
2387
|
+
.replace(/\{\{issue_title\}\}/g, issueTitle);
|
|
2388
|
+
|
|
2389
|
+
await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
|
|
2390
|
+
this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
|
|
2391
|
+
} else {
|
|
2392
|
+
// Default completion-detector
|
|
2393
|
+
const completionDetector = {
|
|
2394
|
+
id: 'completion-detector',
|
|
2395
|
+
role: 'orchestrator',
|
|
2396
|
+
model: 'haiku',
|
|
2397
|
+
timeout: 0,
|
|
2398
|
+
triggers: [
|
|
2399
|
+
{
|
|
2400
|
+
topic: 'VALIDATION_RESULT',
|
|
2401
|
+
logic: {
|
|
2402
|
+
engine: 'javascript',
|
|
2403
|
+
script: `const validators = cluster.getAgentsByRole('validator');
|
|
2404
|
+
const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });
|
|
2405
|
+
if (!lastPush) return false;
|
|
2406
|
+
if (validators.length === 0) return true;
|
|
2407
|
+
|
|
2408
|
+
const validatorIds = new Set(validators.map((v) => v.id));
|
|
2409
|
+
const results = ledger.query({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });
|
|
2410
|
+
|
|
2411
|
+
const latestByValidator = new Map();
|
|
2412
|
+
for (const msg of results) {
|
|
2413
|
+
if (!validatorIds.has(msg.sender)) continue;
|
|
2414
|
+
latestByValidator.set(msg.sender, msg);
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
if (latestByValidator.size < validators.length) return false;
|
|
2418
|
+
|
|
2419
|
+
for (const validator of validators) {
|
|
2420
|
+
const msg = latestByValidator.get(validator.id);
|
|
2421
|
+
const approved = msg?.content?.data?.approved;
|
|
2422
|
+
if (!(approved === true || approved === 'true')) return false;
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
return true;`,
|
|
2426
|
+
},
|
|
2427
|
+
action: 'stop_cluster',
|
|
2428
|
+
},
|
|
2429
|
+
],
|
|
2430
|
+
};
|
|
2431
|
+
|
|
2432
|
+
await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
|
|
2433
|
+
this._log(` Injected completion-detector agent`);
|
|
2434
|
+
}
|
|
1908
2435
|
}
|
|
1909
2436
|
|
|
1910
2437
|
/**
|
|
@@ -2047,7 +2574,7 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
2047
2574
|
* @private
|
|
2048
2575
|
*/
|
|
2049
2576
|
_exportMarkdown(cluster, clusterId, messages) {
|
|
2050
|
-
const {
|
|
2577
|
+
const { parseProviderChunk } = require('./providers');
|
|
2051
2578
|
|
|
2052
2579
|
// Find task info
|
|
2053
2580
|
const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
@@ -2071,100 +2598,164 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
2071
2598
|
md += `## Task\n\n${taskText}\n\n`;
|
|
2072
2599
|
|
|
2073
2600
|
// Group messages by agent for cleaner output
|
|
2601
|
+
const agentOutputs = this._collectAgentOutputs(messages);
|
|
2602
|
+
|
|
2603
|
+
for (const [agentId, agentMsgs] of agentOutputs) {
|
|
2604
|
+
md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
md += this._renderValidationMarkdown(messages);
|
|
2608
|
+
|
|
2609
|
+
// Final status
|
|
2610
|
+
const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
|
|
2611
|
+
if (clusterComplete) {
|
|
2612
|
+
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
2613
|
+
}
|
|
2614
|
+
|
|
2615
|
+
return md;
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
_collectAgentOutputs(messages) {
|
|
2074
2619
|
const agentOutputs = new Map();
|
|
2075
2620
|
|
|
2076
2621
|
for (const msg of messages) {
|
|
2077
|
-
if (msg.topic
|
|
2078
|
-
|
|
2079
|
-
|
|
2622
|
+
if (msg.topic !== 'AGENT_OUTPUT') continue;
|
|
2623
|
+
if (!agentOutputs.has(msg.sender)) {
|
|
2624
|
+
agentOutputs.set(msg.sender, []);
|
|
2625
|
+
}
|
|
2626
|
+
agentOutputs.get(msg.sender).push(msg);
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
return agentOutputs;
|
|
2630
|
+
}
|
|
2631
|
+
|
|
2632
|
+
_extractAgentOutput(agentMsgs, parseProviderChunk) {
|
|
2633
|
+
let text = '';
|
|
2634
|
+
const tools = [];
|
|
2635
|
+
|
|
2636
|
+
for (const msg of agentMsgs) {
|
|
2637
|
+
const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
|
|
2638
|
+
if (!content) continue;
|
|
2639
|
+
|
|
2640
|
+
const provider = normalizeProviderName(
|
|
2641
|
+
msg.content?.data?.provider || msg.sender_provider || 'claude'
|
|
2642
|
+
);
|
|
2643
|
+
const events = parseProviderChunk(provider, content);
|
|
2644
|
+
for (const event of events) {
|
|
2645
|
+
switch (event.type) {
|
|
2646
|
+
case 'text':
|
|
2647
|
+
text += event.text;
|
|
2648
|
+
break;
|
|
2649
|
+
case 'tool_call':
|
|
2650
|
+
tools.push({ name: event.toolName, input: event.input });
|
|
2651
|
+
break;
|
|
2652
|
+
case 'tool_result':
|
|
2653
|
+
if (tools.length > 0) {
|
|
2654
|
+
const lastTool = tools[tools.length - 1];
|
|
2655
|
+
lastTool.result = event.content;
|
|
2656
|
+
lastTool.isError = event.isError;
|
|
2657
|
+
}
|
|
2658
|
+
break;
|
|
2080
2659
|
}
|
|
2081
|
-
agentOutputs.get(msg.sender).push(msg);
|
|
2082
2660
|
}
|
|
2083
2661
|
}
|
|
2084
2662
|
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
switch (event.type) {
|
|
2099
|
-
case 'text':
|
|
2100
|
-
text += event.text;
|
|
2101
|
-
break;
|
|
2102
|
-
case 'tool_call':
|
|
2103
|
-
tools.push({ name: event.toolName, input: event.input });
|
|
2104
|
-
break;
|
|
2105
|
-
case 'tool_result':
|
|
2106
|
-
if (tools.length > 0) {
|
|
2107
|
-
const lastTool = tools[tools.length - 1];
|
|
2108
|
-
lastTool.result = event.content;
|
|
2109
|
-
lastTool.isError = event.isError;
|
|
2110
|
-
}
|
|
2111
|
-
break;
|
|
2112
|
-
}
|
|
2663
|
+
return { text, tools };
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
_renderToolMarkdown(tools) {
|
|
2667
|
+
let md = `### Tools Used\n\n`;
|
|
2668
|
+
|
|
2669
|
+
for (const tool of tools) {
|
|
2670
|
+
const status = tool.isError ? '❌' : '✓';
|
|
2671
|
+
md += `- **${tool.name}** ${status}\n`;
|
|
2672
|
+
if (tool.input) {
|
|
2673
|
+
const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
|
|
2674
|
+
if (inputStr.length < 100) {
|
|
2675
|
+
md += ` - Input: \`${inputStr}\`\n`;
|
|
2113
2676
|
}
|
|
2114
2677
|
}
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
return `${md}\n`;
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
_renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
|
|
2684
|
+
let md = `## Agent: ${agentId}\n\n`;
|
|
2685
|
+
const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
|
|
2686
|
+
|
|
2687
|
+
if (text.trim()) {
|
|
2688
|
+
md += `### Output\n\n${text.trim()}\n\n`;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
if (tools.length > 0) {
|
|
2692
|
+
md += this._renderToolMarkdown(tools);
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
return md;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
_renderCriteriaMarkdown(criteriaResults) {
|
|
2699
|
+
if (!Array.isArray(criteriaResults)) {
|
|
2700
|
+
return '';
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
let md = '';
|
|
2704
|
+
const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
|
|
2705
|
+
if (cannotValidateYet.length > 0) {
|
|
2706
|
+
md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
|
|
2707
|
+
for (const cv of cannotValidateYet) {
|
|
2708
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2709
|
+
}
|
|
2710
|
+
md += '\n';
|
|
2711
|
+
}
|
|
2115
2712
|
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2713
|
+
const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
|
|
2714
|
+
if (cannotValidate.length > 0) {
|
|
2715
|
+
md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
|
|
2716
|
+
for (const cv of cannotValidate) {
|
|
2717
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2119
2718
|
}
|
|
2719
|
+
md += '\n';
|
|
2720
|
+
}
|
|
2120
2721
|
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2722
|
+
return md;
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
_renderValidationEntryMarkdown(validation) {
|
|
2726
|
+
const data = validation.content?.data || {};
|
|
2727
|
+
const approved = data.approved === true || data.approved === 'true';
|
|
2728
|
+
const icon = approved ? '✅' : '❌';
|
|
2729
|
+
let md = `### ${validation.sender} ${icon}\n\n`;
|
|
2730
|
+
|
|
2731
|
+
if (data.summary) {
|
|
2732
|
+
md += `${data.summary}\n\n`;
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
if (!approved && data.issues) {
|
|
2736
|
+
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
2737
|
+
if (Array.isArray(issues) && issues.length > 0) {
|
|
2738
|
+
md += `**Issues:**\n`;
|
|
2739
|
+
for (const issue of issues) {
|
|
2740
|
+
md += `- ${issue}\n`;
|
|
2134
2741
|
}
|
|
2135
2742
|
md += '\n';
|
|
2136
2743
|
}
|
|
2137
2744
|
}
|
|
2138
2745
|
|
|
2139
|
-
|
|
2746
|
+
md += this._renderCriteriaMarkdown(data.criteriaResults);
|
|
2747
|
+
return md;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
_renderValidationMarkdown(messages) {
|
|
2140
2751
|
const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
|
|
2141
|
-
if (validations.length
|
|
2142
|
-
|
|
2143
|
-
for (const v of validations) {
|
|
2144
|
-
const data = v.content?.data || {};
|
|
2145
|
-
const approved = data.approved === true || data.approved === 'true';
|
|
2146
|
-
const icon = approved ? '✅' : '❌';
|
|
2147
|
-
md += `### ${v.sender} ${icon}\n\n`;
|
|
2148
|
-
if (data.summary) {
|
|
2149
|
-
md += `${data.summary}\n\n`;
|
|
2150
|
-
}
|
|
2151
|
-
if (!approved && data.issues) {
|
|
2152
|
-
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
2153
|
-
if (Array.isArray(issues) && issues.length > 0) {
|
|
2154
|
-
md += `**Issues:**\n`;
|
|
2155
|
-
for (const issue of issues) {
|
|
2156
|
-
md += `- ${issue}\n`;
|
|
2157
|
-
}
|
|
2158
|
-
md += '\n';
|
|
2159
|
-
}
|
|
2160
|
-
}
|
|
2161
|
-
}
|
|
2752
|
+
if (validations.length === 0) {
|
|
2753
|
+
return '';
|
|
2162
2754
|
}
|
|
2163
2755
|
|
|
2164
|
-
|
|
2165
|
-
const
|
|
2166
|
-
|
|
2167
|
-
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
2756
|
+
let md = `## Validation Results\n\n`;
|
|
2757
|
+
for (const validation of validations) {
|
|
2758
|
+
md += this._renderValidationEntryMarkdown(validation);
|
|
2168
2759
|
}
|
|
2169
2760
|
|
|
2170
2761
|
return md;
|