@covibes/zeroshot 5.4.0 → 5.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +8 -465
- package/cli/index.js +101 -5231
- package/package.json +16 -130
- package/CHANGELOG.md +0 -537
- package/cli/commands/providers.js +0 -149
- package/cli/lib/first-run.js +0 -211
- package/cli/lib/update-checker.js +0 -281
- package/cli/message-formatter-utils.js +0 -75
- package/cli/message-formatters-normal.js +0 -275
- package/cli/message-formatters-watch.js +0 -181
- package/cluster-templates/base-templates/debug-workflow.json +0 -375
- package/cluster-templates/base-templates/full-workflow.json +0 -581
- package/cluster-templates/base-templates/single-worker.json +0 -63
- package/cluster-templates/base-templates/worker-validator.json +0 -208
- package/cluster-templates/conductor-bootstrap.json +0 -124
- package/docker/zeroshot-cluster/Dockerfile +0 -152
- package/docker/zeroshot-cluster/pre-baked-deps.json +0 -28
- package/lib/completion.js +0 -174
- package/lib/docker-config.js +0 -220
- package/lib/git-remote-utils.js +0 -165
- package/lib/id-detector.js +0 -56
- package/lib/provider-defaults.js +0 -62
- package/lib/provider-detection.js +0 -59
- package/lib/provider-names.js +0 -57
- package/lib/settings/claude-auth.js +0 -78
- package/lib/settings.js +0 -503
- package/lib/stream-json-parser.js +0 -10
- package/scripts/fix-node-pty-permissions.js +0 -75
- package/scripts/record-demo.sh +0 -279
- package/scripts/setup-merge-queue.sh +0 -170
- package/scripts/test-install.sh +0 -40
- package/scripts/validate-templates.js +0 -100
- package/src/agent/agent-config.js +0 -246
- package/src/agent/agent-context-builder.js +0 -444
- package/src/agent/agent-hook-executor.js +0 -599
- package/src/agent/agent-lifecycle.js +0 -743
- package/src/agent/agent-stuck-detector.js +0 -256
- package/src/agent/agent-task-executor.js +0 -1690
- package/src/agent/agent-trigger-evaluator.js +0 -67
- package/src/agent/output-extraction.js +0 -217
- package/src/agent/output-reformatter.js +0 -175
- package/src/agent/schema-utils.js +0 -146
- package/src/agent-wrapper.js +0 -579
- package/src/agents/git-pusher-template.js +0 -285
- package/src/attach/attach-client.js +0 -438
- package/src/attach/attach-server.js +0 -543
- package/src/attach/index.js +0 -35
- package/src/attach/protocol.js +0 -220
- package/src/attach/ring-buffer.js +0 -121
- package/src/attach/socket-discovery.js +0 -267
- package/src/claude-task-runner.js +0 -574
- package/src/config-router.js +0 -80
- package/src/config-validator.js +0 -1925
- package/src/copy-worker.js +0 -43
- package/src/input-helpers.js +0 -65
- package/src/isolation-manager.js +0 -1471
- package/src/issue-providers/README.md +0 -305
- package/src/issue-providers/azure-devops-provider.js +0 -273
- package/src/issue-providers/base-provider.js +0 -232
- package/src/issue-providers/github-provider.js +0 -179
- package/src/issue-providers/gitlab-provider.js +0 -241
- package/src/issue-providers/index.js +0 -196
- package/src/issue-providers/jira-provider.js +0 -239
- package/src/ledger.js +0 -631
- package/src/lib/safe-exec.js +0 -88
- package/src/logic-engine.js +0 -223
- package/src/message-bus-bridge.js +0 -139
- package/src/message-bus.js +0 -249
- package/src/name-generator.js +0 -232
- package/src/orchestrator.js +0 -2815
- package/src/preflight.js +0 -639
- package/src/process-metrics.js +0 -608
- package/src/providers/anthropic/cli-builder.js +0 -73
- package/src/providers/anthropic/index.js +0 -204
- package/src/providers/anthropic/models.js +0 -23
- package/src/providers/anthropic/output-parser.js +0 -177
- package/src/providers/base-provider.js +0 -251
- package/src/providers/capabilities.js +0 -60
- package/src/providers/google/cli-builder.js +0 -55
- package/src/providers/google/index.js +0 -116
- package/src/providers/google/models.js +0 -24
- package/src/providers/google/output-parser.js +0 -101
- package/src/providers/index.js +0 -91
- package/src/providers/openai/cli-builder.js +0 -133
- package/src/providers/openai/index.js +0 -136
- package/src/providers/openai/models.js +0 -21
- package/src/providers/openai/output-parser.js +0 -143
- package/src/providers/opencode/cli-builder.js +0 -42
- package/src/providers/opencode/index.js +0 -103
- package/src/providers/opencode/models.js +0 -55
- package/src/providers/opencode/output-parser.js +0 -122
- package/src/schemas/sub-cluster.js +0 -185
- package/src/status-footer.js +0 -1026
- package/src/sub-cluster-wrapper.js +0 -601
- package/src/task-runner.js +0 -30
- package/src/template-resolver.js +0 -372
- package/src/tui/CHANGES.txt +0 -133
- package/src/tui/LAYOUT.md +0 -261
- package/src/tui/README.txt +0 -192
- package/src/tui/TWO-LEVEL-NAVIGATION.md +0 -186
- package/src/tui/data-poller.js +0 -349
- package/src/tui/demo.js +0 -208
- package/src/tui/formatters.js +0 -129
- package/src/tui/index.js +0 -193
- package/src/tui/keybindings.js +0 -324
- package/src/tui/layout.js +0 -334
- package/src/tui/renderer.js +0 -184
- package/task-lib/attachable-watcher.js +0 -309
- package/task-lib/claude-recovery.js +0 -156
- package/task-lib/commands/clean.js +0 -50
- package/task-lib/commands/episodes.js +0 -105
- package/task-lib/commands/get-log-path.js +0 -23
- package/task-lib/commands/kill.js +0 -32
- package/task-lib/commands/list.js +0 -105
- package/task-lib/commands/logs.js +0 -453
- package/task-lib/commands/resume.js +0 -42
- package/task-lib/commands/run.js +0 -57
- package/task-lib/commands/schedule.js +0 -105
- package/task-lib/commands/scheduler-cmd.js +0 -96
- package/task-lib/commands/schedules.js +0 -148
- package/task-lib/commands/status.js +0 -44
- package/task-lib/commands/unschedule.js +0 -16
- package/task-lib/completion.js +0 -9
- package/task-lib/config.js +0 -8
- package/task-lib/name-generator.js +0 -230
- package/task-lib/package.json +0 -3
- package/task-lib/runner.js +0 -201
- package/task-lib/scheduler.js +0 -252
- package/task-lib/store.js +0 -529
- package/task-lib/tui/formatters.js +0 -215
- package/task-lib/tui/index.js +0 -197
- package/task-lib/tui/layout.js +0 -111
- package/task-lib/tui/renderer.js +0 -129
- package/task-lib/tui.js +0 -425
- package/task-lib/watcher.js +0 -226
package/src/orchestrator.js
DELETED
|
@@ -1,2815 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Orchestrator - Manages cluster lifecycle
|
|
3
|
-
*
|
|
4
|
-
* Provides:
|
|
5
|
-
* - Cluster initialization and configuration
|
|
6
|
-
* - Agent lifecycle management
|
|
7
|
-
* - GitHub issue integration
|
|
8
|
-
* - Cluster state tracking
|
|
9
|
-
* - Crash recovery
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
const fs = require('fs');
|
|
13
|
-
const path = require('path');
|
|
14
|
-
const os = require('os');
|
|
15
|
-
const lockfile = require('proper-lockfile');
|
|
16
|
-
|
|
17
|
-
// Stale lock timeout in ms - if lock file is older than this, delete it
|
|
18
|
-
const LOCK_STALE_MS = 5000;
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Remove lock file if it's stale (older than LOCK_STALE_MS)
|
|
22
|
-
* Handles crashes that leave orphaned lock files
|
|
23
|
-
*/
|
|
24
|
-
function cleanStaleLock(lockPath) {
|
|
25
|
-
try {
|
|
26
|
-
if (fs.existsSync(lockPath)) {
|
|
27
|
-
const age = Date.now() - fs.statSync(lockPath).mtimeMs;
|
|
28
|
-
if (age > LOCK_STALE_MS) {
|
|
29
|
-
fs.unlinkSync(lockPath);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
} catch {
|
|
33
|
-
// Ignore - another process may have cleaned it
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
const AgentWrapper = require('./agent-wrapper');
|
|
37
|
-
const SubClusterWrapper = require('./sub-cluster-wrapper');
|
|
38
|
-
const MessageBus = require('./message-bus');
|
|
39
|
-
const Ledger = require('./ledger');
|
|
40
|
-
const InputHelpers = require('./input-helpers');
|
|
41
|
-
const { detectProvider } = require('./issue-providers');
|
|
42
|
-
const IsolationManager = require('./isolation-manager');
|
|
43
|
-
const { generateName } = require('./name-generator');
|
|
44
|
-
const configValidator = require('./config-validator');
|
|
45
|
-
const TemplateResolver = require('./template-resolver');
|
|
46
|
-
const { loadSettings } = require('../lib/settings');
|
|
47
|
-
const { normalizeProviderName } = require('../lib/provider-names');
|
|
48
|
-
const crypto = require('crypto');
|
|
49
|
-
|
|
50
|
-
function applyModelOverride(agentConfig, modelOverride) {
|
|
51
|
-
if (!modelOverride) return;
|
|
52
|
-
|
|
53
|
-
agentConfig.model = modelOverride;
|
|
54
|
-
if (agentConfig.modelRules) {
|
|
55
|
-
delete agentConfig.modelRules;
|
|
56
|
-
}
|
|
57
|
-
if (agentConfig.modelConfig) {
|
|
58
|
-
delete agentConfig.modelConfig;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Operation Chain Schema
|
|
64
|
-
* Conductor (or any agent) can publish CLUSTER_OPERATIONS to dynamically modify cluster
|
|
65
|
-
*
|
|
66
|
-
* Supported operations:
|
|
67
|
-
* - add_agents: Spawn new agents with given configs
|
|
68
|
-
* - remove_agents: Stop and remove agents by ID
|
|
69
|
-
* - update_agent: Modify existing agent config
|
|
70
|
-
* - publish: Publish a message to the bus
|
|
71
|
-
* - load_config: Load agents from a named cluster config template
|
|
72
|
-
*/
|
|
73
|
-
const VALID_OPERATIONS = ['add_agents', 'remove_agents', 'update_agent', 'publish', 'load_config'];
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Workflow-triggering topics that indicate cluster state progression
|
|
77
|
-
* These are the topics that MATTER for resume - not AGENT_OUTPUT noise
|
|
78
|
-
*/
|
|
79
|
-
const WORKFLOW_TRIGGERS = Object.freeze([
|
|
80
|
-
'ISSUE_OPENED',
|
|
81
|
-
'PLAN_READY',
|
|
82
|
-
'IMPLEMENTATION_READY',
|
|
83
|
-
'VALIDATION_RESULT',
|
|
84
|
-
'CONDUCTOR_ESCALATE',
|
|
85
|
-
]);
|
|
86
|
-
|
|
87
|
-
class Orchestrator {
|
|
88
|
-
constructor(options = {}) {
|
|
89
|
-
this.clusters = new Map(); // cluster_id -> cluster object
|
|
90
|
-
this.quiet = options.quiet || false; // Suppress verbose logging
|
|
91
|
-
|
|
92
|
-
// TaskRunner DI - allows injecting MockTaskRunner for testing
|
|
93
|
-
// When set, passed to all AgentWrappers to control task execution
|
|
94
|
-
this.taskRunner = options.taskRunner || null;
|
|
95
|
-
|
|
96
|
-
// Set up persistent storage directory (can be overridden for testing)
|
|
97
|
-
this.storageDir = options.storageDir || path.join(os.homedir(), '.zeroshot');
|
|
98
|
-
if (!fs.existsSync(this.storageDir)) {
|
|
99
|
-
fs.mkdirSync(this.storageDir, { recursive: true });
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
// Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
|
|
103
|
-
this.closed = false;
|
|
104
|
-
|
|
105
|
-
// Track if clusters are loaded (for lazy loading pattern)
|
|
106
|
-
this._clustersLoaded = options.skipLoad === true;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Factory method for async initialization
|
|
111
|
-
* Use this instead of `new Orchestrator()` for proper async cluster loading
|
|
112
|
-
* @param {Object} options - Same options as constructor
|
|
113
|
-
* @returns {Promise<Orchestrator>}
|
|
114
|
-
*/
|
|
115
|
-
static async create(options = {}) {
|
|
116
|
-
const instance = new Orchestrator({ ...options, skipLoad: true });
|
|
117
|
-
if (options.skipLoad !== true) {
|
|
118
|
-
await instance._loadClusters();
|
|
119
|
-
instance._clustersLoaded = true;
|
|
120
|
-
}
|
|
121
|
-
return instance;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Log message (respects quiet mode)
|
|
126
|
-
* @private
|
|
127
|
-
*/
|
|
128
|
-
_log(...args) {
|
|
129
|
-
if (!this.quiet) {
|
|
130
|
-
console.log(...args);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
/**
|
|
135
|
-
* Get input source type for metadata
|
|
136
|
-
* @param {Object} input - Input object
|
|
137
|
-
* @returns {string} Source type: 'github', 'file', or 'text'
|
|
138
|
-
* @private
|
|
139
|
-
*/
|
|
140
|
-
_getInputSource(input) {
|
|
141
|
-
if (input.issue) return 'github';
|
|
142
|
-
if (input.file) return 'file';
|
|
143
|
-
return 'text';
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
/**
|
|
147
|
-
* Load clusters from persistent storage
|
|
148
|
-
* Uses file locking for consistent reads
|
|
149
|
-
* @private
|
|
150
|
-
*/
|
|
151
|
-
async _loadClusters() {
|
|
152
|
-
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
153
|
-
this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
|
|
154
|
-
|
|
155
|
-
if (!fs.existsSync(clustersFile)) {
|
|
156
|
-
this._log(`[Orchestrator] No clusters file found at ${clustersFile}`);
|
|
157
|
-
return;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
161
|
-
let release;
|
|
162
|
-
|
|
163
|
-
try {
|
|
164
|
-
// Clean stale locks from crashed processes
|
|
165
|
-
cleanStaleLock(lockfilePath);
|
|
166
|
-
|
|
167
|
-
// Acquire lock with async API (proper retries without CPU spin-wait)
|
|
168
|
-
release = await lockfile.lock(clustersFile, {
|
|
169
|
-
lockfilePath,
|
|
170
|
-
stale: LOCK_STALE_MS,
|
|
171
|
-
retries: {
|
|
172
|
-
retries: 20,
|
|
173
|
-
minTimeout: 100,
|
|
174
|
-
maxTimeout: 200,
|
|
175
|
-
randomize: true,
|
|
176
|
-
},
|
|
177
|
-
});
|
|
178
|
-
|
|
179
|
-
const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
|
|
180
|
-
const clusterIds = Object.keys(data);
|
|
181
|
-
this._log(`[Orchestrator] Found ${clusterIds.length} clusters in file:`, clusterIds);
|
|
182
|
-
|
|
183
|
-
// Track clusters to remove (missing .db files or 0 messages)
|
|
184
|
-
const clustersToRemove = [];
|
|
185
|
-
// Track clusters with 0 messages (corrupted from SIGINT race condition)
|
|
186
|
-
const corruptedClusters = [];
|
|
187
|
-
|
|
188
|
-
for (const [clusterId, clusterData] of Object.entries(data)) {
|
|
189
|
-
// Skip clusters whose .db file doesn't exist (orphaned registry entries)
|
|
190
|
-
const dbPath = path.join(this.storageDir, `${clusterId}.db`);
|
|
191
|
-
if (!fs.existsSync(dbPath)) {
|
|
192
|
-
console.warn(
|
|
193
|
-
`[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
|
|
194
|
-
);
|
|
195
|
-
clustersToRemove.push(clusterId);
|
|
196
|
-
continue;
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
this._log(`[Orchestrator] Loading cluster: ${clusterId}`);
|
|
200
|
-
const cluster = this._loadSingleCluster(clusterId, clusterData);
|
|
201
|
-
|
|
202
|
-
// VALIDATION: Detect 0-message clusters (corrupted from SIGINT during initialization)
|
|
203
|
-
// These clusters were created before the initCompletePromise fix was applied
|
|
204
|
-
if (cluster && cluster.messageBus) {
|
|
205
|
-
const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
|
|
206
|
-
if (messageCount === 0) {
|
|
207
|
-
console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
|
|
208
|
-
console.warn(
|
|
209
|
-
`[Orchestrator] This likely occurred from SIGINT during initialization.`
|
|
210
|
-
);
|
|
211
|
-
console.warn(
|
|
212
|
-
`[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
|
|
213
|
-
);
|
|
214
|
-
corruptedClusters.push(clusterId);
|
|
215
|
-
// Mark cluster as corrupted for visibility in status/list commands
|
|
216
|
-
cluster.state = 'corrupted';
|
|
217
|
-
cluster.corruptedReason = 'SIGINT during initialization (0 messages in ledger)';
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
// Clean up orphaned entries from clusters.json
|
|
223
|
-
if (clustersToRemove.length > 0) {
|
|
224
|
-
for (const clusterId of clustersToRemove) {
|
|
225
|
-
delete data[clusterId];
|
|
226
|
-
}
|
|
227
|
-
fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
|
|
228
|
-
this._log(
|
|
229
|
-
`[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
|
|
230
|
-
);
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
// Log summary of corrupted clusters
|
|
234
|
-
if (corruptedClusters.length > 0) {
|
|
235
|
-
console.warn(
|
|
236
|
-
`\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
|
|
237
|
-
);
|
|
238
|
-
for (const clusterId of corruptedClusters) {
|
|
239
|
-
console.warn(` - ${clusterId}`);
|
|
240
|
-
}
|
|
241
|
-
console.warn(`[Orchestrator] Run 'zeroshot clear' to remove all corrupted clusters.\n`);
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
this._log(`[Orchestrator] Total clusters loaded: ${this.clusters.size}`);
|
|
245
|
-
} catch (error) {
|
|
246
|
-
console.error('[Orchestrator] Failed to load clusters:', error.message);
|
|
247
|
-
console.error(error.stack);
|
|
248
|
-
} finally {
|
|
249
|
-
if (release) {
|
|
250
|
-
await release();
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
/**
|
|
256
|
-
* Load a single cluster from data
|
|
257
|
-
* @private
|
|
258
|
-
*/
|
|
259
|
-
_loadSingleCluster(clusterId, clusterData) {
|
|
260
|
-
// Skip if already loaded
|
|
261
|
-
if (this.clusters.has(clusterId)) {
|
|
262
|
-
return this.clusters.get(clusterId);
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
// Restore ledger and message bus
|
|
266
|
-
const dbPath = path.join(this.storageDir, `${clusterId}.db`);
|
|
267
|
-
const ledger = new Ledger(dbPath);
|
|
268
|
-
const messageBus = new MessageBus(ledger);
|
|
269
|
-
|
|
270
|
-
// Restore isolation manager FIRST if cluster was running in isolation mode
|
|
271
|
-
const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
|
|
272
|
-
|
|
273
|
-
// Reconstruct agent metadata from config (processes are ephemeral)
|
|
274
|
-
// CRITICAL: Pass isolation context to agents if cluster was running in isolation
|
|
275
|
-
const agents = this._rebuildClusterAgents(
|
|
276
|
-
clusterId,
|
|
277
|
-
clusterData,
|
|
278
|
-
messageBus,
|
|
279
|
-
isolation,
|
|
280
|
-
isolationManager
|
|
281
|
-
);
|
|
282
|
-
|
|
283
|
-
const cluster = {
|
|
284
|
-
...clusterData,
|
|
285
|
-
ledger,
|
|
286
|
-
messageBus,
|
|
287
|
-
agents,
|
|
288
|
-
isolation,
|
|
289
|
-
autoPr: clusterData.autoPr || false,
|
|
290
|
-
};
|
|
291
|
-
|
|
292
|
-
this.clusters.set(clusterId, cluster);
|
|
293
|
-
this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
|
|
294
|
-
|
|
295
|
-
return cluster;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
_restoreClusterIsolation(clusterId, clusterData) {
|
|
299
|
-
let isolation = clusterData.isolation || null;
|
|
300
|
-
let isolationManager = null;
|
|
301
|
-
|
|
302
|
-
if (isolation?.enabled && isolation.containerId) {
|
|
303
|
-
isolationManager = new IsolationManager({ image: isolation.image });
|
|
304
|
-
// Restore the container mapping so cleanup works
|
|
305
|
-
isolationManager.containers.set(clusterId, isolation.containerId);
|
|
306
|
-
// Restore isolated dir mapping for workspace preservation during cleanup
|
|
307
|
-
if (isolation.workDir) {
|
|
308
|
-
isolationManager.isolatedDirs.set(clusterId, {
|
|
309
|
-
path: path.join(os.tmpdir(), 'zeroshot-isolated', clusterId),
|
|
310
|
-
originalDir: isolation.workDir,
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
isolation = {
|
|
314
|
-
...isolation,
|
|
315
|
-
manager: isolationManager,
|
|
316
|
-
};
|
|
317
|
-
this._log(
|
|
318
|
-
`[Orchestrator] Restored isolation manager for ${clusterId} (container: ${isolation.containerId}, workDir: ${isolation.workDir || 'unknown'})`
|
|
319
|
-
);
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
return { isolation, isolationManager };
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
_resolveAgentCwd(clusterData) {
|
|
326
|
-
const worktreePath = clusterData.worktree?.path;
|
|
327
|
-
const isolationWorkDir = clusterData.isolation?.workDir;
|
|
328
|
-
return worktreePath || isolationWorkDir || null;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
_buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
|
|
332
|
-
const agentOptions = {
|
|
333
|
-
id: clusterId,
|
|
334
|
-
quiet: this.quiet,
|
|
335
|
-
modelOverride: clusterData.modelOverride || null,
|
|
336
|
-
};
|
|
337
|
-
|
|
338
|
-
if (isolation?.enabled && isolationManager) {
|
|
339
|
-
agentOptions.isolation = {
|
|
340
|
-
enabled: true,
|
|
341
|
-
manager: isolationManager,
|
|
342
|
-
clusterId,
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
return agentOptions;
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
_instantiateAgent(agentConfig, messageBus, clusterId, agentOptions) {
|
|
350
|
-
if (agentConfig.type === 'subcluster') {
|
|
351
|
-
return new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
return new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
_restoreAgentState(agent, agentConfig, clusterData) {
|
|
358
|
-
if (!clusterData.agentStates) return;
|
|
359
|
-
|
|
360
|
-
const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
|
|
361
|
-
if (!savedState) return;
|
|
362
|
-
|
|
363
|
-
agent.state = savedState.state || 'idle';
|
|
364
|
-
agent.iteration = savedState.iteration || 0;
|
|
365
|
-
agent.currentTask = savedState.currentTask || false;
|
|
366
|
-
agent.currentTaskId = savedState.currentTaskId || null;
|
|
367
|
-
agent.processPid = savedState.processPid || null;
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
_rebuildClusterAgents(clusterId, clusterData, messageBus, isolation, isolationManager) {
|
|
371
|
-
const agents = [];
|
|
372
|
-
const agentCwd = this._resolveAgentCwd(clusterData);
|
|
373
|
-
|
|
374
|
-
if (!clusterData.config?.agents) {
|
|
375
|
-
return agents;
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
for (const agentConfig of clusterData.config.agents) {
|
|
379
|
-
if (!agentConfig.cwd && agentCwd) {
|
|
380
|
-
agentConfig.cwd = agentCwd;
|
|
381
|
-
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
if (clusterData.modelOverride) {
|
|
385
|
-
applyModelOverride(agentConfig, clusterData.modelOverride);
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
const agentOptions = this._buildAgentOptions(
|
|
389
|
-
clusterId,
|
|
390
|
-
clusterData,
|
|
391
|
-
isolation,
|
|
392
|
-
isolationManager
|
|
393
|
-
);
|
|
394
|
-
const agent = this._instantiateAgent(agentConfig, messageBus, clusterId, agentOptions);
|
|
395
|
-
this._restoreAgentState(agent, agentConfig, clusterData);
|
|
396
|
-
agents.push(agent);
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
return agents;
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
/**
|
|
403
|
-
* Ensure clusters file exists (required for file locking)
|
|
404
|
-
* @private
|
|
405
|
-
*/
|
|
406
|
-
_ensureClustersFile() {
|
|
407
|
-
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
408
|
-
if (!fs.existsSync(clustersFile)) {
|
|
409
|
-
fs.writeFileSync(clustersFile, '{}');
|
|
410
|
-
}
|
|
411
|
-
return clustersFile;
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
/**
|
|
415
|
-
* Save clusters to persistent storage
|
|
416
|
-
* Uses file locking to prevent race conditions with other processes
|
|
417
|
-
* @private
|
|
418
|
-
*/
|
|
419
|
-
async _saveClusters() {
|
|
420
|
-
// Skip saving if orchestrator is closed (prevents race conditions during cleanup)
|
|
421
|
-
if (this.closed) {
|
|
422
|
-
return;
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
const clustersFile = this._ensureClustersFile();
|
|
426
|
-
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
427
|
-
let release;
|
|
428
|
-
|
|
429
|
-
try {
|
|
430
|
-
// Clean stale locks from crashed processes
|
|
431
|
-
cleanStaleLock(lockfilePath);
|
|
432
|
-
|
|
433
|
-
// Acquire exclusive lock with async API (proper retries without CPU spin-wait)
|
|
434
|
-
release = await lockfile.lock(clustersFile, {
|
|
435
|
-
lockfilePath,
|
|
436
|
-
stale: LOCK_STALE_MS,
|
|
437
|
-
retries: {
|
|
438
|
-
retries: 50,
|
|
439
|
-
minTimeout: 100,
|
|
440
|
-
maxTimeout: 300,
|
|
441
|
-
randomize: true,
|
|
442
|
-
},
|
|
443
|
-
});
|
|
444
|
-
|
|
445
|
-
// Read existing clusters from file (other processes may have added clusters)
|
|
446
|
-
let existingClusters = {};
|
|
447
|
-
try {
|
|
448
|
-
const content = fs.readFileSync(clustersFile, 'utf8');
|
|
449
|
-
existingClusters = JSON.parse(content);
|
|
450
|
-
} catch (error) {
|
|
451
|
-
console.error('[Orchestrator] Failed to read existing clusters:', error.message);
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
// Merge: update/add clusters from this process
|
|
455
|
-
for (const [clusterId, cluster] of this.clusters.entries()) {
|
|
456
|
-
// CRITICAL: Only update clusters this process actually owns or has modified
|
|
457
|
-
// A process owns a cluster if: it started it (pid matches) OR it explicitly stopped/killed it
|
|
458
|
-
const isOwnedByThisProcess = cluster.pid === process.pid;
|
|
459
|
-
const wasModifiedByThisProcess = cluster.state === 'stopped' || cluster.state === 'killed';
|
|
460
|
-
|
|
461
|
-
// Skip clusters we don't own and haven't modified - prevents race condition
|
|
462
|
-
// where a running cluster overwrites another process's stop/kill operation
|
|
463
|
-
if (!isOwnedByThisProcess && !wasModifiedByThisProcess) {
|
|
464
|
-
// Preserve existing state from file for clusters we don't own
|
|
465
|
-
continue;
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// CRITICAL: Killed clusters are DELETED from disk, not persisted
|
|
469
|
-
// This ensures they can't be accidentally resumed
|
|
470
|
-
if (cluster.state === 'killed') {
|
|
471
|
-
delete existingClusters[clusterId];
|
|
472
|
-
continue;
|
|
473
|
-
}
|
|
474
|
-
|
|
475
|
-
existingClusters[clusterId] = {
|
|
476
|
-
id: cluster.id,
|
|
477
|
-
config: cluster.config,
|
|
478
|
-
state: cluster.state,
|
|
479
|
-
createdAt: cluster.createdAt,
|
|
480
|
-
// Track PID for zombie detection (null if cluster is stopped/killed)
|
|
481
|
-
pid: cluster.state === 'running' ? cluster.pid : null,
|
|
482
|
-
// Persist failure info for resume capability
|
|
483
|
-
failureInfo: cluster.failureInfo || null,
|
|
484
|
-
// Persist PR mode for completion agent selection
|
|
485
|
-
autoPr: cluster.autoPr || false,
|
|
486
|
-
// Persist model override for consistent agent spawning on resume
|
|
487
|
-
modelOverride: cluster.modelOverride || null,
|
|
488
|
-
// Persist isolation info (excluding manager instance which can't be serialized)
|
|
489
|
-
// CRITICAL: workDir is required for resume() to recreate container with same workspace
|
|
490
|
-
isolation: cluster.isolation
|
|
491
|
-
? {
|
|
492
|
-
enabled: cluster.isolation.enabled,
|
|
493
|
-
containerId: cluster.isolation.containerId,
|
|
494
|
-
image: cluster.isolation.image,
|
|
495
|
-
workDir: cluster.isolation.workDir, // Required for resume
|
|
496
|
-
}
|
|
497
|
-
: null,
|
|
498
|
-
// Persist agent runtime states for accurate status display from other processes
|
|
499
|
-
agentStates: cluster.agents
|
|
500
|
-
? cluster.agents.map((a) => ({
|
|
501
|
-
id: a.id,
|
|
502
|
-
state: a.state,
|
|
503
|
-
iteration: a.iteration,
|
|
504
|
-
currentTask: a.currentTask ? true : false,
|
|
505
|
-
currentTaskId: a.currentTaskId,
|
|
506
|
-
processPid: a.processPid,
|
|
507
|
-
}))
|
|
508
|
-
: null,
|
|
509
|
-
};
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
// Write merged data
|
|
513
|
-
fs.writeFileSync(clustersFile, JSON.stringify(existingClusters, null, 2));
|
|
514
|
-
this._log(
|
|
515
|
-
`[Orchestrator] Saved ${this.clusters.size} cluster(s), file now has ${Object.keys(existingClusters).length} total`
|
|
516
|
-
);
|
|
517
|
-
} finally {
|
|
518
|
-
// Always release lock
|
|
519
|
-
if (release) {
|
|
520
|
-
await release();
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
/**
|
|
526
|
-
* Watch for new clusters and call callback when found
|
|
527
|
-
* Polls the clusters file for changes with file locking
|
|
528
|
-
* @param {Function} onNewCluster - Callback(cluster) for each new cluster
|
|
529
|
-
* @param {Number} intervalMs - Poll interval in ms (default: 2000)
|
|
530
|
-
* @returns {Function} Stop function to cancel watching
|
|
531
|
-
*/
|
|
532
|
-
watchForNewClusters(onNewCluster, intervalMs = 2000) {
|
|
533
|
-
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
534
|
-
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
535
|
-
const knownClusterIds = new Set(this.clusters.keys());
|
|
536
|
-
|
|
537
|
-
const intervalId = setInterval(() => {
|
|
538
|
-
let release;
|
|
539
|
-
try {
|
|
540
|
-
if (!fs.existsSync(clustersFile)) return;
|
|
541
|
-
|
|
542
|
-
// Clean stale locks from crashed processes
|
|
543
|
-
cleanStaleLock(lockfilePath);
|
|
544
|
-
|
|
545
|
-
// Try to acquire lock once (polling is best-effort, will retry on next cycle)
|
|
546
|
-
try {
|
|
547
|
-
release = lockfile.lockSync(clustersFile, {
|
|
548
|
-
lockfilePath,
|
|
549
|
-
stale: LOCK_STALE_MS,
|
|
550
|
-
});
|
|
551
|
-
} catch (lockErr) {
|
|
552
|
-
// Lock busy - skip this poll cycle, try again next interval
|
|
553
|
-
if (lockErr.code === 'ELOCKED') return;
|
|
554
|
-
throw lockErr;
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
|
|
558
|
-
|
|
559
|
-
for (const [clusterId, clusterData] of Object.entries(data)) {
|
|
560
|
-
if (!knownClusterIds.has(clusterId)) {
|
|
561
|
-
// New cluster found
|
|
562
|
-
knownClusterIds.add(clusterId);
|
|
563
|
-
const cluster = this._loadSingleCluster(clusterId, clusterData);
|
|
564
|
-
if (cluster && onNewCluster) {
|
|
565
|
-
onNewCluster(cluster);
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
} catch (error) {
|
|
570
|
-
// File access during polling can fail transiently - log and continue
|
|
571
|
-
console.error(`[Orchestrator] watchForNewClusters error (will retry): ${error.message}`);
|
|
572
|
-
} finally {
|
|
573
|
-
if (release) {
|
|
574
|
-
release();
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
}, intervalMs);
|
|
578
|
-
|
|
579
|
-
// Return stop function
|
|
580
|
-
return () => clearInterval(intervalId);
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
/**
|
|
584
|
-
* Start a new cluster with mocked agent executors (TESTING ONLY)
|
|
585
|
-
*
|
|
586
|
-
* CRITICAL: This method PREVENTS real Claude API calls.
|
|
587
|
-
* All agent behaviors must be defined in mockExecutor.
|
|
588
|
-
*
|
|
589
|
-
* @param {Object} config - Cluster configuration
|
|
590
|
-
* @param {Object} input - Input source { issue, text, or config }
|
|
591
|
-
* @param {MockAgentExecutor} mockExecutor - Mock executor with agent behaviors
|
|
592
|
-
* @returns {Object} Cluster object
|
|
593
|
-
*/
|
|
594
|
-
startWithMock(config, input, mockExecutor) {
|
|
595
|
-
if (!mockExecutor) {
|
|
596
|
-
throw new Error('Orchestrator.startWithMock: mockExecutor is required');
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
// Validate all agents that execute tasks have mock behaviors defined
|
|
600
|
-
// Orchestrator agents (action: 'stop_cluster') don't execute tasks, so don't need mocks
|
|
601
|
-
for (const agentConfig of config.agents) {
|
|
602
|
-
const agentId = agentConfig.id;
|
|
603
|
-
|
|
604
|
-
// Check if agent has any triggers that execute tasks
|
|
605
|
-
const executesTask = agentConfig.triggers?.some(
|
|
606
|
-
(trigger) => !trigger.action || trigger.action === 'execute_task'
|
|
607
|
-
);
|
|
608
|
-
|
|
609
|
-
if (executesTask && !mockExecutor.behaviors[agentId]) {
|
|
610
|
-
throw new Error(
|
|
611
|
-
`Orchestrator.startWithMock: No behavior defined for agent '${agentId}'. ` +
|
|
612
|
-
`This would cause real Claude API calls. ABORTING.\n` +
|
|
613
|
-
`Available behaviors: ${Object.keys(mockExecutor.behaviors).join(', ')}`
|
|
614
|
-
);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
|
|
618
|
-
return this._startInternal(config, input, {
|
|
619
|
-
mockExecutor,
|
|
620
|
-
testMode: true,
|
|
621
|
-
});
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
/**
|
|
625
|
-
* Start a new cluster
|
|
626
|
-
* @param {Object} config - Cluster configuration
|
|
627
|
-
* @param {Object} input - Input source { issue, text, or config }
|
|
628
|
-
* @param {Object} options - Start options
|
|
629
|
-
* @param {boolean} options.isolation - Run in Docker container
|
|
630
|
-
* @param {string} options.isolationImage - Docker image to use
|
|
631
|
-
* @param {boolean} options.worktree - Run in git worktree isolation (lightweight, no Docker)
|
|
632
|
-
* @returns {Object} Cluster object
|
|
633
|
-
*/
|
|
634
|
-
start(config, input = {}, options = {}) {
|
|
635
|
-
return this._startInternal(config, input, {
|
|
636
|
-
testMode: false,
|
|
637
|
-
cwd: options.cwd || process.cwd(), // Target working directory for agents
|
|
638
|
-
isolation: options.isolation || false,
|
|
639
|
-
isolationImage: options.isolationImage,
|
|
640
|
-
worktree: options.worktree || false,
|
|
641
|
-
autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
|
|
642
|
-
modelOverride: options.modelOverride, // Model override for all agents
|
|
643
|
-
clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
|
|
644
|
-
settings: options.settings, // User settings for issue provider detection
|
|
645
|
-
forceProvider: options.forceProvider, // Force specific issue provider
|
|
646
|
-
});
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Internal start implementation (shared by start and startWithMock)
|
|
651
|
-
* @private
|
|
652
|
-
*/
|
|
653
|
-
async _startInternal(config, input = {}, options = {}) {
|
|
654
|
-
// Generate a unique cluster ID for this process call.
|
|
655
|
-
// IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
|
|
656
|
-
// - test harnesses may set it globally (breaking multi-start tests)
|
|
657
|
-
// - callers may start multiple clusters in one process
|
|
658
|
-
// Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
|
|
659
|
-
const clusterId = this._generateUniqueClusterId(
|
|
660
|
-
options.clusterId || null,
|
|
661
|
-
config?.dbPath || null
|
|
662
|
-
);
|
|
663
|
-
|
|
664
|
-
// Create ledger and message bus with persistent storage
|
|
665
|
-
const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
|
|
666
|
-
const ledger = new Ledger(dbPath);
|
|
667
|
-
const messageBus = new MessageBus(ledger);
|
|
668
|
-
|
|
669
|
-
// Handle isolation mode (Docker container OR git worktree)
|
|
670
|
-
const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
|
|
671
|
-
options,
|
|
672
|
-
config,
|
|
673
|
-
clusterId
|
|
674
|
-
);
|
|
675
|
-
|
|
676
|
-
// Build cluster object
|
|
677
|
-
// CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
|
|
678
|
-
// This prevents 0-message clusters from SIGINT during async initialization
|
|
679
|
-
let resolveInitComplete;
|
|
680
|
-
const initCompletePromise = new Promise((resolve) => {
|
|
681
|
-
resolveInitComplete = resolve;
|
|
682
|
-
});
|
|
683
|
-
|
|
684
|
-
const cluster = {
|
|
685
|
-
id: clusterId,
|
|
686
|
-
config,
|
|
687
|
-
state: 'initializing',
|
|
688
|
-
messageBus,
|
|
689
|
-
ledger,
|
|
690
|
-
agents: [],
|
|
691
|
-
createdAt: Date.now(),
|
|
692
|
-
// Track PID for zombie detection (this process owns the cluster)
|
|
693
|
-
pid: process.pid,
|
|
694
|
-
// Initialization completion tracking (for safe SIGINT handling)
|
|
695
|
-
initCompletePromise,
|
|
696
|
-
_resolveInitComplete: resolveInitComplete,
|
|
697
|
-
autoPr: options.autoPr || false,
|
|
698
|
-
// Model override for all agents (applied to dynamically added agents)
|
|
699
|
-
modelOverride: options.modelOverride || null,
|
|
700
|
-
// Issue provider tracking (where issue was fetched from)
|
|
701
|
-
issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
|
|
702
|
-
// Git platform tracking (where PR/MR will be created - independent of issue provider)
|
|
703
|
-
gitPlatform: null, // Set when --pr mode is active
|
|
704
|
-
// Isolation state (only if enabled)
|
|
705
|
-
// CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
|
|
706
|
-
isolation: options.isolation
|
|
707
|
-
? {
|
|
708
|
-
enabled: true,
|
|
709
|
-
containerId,
|
|
710
|
-
image: options.isolationImage || 'zeroshot-cluster-base',
|
|
711
|
-
manager: isolationManager,
|
|
712
|
-
workDir: options.cwd || process.cwd(), // Persisted for resume
|
|
713
|
-
}
|
|
714
|
-
: null,
|
|
715
|
-
// Worktree isolation state (lightweight alternative to Docker)
|
|
716
|
-
worktree: options.worktree
|
|
717
|
-
? {
|
|
718
|
-
enabled: true,
|
|
719
|
-
path: worktreeInfo.path,
|
|
720
|
-
branch: worktreeInfo.branch,
|
|
721
|
-
repoRoot: worktreeInfo.repoRoot,
|
|
722
|
-
manager: isolationManager,
|
|
723
|
-
workDir: options.cwd || process.cwd(), // Persisted for resume
|
|
724
|
-
}
|
|
725
|
-
: null,
|
|
726
|
-
};
|
|
727
|
-
|
|
728
|
-
this.clusters.set(clusterId, cluster);
|
|
729
|
-
|
|
730
|
-
try {
|
|
731
|
-
// Fetch input (issue from provider, file, or text)
|
|
732
|
-
let inputData;
|
|
733
|
-
if (input.issue) {
|
|
734
|
-
// Detect provider and fetch issue
|
|
735
|
-
const ProviderClass = detectProvider(
|
|
736
|
-
input.issue,
|
|
737
|
-
options.settings || {},
|
|
738
|
-
options.forceProvider
|
|
739
|
-
);
|
|
740
|
-
if (!ProviderClass) {
|
|
741
|
-
throw new Error(`No issue provider matched input: ${input.issue}`);
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
const provider = new ProviderClass();
|
|
745
|
-
inputData = await provider.fetchIssue(input.issue, options.settings || {});
|
|
746
|
-
|
|
747
|
-
// Store issue provider for logging/debugging and cross-provider workflows
|
|
748
|
-
cluster.issueProvider = ProviderClass.id;
|
|
749
|
-
|
|
750
|
-
// Log clickable issue link
|
|
751
|
-
if (inputData.url) {
|
|
752
|
-
this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
|
|
753
|
-
}
|
|
754
|
-
} else if (input.file) {
|
|
755
|
-
inputData = InputHelpers.createFileInput(input.file);
|
|
756
|
-
this._log(`[Orchestrator] File: ${input.file}`);
|
|
757
|
-
} else if (input.text) {
|
|
758
|
-
inputData = InputHelpers.createTextInput(input.text);
|
|
759
|
-
} else {
|
|
760
|
-
throw new Error('Either issue, file, or text input is required');
|
|
761
|
-
}
|
|
762
|
-
|
|
763
|
-
// Detect git platform for --pr mode (independent of issue provider)
|
|
764
|
-
if (options.autoPr) {
|
|
765
|
-
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
766
|
-
const gitContext = detectGitContext(options.cwd);
|
|
767
|
-
cluster.gitPlatform = gitContext?.provider || null;
|
|
768
|
-
|
|
769
|
-
if (cluster.gitPlatform) {
|
|
770
|
-
this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
// Inject git-pusher agent if --pr is set (replaces completion-detector)
|
|
775
|
-
this._applyAutoPrConfig(config, inputData, options);
|
|
776
|
-
|
|
777
|
-
// Inject workers instruction if --workers explicitly provided and > 1
|
|
778
|
-
this._applyWorkerInstruction(config);
|
|
779
|
-
|
|
780
|
-
// Initialize agents with optional mock injection
|
|
781
|
-
// Check agent type: regular agent or subcluster
|
|
782
|
-
// CRITICAL: Inject cwd into each agent config for proper working directory
|
|
783
|
-
// In worktree mode, agents run in the worktree path (not original cwd)
|
|
784
|
-
this._initializeClusterAgents({
|
|
785
|
-
config,
|
|
786
|
-
cluster,
|
|
787
|
-
messageBus,
|
|
788
|
-
options,
|
|
789
|
-
isolationManager,
|
|
790
|
-
clusterId,
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
// ========================================================================
|
|
794
|
-
// CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
|
|
795
|
-
// ========================================================================
|
|
796
|
-
// MUST register ALL subscriptions BEFORE starting agents.
|
|
797
|
-
//
|
|
798
|
-
// WHY: EventEmitter is synchronous and doesn't replay past events to new
|
|
799
|
-
// listeners. If an agent completes and publishes CLUSTER_COMPLETE before
|
|
800
|
-
// we register the subscription, the orchestrator will never receive the
|
|
801
|
-
// message and the cluster will appear stuck in 'executing_task' forever.
|
|
802
|
-
//
|
|
803
|
-
// ORDER:
|
|
804
|
-
// 1. Register subscriptions (lines below)
|
|
805
|
-
// 2. Start agents
|
|
806
|
-
// 3. Publish ISSUE_OPENED
|
|
807
|
-
//
|
|
808
|
-
// DO NOT move subscriptions after agent.start() - this will reintroduce
|
|
809
|
-
// the race condition fixed in issue #31.
|
|
810
|
-
// ========================================================================
|
|
811
|
-
this._registerClusterSubscriptions({
|
|
812
|
-
messageBus,
|
|
813
|
-
clusterId,
|
|
814
|
-
isolationManager,
|
|
815
|
-
containerId,
|
|
816
|
-
});
|
|
817
|
-
|
|
818
|
-
// Start all agents
|
|
819
|
-
for (const agent of cluster.agents) {
|
|
820
|
-
await agent.start();
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
cluster.state = 'running';
|
|
824
|
-
|
|
825
|
-
// Publish ISSUE_OPENED message to bootstrap workflow
|
|
826
|
-
messageBus.publish({
|
|
827
|
-
cluster_id: clusterId,
|
|
828
|
-
topic: 'ISSUE_OPENED',
|
|
829
|
-
sender: 'system',
|
|
830
|
-
receiver: 'broadcast',
|
|
831
|
-
content: {
|
|
832
|
-
text: inputData.context,
|
|
833
|
-
data: {
|
|
834
|
-
issue_number: inputData.number,
|
|
835
|
-
title: inputData.title,
|
|
836
|
-
},
|
|
837
|
-
},
|
|
838
|
-
metadata: {
|
|
839
|
-
source: this._getInputSource(input),
|
|
840
|
-
},
|
|
841
|
-
});
|
|
842
|
-
|
|
843
|
-
// CRITICAL: Mark initialization complete AFTER ISSUE_OPENED is published
|
|
844
|
-
// This ensures stop() waits for at least 1 message before stopping
|
|
845
|
-
if (cluster._resolveInitComplete) {
|
|
846
|
-
cluster._resolveInitComplete();
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
this._log(`Cluster ${clusterId} started with ${cluster.agents.length} agents`);
|
|
850
|
-
|
|
851
|
-
// DISABLED: Idle timeout auto-stop mechanism
|
|
852
|
-
// WHY DISABLED: Clusters should only stop on explicit signals:
|
|
853
|
-
// - User `zeroshot kill` command
|
|
854
|
-
// - CLUSTER_COMPLETE message (successful completion)
|
|
855
|
-
// - CLUSTER_FAILED message (failure/abort)
|
|
856
|
-
// Being "idle" is NOT a reason to auto-stop - agents may be legitimately
|
|
857
|
-
// waiting for external events, user input (in interactive mode), or
|
|
858
|
-
// processing that doesn't show as "executing" (e.g., polling, monitoring).
|
|
859
|
-
//
|
|
860
|
-
// Previous behavior: Stopped cluster after 2 minutes of all agents idle
|
|
861
|
-
// Result: Clusters were killed while legitimately waiting, causing confusion
|
|
862
|
-
//
|
|
863
|
-
// cluster.idleCheckInterval = setInterval(() => { ... }, 30000);
|
|
864
|
-
// ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
|
|
865
|
-
|
|
866
|
-
// Save cluster to disk
|
|
867
|
-
await this._saveClusters();
|
|
868
|
-
|
|
869
|
-
return {
|
|
870
|
-
id: clusterId,
|
|
871
|
-
state: cluster.state,
|
|
872
|
-
agents: cluster.agents.map((a) => a.getState()),
|
|
873
|
-
ledger: cluster.ledger, // Expose ledger for testing
|
|
874
|
-
messageBus: cluster.messageBus, // Expose messageBus for testing
|
|
875
|
-
};
|
|
876
|
-
} catch (error) {
|
|
877
|
-
cluster.state = 'failed';
|
|
878
|
-
// CRITICAL: Resolve the promise on failure too, so stop() doesn't hang
|
|
879
|
-
if (cluster._resolveInitComplete) {
|
|
880
|
-
cluster._resolveInitComplete();
|
|
881
|
-
}
|
|
882
|
-
console.error(`Cluster ${clusterId} failed to start:`, error);
|
|
883
|
-
throw error;
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
_initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
|
|
888
|
-
const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
|
|
889
|
-
|
|
890
|
-
for (const agentConfig of config.agents) {
|
|
891
|
-
if (!agentConfig.cwd) {
|
|
892
|
-
agentConfig.cwd = agentCwd;
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
if (options.modelOverride) {
|
|
896
|
-
applyModelOverride(agentConfig, options.modelOverride);
|
|
897
|
-
this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
const agentOptions = {
|
|
901
|
-
testMode: options.testMode || !!this.taskRunner,
|
|
902
|
-
quiet: this.quiet,
|
|
903
|
-
modelOverride: options.modelOverride || null,
|
|
904
|
-
};
|
|
905
|
-
|
|
906
|
-
if (options.mockExecutor) {
|
|
907
|
-
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
if (this.taskRunner) {
|
|
911
|
-
agentOptions.taskRunner = this.taskRunner;
|
|
912
|
-
}
|
|
913
|
-
|
|
914
|
-
if (cluster.isolation) {
|
|
915
|
-
agentOptions.isolation = {
|
|
916
|
-
enabled: true,
|
|
917
|
-
manager: isolationManager,
|
|
918
|
-
clusterId,
|
|
919
|
-
};
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
if (cluster.worktree) {
|
|
923
|
-
agentOptions.worktree = {
|
|
924
|
-
enabled: true,
|
|
925
|
-
path: cluster.worktree.path,
|
|
926
|
-
branch: cluster.worktree.branch,
|
|
927
|
-
repoRoot: cluster.worktree.repoRoot,
|
|
928
|
-
};
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
const agent =
|
|
932
|
-
agentConfig.type === 'subcluster'
|
|
933
|
-
? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
|
|
934
|
-
: new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
935
|
-
|
|
936
|
-
cluster.agents.push(agent);
|
|
937
|
-
}
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
_subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
|
|
941
|
-
messageBus.subscribe((message) => {
|
|
942
|
-
if (message.topic === topic && message.cluster_id === clusterId) {
|
|
943
|
-
handler(message);
|
|
944
|
-
}
|
|
945
|
-
});
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
_registerClusterCompletionHandlers(messageBus, clusterId) {
|
|
949
|
-
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
|
|
950
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
951
|
-
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
952
|
-
this._log(`${'='.repeat(80)}`);
|
|
953
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
954
|
-
this._log(`Initiated by: ${message.sender}`);
|
|
955
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
956
|
-
|
|
957
|
-
this.stop(clusterId).catch((err) => {
|
|
958
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
959
|
-
});
|
|
960
|
-
});
|
|
961
|
-
|
|
962
|
-
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
|
|
963
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
964
|
-
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
965
|
-
this._log(`${'='.repeat(80)}`);
|
|
966
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
967
|
-
this._log(`Agent: ${message.sender}`);
|
|
968
|
-
if (message.content?.text) {
|
|
969
|
-
this._log(`Details: ${message.content.text}`);
|
|
970
|
-
}
|
|
971
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
972
|
-
|
|
973
|
-
this.stop(clusterId).catch((err) => {
|
|
974
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
975
|
-
});
|
|
976
|
-
});
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
_registerAgentErrorHandler(messageBus, clusterId) {
|
|
980
|
-
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
|
|
981
|
-
const agentRole = message.content?.data?.role;
|
|
982
|
-
const attempts = message.content?.data?.attempts || 1;
|
|
983
|
-
|
|
984
|
-
await this._saveClusters();
|
|
985
|
-
|
|
986
|
-
if (agentRole === 'implementation' && attempts >= 3) {
|
|
987
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
988
|
-
this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
|
|
989
|
-
this._log(`${'='.repeat(80)}`);
|
|
990
|
-
this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
|
|
991
|
-
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
992
|
-
this._log(`Stopping cluster - worker cannot continue`);
|
|
993
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
994
|
-
|
|
995
|
-
this.stop(clusterId).catch((err) => {
|
|
996
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
997
|
-
});
|
|
998
|
-
}
|
|
999
|
-
});
|
|
1000
|
-
}
|
|
1001
|
-
|
|
1002
|
-
_registerAgentLifecycleHandlers(messageBus, clusterId) {
|
|
1003
|
-
messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
|
|
1004
|
-
const event = message.content?.data?.event;
|
|
1005
|
-
if (
|
|
1006
|
-
[
|
|
1007
|
-
'TASK_STARTED',
|
|
1008
|
-
'TASK_COMPLETED',
|
|
1009
|
-
'PROCESS_SPAWNED',
|
|
1010
|
-
'TASK_ID_ASSIGNED',
|
|
1011
|
-
'STARTED',
|
|
1012
|
-
].includes(event)
|
|
1013
|
-
) {
|
|
1014
|
-
await this._saveClusters();
|
|
1015
|
-
}
|
|
1016
|
-
});
|
|
1017
|
-
|
|
1018
|
-
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
1019
|
-
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
1020
|
-
|
|
1021
|
-
const agentId = message.content?.data?.agent;
|
|
1022
|
-
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
1023
|
-
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
1024
|
-
|
|
1025
|
-
this._log(
|
|
1026
|
-
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
1027
|
-
);
|
|
1028
|
-
this._log(` Analysis: ${analysis}`);
|
|
1029
|
-
this._log(
|
|
1030
|
-
` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
|
|
1031
|
-
);
|
|
1032
|
-
});
|
|
1033
|
-
}
|
|
1034
|
-
|
|
1035
|
-
_registerConductorWatchdog(messageBus, clusterId) {
|
|
1036
|
-
const timeoutMs = 30000;
|
|
1037
|
-
let watchdogTimer = null;
|
|
1038
|
-
let completedAt = null;
|
|
1039
|
-
|
|
1040
|
-
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
|
|
1041
|
-
const event = message.content?.data?.event;
|
|
1042
|
-
const role = message.content?.data?.role;
|
|
1043
|
-
|
|
1044
|
-
if (event === 'TASK_COMPLETED' && role === 'conductor') {
|
|
1045
|
-
completedAt = Date.now();
|
|
1046
|
-
this._log(
|
|
1047
|
-
`⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
|
|
1048
|
-
);
|
|
1049
|
-
|
|
1050
|
-
watchdogTimer = setTimeout(() => {
|
|
1051
|
-
const clusterOps = messageBus.query({
|
|
1052
|
-
cluster_id: clusterId,
|
|
1053
|
-
topic: 'CLUSTER_OPERATIONS',
|
|
1054
|
-
limit: 1,
|
|
1055
|
-
});
|
|
1056
|
-
if (clusterOps.length === 0) {
|
|
1057
|
-
console.error(`\n${'='.repeat(80)}`);
|
|
1058
|
-
console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
|
|
1059
|
-
console.error(`${'='.repeat(80)}`);
|
|
1060
|
-
console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
|
|
1061
|
-
console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
|
|
1062
|
-
console.error(
|
|
1063
|
-
`Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
|
|
1064
|
-
);
|
|
1065
|
-
console.error(`${'='.repeat(80)}\n`);
|
|
1066
|
-
|
|
1067
|
-
messageBus.publish({
|
|
1068
|
-
cluster_id: clusterId,
|
|
1069
|
-
topic: 'CLUSTER_FAILED',
|
|
1070
|
-
sender: 'orchestrator',
|
|
1071
|
-
content: {
|
|
1072
|
-
text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
|
|
1073
|
-
data: {
|
|
1074
|
-
reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
|
|
1075
|
-
conductorCompletedAt: completedAt,
|
|
1076
|
-
timeoutMs: timeoutMs,
|
|
1077
|
-
},
|
|
1078
|
-
},
|
|
1079
|
-
});
|
|
1080
|
-
}
|
|
1081
|
-
}, timeoutMs);
|
|
1082
|
-
}
|
|
1083
|
-
});
|
|
1084
|
-
|
|
1085
|
-
return {
|
|
1086
|
-
clear: () => {
|
|
1087
|
-
if (!watchdogTimer) {
|
|
1088
|
-
return;
|
|
1089
|
-
}
|
|
1090
|
-
clearTimeout(watchdogTimer);
|
|
1091
|
-
watchdogTimer = null;
|
|
1092
|
-
const elapsed = completedAt ? Date.now() - completedAt : 0;
|
|
1093
|
-
this._log(
|
|
1094
|
-
`✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
|
|
1095
|
-
);
|
|
1096
|
-
},
|
|
1097
|
-
};
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
_registerClusterOperationsHandler(
|
|
1101
|
-
messageBus,
|
|
1102
|
-
clusterId,
|
|
1103
|
-
isolationManager,
|
|
1104
|
-
containerId,
|
|
1105
|
-
watchdog
|
|
1106
|
-
) {
|
|
1107
|
-
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
|
|
1108
|
-
watchdog?.clear?.();
|
|
1109
|
-
|
|
1110
|
-
let operations = message.content?.data?.operations;
|
|
1111
|
-
if (typeof operations === 'string') {
|
|
1112
|
-
try {
|
|
1113
|
-
operations = JSON.parse(operations);
|
|
1114
|
-
} catch (e) {
|
|
1115
|
-
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
1116
|
-
return;
|
|
1117
|
-
}
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
if (!operations || !Array.isArray(operations)) {
|
|
1121
|
-
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
1122
|
-
return;
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
1126
|
-
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
1127
|
-
this._log(`${'='.repeat(80)}`);
|
|
1128
|
-
if (message.content?.data?.reasoning) {
|
|
1129
|
-
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
1130
|
-
}
|
|
1131
|
-
this._log(`Operations: ${operations.length}`);
|
|
1132
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
1133
|
-
|
|
1134
|
-
this._handleOperations(clusterId, operations, message.sender, {
|
|
1135
|
-
isolationManager,
|
|
1136
|
-
containerId,
|
|
1137
|
-
}).catch((err) => {
|
|
1138
|
-
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
1139
|
-
messageBus.publish({
|
|
1140
|
-
cluster_id: clusterId,
|
|
1141
|
-
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
1142
|
-
sender: 'orchestrator',
|
|
1143
|
-
content: {
|
|
1144
|
-
text: `Operation chain failed: ${err.message}`,
|
|
1145
|
-
data: {
|
|
1146
|
-
error: err.message,
|
|
1147
|
-
operations: operations,
|
|
1148
|
-
},
|
|
1149
|
-
},
|
|
1150
|
-
});
|
|
1151
|
-
|
|
1152
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
1153
|
-
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
1154
|
-
this._log(`${'='.repeat(80)}`);
|
|
1155
|
-
this._log(`Error: ${err.message}`);
|
|
1156
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
1157
|
-
|
|
1158
|
-
this.stop(clusterId).catch((stopErr) => {
|
|
1159
|
-
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
1160
|
-
});
|
|
1161
|
-
});
|
|
1162
|
-
});
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
_registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
|
|
1166
|
-
this._registerClusterCompletionHandlers(messageBus, clusterId);
|
|
1167
|
-
this._registerAgentErrorHandler(messageBus, clusterId);
|
|
1168
|
-
this._registerAgentLifecycleHandlers(messageBus, clusterId);
|
|
1169
|
-
|
|
1170
|
-
const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
|
|
1171
|
-
this._registerClusterOperationsHandler(
|
|
1172
|
-
messageBus,
|
|
1173
|
-
clusterId,
|
|
1174
|
-
isolationManager,
|
|
1175
|
-
containerId,
|
|
1176
|
-
watchdog
|
|
1177
|
-
);
|
|
1178
|
-
}
|
|
1179
|
-
|
|
1180
|
-
async _initializeIsolation(options, config, clusterId) {
|
|
1181
|
-
let isolationManager = null;
|
|
1182
|
-
let containerId = null;
|
|
1183
|
-
let worktreeInfo = null;
|
|
1184
|
-
|
|
1185
|
-
if (options.isolation) {
|
|
1186
|
-
if (!IsolationManager.isDockerAvailable()) {
|
|
1187
|
-
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
1188
|
-
}
|
|
1189
|
-
|
|
1190
|
-
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
1191
|
-
await IsolationManager.ensureImage(image);
|
|
1192
|
-
|
|
1193
|
-
isolationManager = new IsolationManager({ image });
|
|
1194
|
-
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
1195
|
-
|
|
1196
|
-
const workDir = options.cwd || process.cwd();
|
|
1197
|
-
const providerName = normalizeProviderName(
|
|
1198
|
-
config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
|
|
1199
|
-
);
|
|
1200
|
-
containerId = await isolationManager.createContainer(clusterId, {
|
|
1201
|
-
workDir,
|
|
1202
|
-
image,
|
|
1203
|
-
noMounts: options.noMounts,
|
|
1204
|
-
mounts: options.mounts,
|
|
1205
|
-
containerHome: options.containerHome,
|
|
1206
|
-
provider: providerName,
|
|
1207
|
-
});
|
|
1208
|
-
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
1209
|
-
} else if (options.worktree) {
|
|
1210
|
-
const workDir = options.cwd || process.cwd();
|
|
1211
|
-
|
|
1212
|
-
isolationManager = new IsolationManager({});
|
|
1213
|
-
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
|
|
1214
|
-
|
|
1215
|
-
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
1216
|
-
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
1217
|
-
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
1218
|
-
}
|
|
1219
|
-
|
|
1220
|
-
return { isolationManager, containerId, worktreeInfo };
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
_applyAutoPrConfig(config, inputData, options) {
|
|
1224
|
-
if (!options.autoPr) {
|
|
1225
|
-
return;
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
|
|
1229
|
-
|
|
1230
|
-
// Detect git platform (independent of issue provider)
|
|
1231
|
-
const { getPlatformForPR } = require('./issue-providers');
|
|
1232
|
-
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
1233
|
-
|
|
1234
|
-
let platform;
|
|
1235
|
-
try {
|
|
1236
|
-
// ALWAYS use git context, regardless of issue provider
|
|
1237
|
-
// This allows Jira issues in GitHub repos, etc.
|
|
1238
|
-
platform = getPlatformForPR(options.cwd);
|
|
1239
|
-
} catch (error) {
|
|
1240
|
-
throw new Error(`--pr mode failed: ${error.message}`);
|
|
1241
|
-
}
|
|
1242
|
-
|
|
1243
|
-
// Check for repo mismatch (issue URL repo vs current git remote)
|
|
1244
|
-
// This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
|
|
1245
|
-
// from a different repo directory
|
|
1246
|
-
let skipCloseIssue = false;
|
|
1247
|
-
const gitContext = detectGitContext(options.cwd);
|
|
1248
|
-
if (inputData.url && gitContext) {
|
|
1249
|
-
const issueHost = this._extractHostFromUrl(inputData.url);
|
|
1250
|
-
const gitHost = gitContext.host;
|
|
1251
|
-
|
|
1252
|
-
// Compare hosts - if different platforms, warn and don't close issue
|
|
1253
|
-
if (issueHost && gitHost && issueHost !== gitHost) {
|
|
1254
|
-
this._log(
|
|
1255
|
-
`[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
|
|
1256
|
-
);
|
|
1257
|
-
this._log(
|
|
1258
|
-
`[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
|
|
1259
|
-
);
|
|
1260
|
-
this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
|
|
1261
|
-
skipCloseIssue = true;
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
// Generate platform-specific git-pusher agent from template
|
|
1266
|
-
const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
|
|
1267
|
-
|
|
1268
|
-
if (!isPlatformSupported(platform)) {
|
|
1269
|
-
throw new Error(
|
|
1270
|
-
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
1271
|
-
);
|
|
1272
|
-
}
|
|
1273
|
-
|
|
1274
|
-
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
1275
|
-
|
|
1276
|
-
// Template replacement for issue context
|
|
1277
|
-
const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
|
|
1278
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
1279
|
-
.replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
|
|
1280
|
-
.replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
|
|
1281
|
-
.replace(/Closes #\{\{issue_number\}\}/g, issueRef);
|
|
1282
|
-
|
|
1283
|
-
config.agents.push(gitPusherConfig);
|
|
1284
|
-
this._log(
|
|
1285
|
-
`[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
|
|
1286
|
-
);
|
|
1287
|
-
}
|
|
1288
|
-
|
|
1289
|
-
/**
|
|
1290
|
-
* Extract hostname from a URL
|
|
1291
|
-
* @private
|
|
1292
|
-
*/
|
|
1293
|
-
_extractHostFromUrl(url) {
|
|
1294
|
-
try {
|
|
1295
|
-
const match = url.match(/https?:\/\/([^/]+)/);
|
|
1296
|
-
return match ? match[1] : null;
|
|
1297
|
-
} catch {
|
|
1298
|
-
return null;
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
|
|
1302
|
-
_applyWorkerInstruction(config) {
|
|
1303
|
-
const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
|
|
1304
|
-
if (workersCount <= 1) {
|
|
1305
|
-
return;
|
|
1306
|
-
}
|
|
1307
|
-
|
|
1308
|
-
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
1309
|
-
if (!workerAgent) {
|
|
1310
|
-
return;
|
|
1311
|
-
}
|
|
1312
|
-
|
|
1313
|
-
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
1314
|
-
|
|
1315
|
-
if (!workerAgent.prompt) {
|
|
1316
|
-
workerAgent.prompt = instruction;
|
|
1317
|
-
} else if (typeof workerAgent.prompt === 'string') {
|
|
1318
|
-
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
1319
|
-
} else if (workerAgent.prompt.system) {
|
|
1320
|
-
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
1321
|
-
}
|
|
1322
|
-
|
|
1323
|
-
this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
|
|
1324
|
-
}
|
|
1325
|
-
|
|
1326
|
-
/**
|
|
1327
|
-
* Generate a unique cluster ID, safe for concurrent starts in-process.
|
|
1328
|
-
* If an explicit ID is provided, uses it as a base and suffixes on collision.
|
|
1329
|
-
* @private
|
|
1330
|
-
*/
|
|
1331
|
-
_generateUniqueClusterId(explicitId, explicitDbPath) {
|
|
1332
|
-
const baseId = explicitId || generateName('cluster');
|
|
1333
|
-
const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
|
|
1334
|
-
|
|
1335
|
-
// Fast path: base is unused.
|
|
1336
|
-
if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
|
|
1337
|
-
return baseId;
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
// Collision: suffix with random bytes to avoid race conditions under concurrency.
|
|
1341
|
-
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1342
|
-
const suffix = crypto.randomBytes(3).toString('hex');
|
|
1343
|
-
const candidateId = `${baseId}-${suffix}`;
|
|
1344
|
-
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1345
|
-
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1346
|
-
return candidateId;
|
|
1347
|
-
}
|
|
1348
|
-
}
|
|
1349
|
-
|
|
1350
|
-
// Last resort: new generated name (should never happen).
|
|
1351
|
-
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1352
|
-
const candidateId = generateName('cluster');
|
|
1353
|
-
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1354
|
-
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1355
|
-
return candidateId;
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
throw new Error('Failed to generate unique cluster ID after many attempts');
|
|
1360
|
-
}
|
|
1361
|
-
|
|
1362
|
-
/**
|
|
1363
|
-
* Stop a cluster
|
|
1364
|
-
* @param {String} clusterId - Cluster ID
|
|
1365
|
-
*/
|
|
1366
|
-
async stop(clusterId) {
|
|
1367
|
-
const cluster = this.clusters.get(clusterId);
|
|
1368
|
-
if (!cluster) {
|
|
1369
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
// CRITICAL: Wait for initialization to complete before stopping
|
|
1373
|
-
// This ensures ISSUE_OPENED is published, preventing 0-message clusters
|
|
1374
|
-
// Timeout after 30s to prevent infinite hang if init truly fails
|
|
1375
|
-
if (cluster.initCompletePromise && cluster.state === 'initializing') {
|
|
1376
|
-
this._log(`[Orchestrator] Waiting for initialization to complete before stopping...`);
|
|
1377
|
-
await Promise.race([
|
|
1378
|
-
cluster.initCompletePromise,
|
|
1379
|
-
new Promise((resolve) => setTimeout(resolve, 30000)),
|
|
1380
|
-
]);
|
|
1381
|
-
}
|
|
1382
|
-
|
|
1383
|
-
cluster.state = 'stopping';
|
|
1384
|
-
|
|
1385
|
-
// Stop all agents (including subclusters which handle their own children)
|
|
1386
|
-
for (const agent of cluster.agents) {
|
|
1387
|
-
await agent.stop();
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
// Clean up isolation container if enabled
|
|
1391
|
-
// CRITICAL: Preserve workspace for resume capability - only delete on kill()
|
|
1392
|
-
if (cluster.isolation?.manager) {
|
|
1393
|
-
this._log(
|
|
1394
|
-
`[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
|
|
1395
|
-
);
|
|
1396
|
-
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
|
|
1397
|
-
this._log(`[Orchestrator] Container stopped, workspace preserved`);
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
// Worktree cleanup on stop: preserve for resume capability
|
|
1401
|
-
// Branch stays, worktree stays - can resume work later
|
|
1402
|
-
if (cluster.worktree?.manager) {
|
|
1403
|
-
this._log(`[Orchestrator] Worktree preserved at ${cluster.worktree.path} for resume`);
|
|
1404
|
-
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
1405
|
-
// Don't cleanup worktree - it will be reused on resume
|
|
1406
|
-
}
|
|
1407
|
-
|
|
1408
|
-
cluster.state = 'stopped';
|
|
1409
|
-
cluster.pid = null; // Clear PID - cluster is no longer running
|
|
1410
|
-
this._log(`Cluster ${clusterId} stopped`);
|
|
1411
|
-
|
|
1412
|
-
// Save updated state
|
|
1413
|
-
await this._saveClusters();
|
|
1414
|
-
}
|
|
1415
|
-
|
|
1416
|
-
/**
|
|
1417
|
-
* Kill a cluster (force stop)
|
|
1418
|
-
* @param {String} clusterId - Cluster ID
|
|
1419
|
-
*/
|
|
1420
|
-
async kill(clusterId) {
|
|
1421
|
-
const cluster = this.clusters.get(clusterId);
|
|
1422
|
-
if (!cluster) {
|
|
1423
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
1424
|
-
}
|
|
1425
|
-
|
|
1426
|
-
cluster.state = 'stopping';
|
|
1427
|
-
|
|
1428
|
-
// Force stop all agents
|
|
1429
|
-
for (const agent of cluster.agents) {
|
|
1430
|
-
await agent.stop();
|
|
1431
|
-
}
|
|
1432
|
-
|
|
1433
|
-
// Force remove isolation container AND workspace (full cleanup, no resume)
|
|
1434
|
-
if (cluster.isolation?.manager) {
|
|
1435
|
-
this._log(
|
|
1436
|
-
`[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
|
|
1437
|
-
);
|
|
1438
|
-
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
|
|
1439
|
-
this._log(`[Orchestrator] Container and workspace removed`);
|
|
1440
|
-
}
|
|
1441
|
-
|
|
1442
|
-
// Force remove worktree (full cleanup, no resume)
|
|
1443
|
-
// Note: Branch is preserved for potential PR creation / inspection
|
|
1444
|
-
if (cluster.worktree?.manager) {
|
|
1445
|
-
this._log(`[Orchestrator] Force removing worktree for ${clusterId}...`);
|
|
1446
|
-
cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
|
|
1447
|
-
this._log(`[Orchestrator] Worktree removed, branch ${cluster.worktree.branch} preserved`);
|
|
1448
|
-
}
|
|
1449
|
-
|
|
1450
|
-
// Close message bus and ledger
|
|
1451
|
-
cluster.messageBus.close();
|
|
1452
|
-
|
|
1453
|
-
cluster.state = 'killed';
|
|
1454
|
-
cluster.pid = null; // Clear PID - cluster is no longer running
|
|
1455
|
-
// DON'T delete from memory - keep it so it gets saved with 'killed' state
|
|
1456
|
-
// this.clusters.delete(clusterId);
|
|
1457
|
-
|
|
1458
|
-
this._log(`Cluster ${clusterId} killed`);
|
|
1459
|
-
|
|
1460
|
-
// Save updated state (will be marked as 'killed' in file)
|
|
1461
|
-
await this._saveClusters();
|
|
1462
|
-
|
|
1463
|
-
// Now remove from memory after persisting
|
|
1464
|
-
this.clusters.delete(clusterId);
|
|
1465
|
-
}
|
|
1466
|
-
|
|
1467
|
-
/**
|
|
1468
|
-
* Kill all running clusters
|
|
1469
|
-
* @returns {Object} { killed: Array<string>, errors: Array<{id, error}> }
|
|
1470
|
-
*/
|
|
1471
|
-
async killAll() {
|
|
1472
|
-
const results = { killed: [], errors: [] };
|
|
1473
|
-
const runningClusters = Array.from(this.clusters.values()).filter(
|
|
1474
|
-
(c) => c.state === 'running' || c.state === 'initializing'
|
|
1475
|
-
);
|
|
1476
|
-
|
|
1477
|
-
for (const cluster of runningClusters) {
|
|
1478
|
-
try {
|
|
1479
|
-
await this.kill(cluster.id);
|
|
1480
|
-
results.killed.push(cluster.id);
|
|
1481
|
-
} catch (error) {
|
|
1482
|
-
results.errors.push({ id: cluster.id, error: error.message });
|
|
1483
|
-
}
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
return results;
|
|
1487
|
-
}
|
|
1488
|
-
|
|
1489
|
-
/**
|
|
1490
|
-
* Close the orchestrator (prevents further _saveClusters operations)
|
|
1491
|
-
* Call before deleting storageDir to prevent ENOENT race conditions during cleanup
|
|
1492
|
-
*/
|
|
1493
|
-
close() {
|
|
1494
|
-
this.closed = true;
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
|
-
/**
|
|
1498
|
-
* Find the last workflow-triggering message in the ledger
|
|
1499
|
-
* Workflow triggers indicate cluster state progression (not AGENT_OUTPUT noise)
|
|
1500
|
-
* @param {Array} messages - All messages from ledger
|
|
1501
|
-
* @returns {Object|null} - Last workflow trigger message or null
|
|
1502
|
-
* @private
|
|
1503
|
-
*/
|
|
1504
|
-
_findLastWorkflowTrigger(messages) {
|
|
1505
|
-
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1506
|
-
if (WORKFLOW_TRIGGERS.includes(messages[i].topic)) {
|
|
1507
|
-
return messages[i];
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
return null;
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
/**
|
|
1514
|
-
* Resume a stopped cluster from where it left off
|
|
1515
|
-
* Handles both failed clusters (with error context) and cleanly stopped clusters
|
|
1516
|
-
* @param {String} clusterId - Cluster ID
|
|
1517
|
-
* @param {String} prompt - Optional custom resume prompt
|
|
1518
|
-
* @returns {Object} Resumed cluster info
|
|
1519
|
-
*/
|
|
1520
|
-
async resume(clusterId, prompt) {
|
|
1521
|
-
const cluster = this.clusters.get(clusterId);
|
|
1522
|
-
if (!cluster) {
|
|
1523
|
-
throw new Error(`Cluster not found: ${clusterId}`);
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
if (cluster.state === 'running') {
|
|
1527
|
-
throw new Error(
|
|
1528
|
-
`Cluster ${clusterId} is still running. Use 'zeroshot stop' first if you want to restart it.`
|
|
1529
|
-
);
|
|
1530
|
-
}
|
|
1531
|
-
|
|
1532
|
-
const failureInfo = this._resolveFailureInfo(cluster, clusterId);
|
|
1533
|
-
|
|
1534
|
-
await this._ensureIsolationForResume(clusterId, cluster);
|
|
1535
|
-
this._ensureWorktreeForResume(clusterId, cluster);
|
|
1536
|
-
await this._restartClusterAgents(cluster);
|
|
1537
|
-
|
|
1538
|
-
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
1539
|
-
|
|
1540
|
-
if (failureInfo) {
|
|
1541
|
-
return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
|
|
1542
|
-
}
|
|
1543
|
-
|
|
1544
|
-
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
|
|
1545
|
-
}
|
|
1546
|
-
|
|
1547
|
-
_resolveFailureInfo(cluster, clusterId) {
|
|
1548
|
-
if (cluster.failureInfo) {
|
|
1549
|
-
return cluster.failureInfo;
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
const errors = cluster.messageBus.query({
|
|
1553
|
-
cluster_id: clusterId,
|
|
1554
|
-
topic: 'AGENT_ERROR',
|
|
1555
|
-
limit: 10,
|
|
1556
|
-
order: 'desc',
|
|
1557
|
-
});
|
|
1558
|
-
|
|
1559
|
-
if (errors.length === 0) {
|
|
1560
|
-
return null;
|
|
1561
|
-
}
|
|
1562
|
-
|
|
1563
|
-
const firstError = errors[0];
|
|
1564
|
-
const failureInfo = {
|
|
1565
|
-
agentId: firstError.sender,
|
|
1566
|
-
taskId: firstError.content?.data?.taskId || null,
|
|
1567
|
-
iteration: firstError.content?.data?.iteration || 0,
|
|
1568
|
-
error: firstError.content?.data?.error || firstError.content?.text,
|
|
1569
|
-
timestamp: firstError.timestamp,
|
|
1570
|
-
};
|
|
1571
|
-
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
1572
|
-
return failureInfo;
|
|
1573
|
-
}
|
|
1574
|
-
|
|
1575
|
-
_checkContainerExists(containerId) {
|
|
1576
|
-
const { spawn } = require('child_process');
|
|
1577
|
-
const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
|
|
1578
|
-
return new Promise((resolve) => {
|
|
1579
|
-
checkContainer.on('close', (code) => resolve(code === 0));
|
|
1580
|
-
});
|
|
1581
|
-
}
|
|
1582
|
-
|
|
1583
|
-
async _ensureIsolationForResume(clusterId, cluster) {
|
|
1584
|
-
if (!cluster.isolation?.enabled) {
|
|
1585
|
-
return;
|
|
1586
|
-
}
|
|
1587
|
-
|
|
1588
|
-
const oldContainerId = cluster.isolation.containerId;
|
|
1589
|
-
const containerExists = await this._checkContainerExists(oldContainerId);
|
|
1590
|
-
if (containerExists) {
|
|
1591
|
-
this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
|
|
1592
|
-
return;
|
|
1593
|
-
}
|
|
1594
|
-
|
|
1595
|
-
this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
|
|
1596
|
-
|
|
1597
|
-
const workDir = cluster.isolation.workDir;
|
|
1598
|
-
if (!workDir) {
|
|
1599
|
-
throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
|
-
const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
|
|
1603
|
-
if (!fs.existsSync(isolatedPath)) {
|
|
1604
|
-
throw new Error(
|
|
1605
|
-
`Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
|
|
1606
|
-
`Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
|
|
1607
|
-
);
|
|
1608
|
-
}
|
|
1609
|
-
|
|
1610
|
-
const providerName = normalizeProviderName(
|
|
1611
|
-
cluster.config?.forceProvider ||
|
|
1612
|
-
cluster.config?.defaultProvider ||
|
|
1613
|
-
loadSettings().defaultProvider ||
|
|
1614
|
-
'claude'
|
|
1615
|
-
);
|
|
1616
|
-
const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
|
|
1617
|
-
workDir,
|
|
1618
|
-
image: cluster.isolation.image,
|
|
1619
|
-
reuseExistingWorkspace: true,
|
|
1620
|
-
provider: providerName,
|
|
1621
|
-
});
|
|
1622
|
-
|
|
1623
|
-
this._log(`[Orchestrator] New container created: ${newContainerId}`);
|
|
1624
|
-
cluster.isolation.containerId = newContainerId;
|
|
1625
|
-
|
|
1626
|
-
for (const agent of cluster.agents) {
|
|
1627
|
-
if (agent.isolation?.enabled) {
|
|
1628
|
-
agent.isolation.containerId = newContainerId;
|
|
1629
|
-
agent.isolation.manager = cluster.isolation.manager;
|
|
1630
|
-
}
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
this._log(`[Orchestrator] All agents updated with new container ID`);
|
|
1634
|
-
}
|
|
1635
|
-
|
|
1636
|
-
_ensureWorktreeForResume(clusterId, cluster) {
|
|
1637
|
-
if (!cluster.worktree?.enabled) {
|
|
1638
|
-
return;
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
const worktreePath = cluster.worktree.path;
|
|
1642
|
-
if (!fs.existsSync(worktreePath)) {
|
|
1643
|
-
throw new Error(
|
|
1644
|
-
`Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
|
|
1645
|
-
`Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
|
|
1646
|
-
);
|
|
1647
|
-
}
|
|
1648
|
-
|
|
1649
|
-
this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
|
|
1650
|
-
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
1651
|
-
}
|
|
1652
|
-
|
|
1653
|
-
async _restartClusterAgents(cluster) {
|
|
1654
|
-
cluster.state = 'running';
|
|
1655
|
-
for (const agent of cluster.agents) {
|
|
1656
|
-
if (!agent.running) {
|
|
1657
|
-
await agent.start();
|
|
1658
|
-
}
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
|
|
1662
|
-
_loadRecentMessages(cluster, clusterId, limit) {
|
|
1663
|
-
return cluster.messageBus
|
|
1664
|
-
.query({
|
|
1665
|
-
cluster_id: clusterId,
|
|
1666
|
-
limit,
|
|
1667
|
-
order: 'desc',
|
|
1668
|
-
})
|
|
1669
|
-
.reverse();
|
|
1670
|
-
}
|
|
1671
|
-
|
|
1672
|
-
_buildResumeContext(recentMessages, prompt, options) {
|
|
1673
|
-
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
1674
|
-
let context = `${options.header}\n\n`;
|
|
1675
|
-
|
|
1676
|
-
if (options.error) {
|
|
1677
|
-
context += `Previous error: ${options.error}\n\n`;
|
|
1678
|
-
}
|
|
1679
|
-
|
|
1680
|
-
context += `## Recent Context\n\n`;
|
|
1681
|
-
for (const msg of recentMessages.slice(-10)) {
|
|
1682
|
-
if (options.topics.includes(msg.topic)) {
|
|
1683
|
-
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1684
|
-
}
|
|
1685
|
-
}
|
|
1686
|
-
|
|
1687
|
-
context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
|
|
1688
|
-
return context;
|
|
1689
|
-
}
|
|
1690
|
-
|
|
1691
|
-
_triggerMatchesTopic(triggerTopic, topic) {
|
|
1692
|
-
if (triggerTopic === topic) return true;
|
|
1693
|
-
if (triggerTopic === '*') return true;
|
|
1694
|
-
if (triggerTopic.endsWith('*')) {
|
|
1695
|
-
const prefix = triggerTopic.slice(0, -1);
|
|
1696
|
-
return topic.startsWith(prefix);
|
|
1697
|
-
}
|
|
1698
|
-
return false;
|
|
1699
|
-
}
|
|
1700
|
-
|
|
1701
|
-
_selectAgentsToResume(cluster, lastTrigger) {
|
|
1702
|
-
const agentsToResume = [];
|
|
1703
|
-
|
|
1704
|
-
for (const agent of cluster.agents) {
|
|
1705
|
-
if (!agent.config.triggers) continue;
|
|
1706
|
-
|
|
1707
|
-
const matchingTrigger = agent.config.triggers.find((trigger) =>
|
|
1708
|
-
this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
|
|
1709
|
-
);
|
|
1710
|
-
if (!matchingTrigger) continue;
|
|
1711
|
-
|
|
1712
|
-
if (matchingTrigger.logic?.script) {
|
|
1713
|
-
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
1714
|
-
if (!shouldTrigger) continue;
|
|
1715
|
-
}
|
|
1716
|
-
|
|
1717
|
-
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
1718
|
-
}
|
|
1719
|
-
|
|
1720
|
-
return agentsToResume;
|
|
1721
|
-
}
|
|
1722
|
-
|
|
1723
|
-
_resumeAgents(agentsToResume, context) {
|
|
1724
|
-
for (const { agent, message } of agentsToResume) {
|
|
1725
|
-
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
1726
|
-
agent.resume(context).catch((err) => {
|
|
1727
|
-
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
1728
|
-
});
|
|
1729
|
-
}
|
|
1730
|
-
}
|
|
1731
|
-
|
|
1732
|
-
_republishIssue(cluster, clusterId, recentMessages) {
|
|
1733
|
-
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
1734
|
-
if (!issueMessage) {
|
|
1735
|
-
return false;
|
|
1736
|
-
}
|
|
1737
|
-
|
|
1738
|
-
cluster.messageBus.publish({
|
|
1739
|
-
cluster_id: clusterId,
|
|
1740
|
-
topic: 'ISSUE_OPENED',
|
|
1741
|
-
sender: 'system',
|
|
1742
|
-
receiver: 'broadcast',
|
|
1743
|
-
content: issueMessage.content,
|
|
1744
|
-
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
1745
|
-
});
|
|
1746
|
-
|
|
1747
|
-
return true;
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
|
|
1751
|
-
const { agentId, iteration, error } = failureInfo;
|
|
1752
|
-
this._log(
|
|
1753
|
-
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
1754
|
-
);
|
|
1755
|
-
this._log(`[Orchestrator] Previous error: ${error}`);
|
|
1756
|
-
|
|
1757
|
-
const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
|
|
1758
|
-
if (!failedAgent) {
|
|
1759
|
-
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
1760
|
-
}
|
|
1761
|
-
|
|
1762
|
-
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1763
|
-
header: 'You are resuming from a previous failed attempt.',
|
|
1764
|
-
error,
|
|
1765
|
-
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
|
|
1766
|
-
});
|
|
1767
|
-
|
|
1768
|
-
cluster.failureInfo = null;
|
|
1769
|
-
await this._saveClusters();
|
|
1770
|
-
|
|
1771
|
-
failedAgent.resume(context).catch((err) => {
|
|
1772
|
-
console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
|
|
1773
|
-
});
|
|
1774
|
-
|
|
1775
|
-
this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
|
|
1776
|
-
|
|
1777
|
-
return {
|
|
1778
|
-
id: clusterId,
|
|
1779
|
-
state: cluster.state,
|
|
1780
|
-
resumeType: 'failure',
|
|
1781
|
-
resumedAgent: agentId,
|
|
1782
|
-
previousError: error,
|
|
1783
|
-
};
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
|
|
1787
|
-
this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
|
|
1788
|
-
|
|
1789
|
-
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1790
|
-
header: 'Resuming cluster from previous session.',
|
|
1791
|
-
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
|
|
1792
|
-
});
|
|
1793
|
-
|
|
1794
|
-
const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
|
|
1795
|
-
if (lastTrigger) {
|
|
1796
|
-
this._log(
|
|
1797
|
-
`[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
|
|
1798
|
-
);
|
|
1799
|
-
} else {
|
|
1800
|
-
this._log(`[Orchestrator] No workflow triggers found in ledger`);
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
|
-
const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
|
|
1804
|
-
if (agentsToResume.length === 0) {
|
|
1805
|
-
if (!lastTrigger) {
|
|
1806
|
-
this._log(
|
|
1807
|
-
`[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
|
|
1808
|
-
);
|
|
1809
|
-
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
1810
|
-
|
|
1811
|
-
if (!this._republishIssue(cluster, clusterId, recentMessages)) {
|
|
1812
|
-
throw new Error(
|
|
1813
|
-
`Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
|
|
1814
|
-
`The cluster may not have started properly. Try: zeroshot run <issue> instead.`
|
|
1815
|
-
);
|
|
1816
|
-
}
|
|
1817
|
-
} else {
|
|
1818
|
-
throw new Error(
|
|
1819
|
-
`Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
|
|
1820
|
-
`Check agent trigger configurations.`
|
|
1821
|
-
);
|
|
1822
|
-
}
|
|
1823
|
-
} else {
|
|
1824
|
-
this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
|
|
1825
|
-
this._resumeAgents(agentsToResume, context);
|
|
1826
|
-
}
|
|
1827
|
-
|
|
1828
|
-
await this._saveClusters();
|
|
1829
|
-
this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
|
|
1830
|
-
|
|
1831
|
-
return {
|
|
1832
|
-
id: clusterId,
|
|
1833
|
-
state: cluster.state,
|
|
1834
|
-
resumeType: 'clean',
|
|
1835
|
-
resumedAgents: agentsToResume.map((a) => a.agent.id),
|
|
1836
|
-
};
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
/**
|
|
1840
|
-
* Force restart a stale agent with imperative prompt injection
|
|
1841
|
-
* @param {string} clusterId - Cluster ID
|
|
1842
|
-
* @param {string} agentId - Agent to restart
|
|
1843
|
-
* @param {number} staleDuration - How long agent was stale (ms)
|
|
1844
|
-
* @private
|
|
1845
|
-
*/
|
|
1846
|
-
async _forceRestartAgent(clusterId, agentId, staleDuration) {
|
|
1847
|
-
const cluster = this.clusters.get(clusterId);
|
|
1848
|
-
if (!cluster) {
|
|
1849
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
1850
|
-
}
|
|
1851
|
-
|
|
1852
|
-
const agent = cluster.agents.find((a) => a.id === agentId);
|
|
1853
|
-
if (!agent) {
|
|
1854
|
-
throw new Error(`Agent ${agentId} not found in cluster ${clusterId}`);
|
|
1855
|
-
}
|
|
1856
|
-
|
|
1857
|
-
// Kill current task
|
|
1858
|
-
try {
|
|
1859
|
-
agent._killTask();
|
|
1860
|
-
} catch (err) {
|
|
1861
|
-
this._log(`⚠️ Failed to kill agent ${agentId} task:`, err.message);
|
|
1862
|
-
}
|
|
1863
|
-
|
|
1864
|
-
// Build imperative restart context
|
|
1865
|
-
const staleMinutes = Math.round(staleDuration / 60000);
|
|
1866
|
-
const imperativePrompt = `
|
|
1867
|
-
🔴 CRITICAL: Your previous session STOPPED PRODUCING OUTPUT for ${staleMinutes} minutes and was detected as STUCK.
|
|
1868
|
-
|
|
1869
|
-
## What Happened
|
|
1870
|
-
- Last output timestamp: ${new Date(Date.now() - staleDuration).toISOString()}
|
|
1871
|
-
- Detected as stale after ${staleMinutes} minutes of silence
|
|
1872
|
-
- Process was forcefully restarted
|
|
1873
|
-
|
|
1874
|
-
## Your Instructions
|
|
1875
|
-
You MUST complete your current task. DO NOT STOP until you either:
|
|
1876
|
-
1. Successfully complete the task and publish your completion message, OR
|
|
1877
|
-
2. Explicitly state WHY you cannot complete the task (missing files, impossible requirements, etc.)
|
|
1878
|
-
|
|
1879
|
-
If you discovered that files you need to modify don't exist:
|
|
1880
|
-
- CREATE them from scratch with the expected implementation
|
|
1881
|
-
- DO NOT silently give up
|
|
1882
|
-
- DO NOT stop working without explicit explanation
|
|
1883
|
-
|
|
1884
|
-
If you are stuck in an impossible situation:
|
|
1885
|
-
- EXPLAIN the problem clearly
|
|
1886
|
-
- PROPOSE alternative solutions
|
|
1887
|
-
- WAIT for guidance - do not exit
|
|
1888
|
-
|
|
1889
|
-
## Resume Your Work
|
|
1890
|
-
Continue from where you left off. Review your previous output to understand what you were working on.
|
|
1891
|
-
`.trim();
|
|
1892
|
-
|
|
1893
|
-
// Get recent context from ledger
|
|
1894
|
-
const recentMessages = cluster.messageBus
|
|
1895
|
-
.query({
|
|
1896
|
-
cluster_id: cluster.id,
|
|
1897
|
-
limit: 10,
|
|
1898
|
-
order: 'desc',
|
|
1899
|
-
})
|
|
1900
|
-
.reverse();
|
|
1901
|
-
|
|
1902
|
-
const contextText = recentMessages
|
|
1903
|
-
.map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
|
|
1904
|
-
.join('\n\n');
|
|
1905
|
-
|
|
1906
|
-
const fullContext = `${imperativePrompt}\n\n## Recent Context\n${contextText}`;
|
|
1907
|
-
|
|
1908
|
-
// Resume agent with imperative prompt
|
|
1909
|
-
this._log(
|
|
1910
|
-
`🔄 Restarting agent ${agentId} with imperative prompt (${imperativePrompt.length} chars)`
|
|
1911
|
-
);
|
|
1912
|
-
|
|
1913
|
-
try {
|
|
1914
|
-
await agent.resume(fullContext);
|
|
1915
|
-
this._log(`✅ Agent ${agentId} successfully restarted`);
|
|
1916
|
-
} catch (err) {
|
|
1917
|
-
this._log(`❌ Failed to resume agent ${agentId}:`, err.message);
|
|
1918
|
-
throw err;
|
|
1919
|
-
}
|
|
1920
|
-
}
|
|
1921
|
-
|
|
1922
|
-
/**
|
|
1923
|
-
* Handle operation chain from CLUSTER_OPERATIONS message
|
|
1924
|
-
* Executes operations sequentially: add_agents, remove_agents, update_agent, publish
|
|
1925
|
-
*
|
|
1926
|
-
* Validation strategy:
|
|
1927
|
-
* 1. Pre-validate all agent configs before executing any operations
|
|
1928
|
-
* 2. Build a mock cluster config with proposed changes
|
|
1929
|
-
* 3. Run config-validator on the mock to catch structural issues
|
|
1930
|
-
* 4. Only execute operations if validation passes
|
|
1931
|
-
*
|
|
1932
|
-
* @param {string} clusterId - Cluster ID
|
|
1933
|
-
* @param {Array} operations - Array of operation objects
|
|
1934
|
-
* @param {string} sender - Who sent the operations (for attribution)
|
|
1935
|
-
* @param {Object} context - Isolation context { isolationManager, containerId }
|
|
1936
|
-
* @private
|
|
1937
|
-
*/
|
|
1938
|
-
async _handleOperations(clusterId, operations, sender, context = {}) {
|
|
1939
|
-
const cluster = this.clusters.get(clusterId);
|
|
1940
|
-
if (!cluster) {
|
|
1941
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
1942
|
-
}
|
|
1943
|
-
|
|
1944
|
-
this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
|
|
1945
|
-
|
|
1946
|
-
// Phase 1: Pre-validate operation structure
|
|
1947
|
-
const validationErrors = this._validateOperationChain(operations);
|
|
1948
|
-
|
|
1949
|
-
if (validationErrors.length > 0) {
|
|
1950
|
-
const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
|
|
1951
|
-
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
1952
|
-
throw new Error(errorMsg);
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
// Phase 2: Build mock cluster config with proposed agents
|
|
1956
|
-
// Collect all agents that would exist after operations complete
|
|
1957
|
-
const existingAgentConfigs = cluster.config.agents || [];
|
|
1958
|
-
const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
|
|
1959
|
-
|
|
1960
|
-
// Phase 3: Validate proposed cluster config
|
|
1961
|
-
const validation = this._validateProposedConfig(
|
|
1962
|
-
clusterId,
|
|
1963
|
-
cluster,
|
|
1964
|
-
proposedAgentConfigs,
|
|
1965
|
-
operations
|
|
1966
|
-
);
|
|
1967
|
-
|
|
1968
|
-
// Log warnings but proceed
|
|
1969
|
-
if (validation.warnings.length > 0) {
|
|
1970
|
-
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
1971
|
-
for (const warning of validation.warnings) {
|
|
1972
|
-
this._log(` - ${warning}`);
|
|
1973
|
-
}
|
|
1974
|
-
}
|
|
1975
|
-
|
|
1976
|
-
// Phase 4: Execute validated operations
|
|
1977
|
-
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
1978
|
-
|
|
1979
|
-
await this._executeOperations(cluster, operations, sender, context);
|
|
1980
|
-
|
|
1981
|
-
this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
|
|
1982
|
-
|
|
1983
|
-
// Publish success notification
|
|
1984
|
-
cluster.messageBus.publish({
|
|
1985
|
-
cluster_id: clusterId,
|
|
1986
|
-
topic: 'CLUSTER_OPERATIONS_SUCCESS',
|
|
1987
|
-
sender: 'orchestrator',
|
|
1988
|
-
content: {
|
|
1989
|
-
text: `Executed ${operations.length} operation(s)`,
|
|
1990
|
-
data: {
|
|
1991
|
-
operationCount: operations.length,
|
|
1992
|
-
agentCount: cluster.agents.length,
|
|
1993
|
-
},
|
|
1994
|
-
},
|
|
1995
|
-
});
|
|
1996
|
-
|
|
1997
|
-
// Save updated cluster state to disk
|
|
1998
|
-
await this._saveClusters();
|
|
1999
|
-
}
|
|
2000
|
-
|
|
2001
|
-
_validateOperationChain(operations) {
|
|
2002
|
-
const validationErrors = [];
|
|
2003
|
-
|
|
2004
|
-
for (let i = 0; i < operations.length; i++) {
|
|
2005
|
-
const op = operations[i];
|
|
2006
|
-
if (!op.action) {
|
|
2007
|
-
validationErrors.push(`Operation ${i}: missing 'action' field`);
|
|
2008
|
-
continue;
|
|
2009
|
-
}
|
|
2010
|
-
if (!VALID_OPERATIONS.includes(op.action)) {
|
|
2011
|
-
validationErrors.push(
|
|
2012
|
-
`Operation ${i}: unknown action '${op.action}'. Valid: ${VALID_OPERATIONS.join(', ')}`
|
|
2013
|
-
);
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
|
|
2017
|
-
return validationErrors;
|
|
2018
|
-
}
|
|
2019
|
-
|
|
2020
|
-
_buildProposedAgentConfigs(existingAgentConfigs, operations) {
|
|
2021
|
-
const proposedAgentConfigs = [...existingAgentConfigs];
|
|
2022
|
-
|
|
2023
|
-
for (const op of operations) {
|
|
2024
|
-
if (op.action === 'add_agents' && op.agents) {
|
|
2025
|
-
for (const agentConfig of op.agents) {
|
|
2026
|
-
const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
|
|
2027
|
-
if (existingIdx === -1) {
|
|
2028
|
-
proposedAgentConfigs.push(agentConfig);
|
|
2029
|
-
}
|
|
2030
|
-
}
|
|
2031
|
-
} else if (op.action === 'remove_agents' && op.agentIds) {
|
|
2032
|
-
for (const agentId of op.agentIds) {
|
|
2033
|
-
const idx = proposedAgentConfigs.findIndex((a) => a.id === agentId);
|
|
2034
|
-
if (idx !== -1) {
|
|
2035
|
-
proposedAgentConfigs.splice(idx, 1);
|
|
2036
|
-
}
|
|
2037
|
-
}
|
|
2038
|
-
} else if (op.action === 'update_agent' && op.agentId && op.updates) {
|
|
2039
|
-
const agentConfig = proposedAgentConfigs.find((a) => a.id === op.agentId);
|
|
2040
|
-
if (agentConfig) {
|
|
2041
|
-
Object.assign(agentConfig, op.updates);
|
|
2042
|
-
}
|
|
2043
|
-
}
|
|
2044
|
-
}
|
|
2045
|
-
|
|
2046
|
-
return proposedAgentConfigs;
|
|
2047
|
-
}
|
|
2048
|
-
|
|
2049
|
-
_validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
|
|
2050
|
-
const mockConfig = { agents: proposedAgentConfigs };
|
|
2051
|
-
const validation = configValidator.validateConfig(mockConfig);
|
|
2052
|
-
|
|
2053
|
-
if (!validation.valid) {
|
|
2054
|
-
const errorMsg = `Proposed cluster configuration is invalid:\n - ${validation.errors.join('\n - ')}`;
|
|
2055
|
-
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
2056
|
-
|
|
2057
|
-
// Publish validation failure for conductor to see and retry
|
|
2058
|
-
cluster.messageBus.publish({
|
|
2059
|
-
cluster_id: clusterId,
|
|
2060
|
-
topic: 'CLUSTER_OPERATIONS_VALIDATION_FAILED',
|
|
2061
|
-
sender: 'orchestrator',
|
|
2062
|
-
content: {
|
|
2063
|
-
text: 'Operation chain would create invalid cluster configuration',
|
|
2064
|
-
data: {
|
|
2065
|
-
errors: validation.errors,
|
|
2066
|
-
warnings: validation.warnings,
|
|
2067
|
-
operations: operations,
|
|
2068
|
-
},
|
|
2069
|
-
},
|
|
2070
|
-
});
|
|
2071
|
-
|
|
2072
|
-
throw new Error(errorMsg);
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
|
-
return validation;
|
|
2076
|
-
}
|
|
2077
|
-
|
|
2078
|
-
async _executeOperations(cluster, operations, sender, context) {
|
|
2079
|
-
for (let i = 0; i < operations.length; i++) {
|
|
2080
|
-
const op = operations[i];
|
|
2081
|
-
this._log(` [${i + 1}/${operations.length}] ${op.action}`);
|
|
2082
|
-
await this._executeOperation(cluster, op, sender, context);
|
|
2083
|
-
}
|
|
2084
|
-
}
|
|
2085
|
-
|
|
2086
|
-
async _executeOperation(cluster, op, sender, context) {
|
|
2087
|
-
switch (op.action) {
|
|
2088
|
-
case 'add_agents':
|
|
2089
|
-
await this._opAddAgents(cluster, op, context);
|
|
2090
|
-
break;
|
|
2091
|
-
case 'remove_agents':
|
|
2092
|
-
await this._opRemoveAgents(cluster, op);
|
|
2093
|
-
break;
|
|
2094
|
-
case 'update_agent':
|
|
2095
|
-
this._opUpdateAgent(cluster, op);
|
|
2096
|
-
break;
|
|
2097
|
-
case 'publish':
|
|
2098
|
-
this._opPublish(cluster, op, sender);
|
|
2099
|
-
break;
|
|
2100
|
-
case 'load_config':
|
|
2101
|
-
await this._opLoadConfig(cluster, op, context);
|
|
2102
|
-
break;
|
|
2103
|
-
}
|
|
2104
|
-
}
|
|
2105
|
-
|
|
2106
|
-
/**
|
|
2107
|
-
* Operation: add_agents - Spawn new agents dynamically
|
|
2108
|
-
* @private
|
|
2109
|
-
*/
|
|
2110
|
-
async _opAddAgents(cluster, op, context) {
|
|
2111
|
-
const agents = op.agents;
|
|
2112
|
-
if (!agents || !Array.isArray(agents)) {
|
|
2113
|
-
throw new Error('add_agents operation missing agents array');
|
|
2114
|
-
}
|
|
2115
|
-
|
|
2116
|
-
// CRITICAL: Inject cluster's worktree cwd into dynamically added agents
|
|
2117
|
-
// Without this, template agents (planning, implementation, validator) run in
|
|
2118
|
-
// original directory instead of worktree, causing --ship mode to pollute main
|
|
2119
|
-
const agentCwd = cluster.worktree?.path || cluster.isolation?.workDir || process.cwd();
|
|
2120
|
-
|
|
2121
|
-
for (const agentConfig of agents) {
|
|
2122
|
-
// Inject cwd if not already set (same as startCluster does for initial agents)
|
|
2123
|
-
if (!agentConfig.cwd) {
|
|
2124
|
-
agentConfig.cwd = agentCwd;
|
|
2125
|
-
this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
|
|
2126
|
-
}
|
|
2127
|
-
|
|
2128
|
-
// Apply model override if set (for consistency with initial agents)
|
|
2129
|
-
if (cluster.modelOverride) {
|
|
2130
|
-
applyModelOverride(agentConfig, cluster.modelOverride);
|
|
2131
|
-
this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
|
|
2132
|
-
}
|
|
2133
|
-
// Validate agent config has required fields
|
|
2134
|
-
if (!agentConfig.id) {
|
|
2135
|
-
throw new Error('Agent config missing required field: id');
|
|
2136
|
-
}
|
|
2137
|
-
|
|
2138
|
-
// Check for duplicate agent ID
|
|
2139
|
-
const existingAgent = cluster.agents.find((a) => a.id === agentConfig.id);
|
|
2140
|
-
if (existingAgent) {
|
|
2141
|
-
this._log(` ⚠️ Agent ${agentConfig.id} already exists, skipping`);
|
|
2142
|
-
continue;
|
|
2143
|
-
}
|
|
2144
|
-
|
|
2145
|
-
// Add to config agents array (for persistence)
|
|
2146
|
-
if (!cluster.config.agents) {
|
|
2147
|
-
cluster.config.agents = [];
|
|
2148
|
-
}
|
|
2149
|
-
cluster.config.agents.push(agentConfig);
|
|
2150
|
-
|
|
2151
|
-
// Build agent options
|
|
2152
|
-
const agentOptions = {
|
|
2153
|
-
testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
|
|
2154
|
-
quiet: this.quiet,
|
|
2155
|
-
modelOverride: cluster.modelOverride || null,
|
|
2156
|
-
};
|
|
2157
|
-
|
|
2158
|
-
// TaskRunner DI - propagate to dynamically spawned agents
|
|
2159
|
-
if (this.taskRunner) {
|
|
2160
|
-
agentOptions.taskRunner = this.taskRunner;
|
|
2161
|
-
}
|
|
2162
|
-
|
|
2163
|
-
// Pass isolation context if cluster is running in isolation mode
|
|
2164
|
-
if (cluster.isolation?.enabled && context.isolationManager) {
|
|
2165
|
-
agentOptions.isolation = {
|
|
2166
|
-
enabled: true,
|
|
2167
|
-
manager: context.isolationManager,
|
|
2168
|
-
clusterId: cluster.id,
|
|
2169
|
-
};
|
|
2170
|
-
}
|
|
2171
|
-
|
|
2172
|
-
// Create and start agent
|
|
2173
|
-
const agent = new AgentWrapper(agentConfig, cluster.messageBus, cluster, agentOptions);
|
|
2174
|
-
cluster.agents.push(agent);
|
|
2175
|
-
await agent.start();
|
|
2176
|
-
|
|
2177
|
-
this._log(
|
|
2178
|
-
` ✓ Added agent: ${agentConfig.id} (role: ${agentConfig.role || 'unspecified'})`
|
|
2179
|
-
);
|
|
2180
|
-
}
|
|
2181
|
-
}
|
|
2182
|
-
|
|
2183
|
-
/**
|
|
2184
|
-
* Operation: remove_agents - Stop and remove agents by ID
|
|
2185
|
-
* @private
|
|
2186
|
-
*/
|
|
2187
|
-
async _opRemoveAgents(cluster, op) {
|
|
2188
|
-
const agentIds = op.agentIds;
|
|
2189
|
-
if (!agentIds || !Array.isArray(agentIds)) {
|
|
2190
|
-
throw new Error('remove_agents operation missing agentIds array');
|
|
2191
|
-
}
|
|
2192
|
-
|
|
2193
|
-
for (const agentId of agentIds) {
|
|
2194
|
-
const agentIndex = cluster.agents.findIndex((a) => a.id === agentId);
|
|
2195
|
-
if (agentIndex === -1) {
|
|
2196
|
-
this._log(` ⚠️ Agent ${agentId} not found, skipping removal`);
|
|
2197
|
-
continue;
|
|
2198
|
-
}
|
|
2199
|
-
|
|
2200
|
-
const agent = cluster.agents[agentIndex];
|
|
2201
|
-
await agent.stop();
|
|
2202
|
-
|
|
2203
|
-
// Remove from cluster.agents
|
|
2204
|
-
cluster.agents.splice(agentIndex, 1);
|
|
2205
|
-
|
|
2206
|
-
// Remove from config.agents
|
|
2207
|
-
if (cluster.config.agents) {
|
|
2208
|
-
const configIndex = cluster.config.agents.findIndex((a) => a.id === agentId);
|
|
2209
|
-
if (configIndex !== -1) {
|
|
2210
|
-
cluster.config.agents.splice(configIndex, 1);
|
|
2211
|
-
}
|
|
2212
|
-
}
|
|
2213
|
-
|
|
2214
|
-
this._log(` ✓ Removed agent: ${agentId}`);
|
|
2215
|
-
}
|
|
2216
|
-
}
|
|
2217
|
-
|
|
2218
|
-
/**
|
|
2219
|
-
* Operation: update_agent - Modify existing agent config at runtime
|
|
2220
|
-
* Note: Some updates may require agent restart to take effect
|
|
2221
|
-
* @private
|
|
2222
|
-
*/
|
|
2223
|
-
_opUpdateAgent(cluster, op) {
|
|
2224
|
-
const { agentId, updates } = op;
|
|
2225
|
-
if (!agentId) {
|
|
2226
|
-
throw new Error('update_agent operation missing agentId');
|
|
2227
|
-
}
|
|
2228
|
-
if (!updates || typeof updates !== 'object') {
|
|
2229
|
-
throw new Error('update_agent operation missing updates object');
|
|
2230
|
-
}
|
|
2231
|
-
|
|
2232
|
-
const agent = cluster.agents.find((a) => a.id === agentId);
|
|
2233
|
-
if (!agent) {
|
|
2234
|
-
throw new Error(`update_agent: Agent ${agentId} not found`);
|
|
2235
|
-
}
|
|
2236
|
-
|
|
2237
|
-
// Apply updates to agent config
|
|
2238
|
-
Object.assign(agent.config, updates);
|
|
2239
|
-
|
|
2240
|
-
// Also update in cluster.config.agents for persistence
|
|
2241
|
-
if (cluster.config.agents) {
|
|
2242
|
-
const configAgent = cluster.config.agents.find((a) => a.id === agentId);
|
|
2243
|
-
if (configAgent) {
|
|
2244
|
-
Object.assign(configAgent, updates);
|
|
2245
|
-
}
|
|
2246
|
-
}
|
|
2247
|
-
|
|
2248
|
-
this._log(` ✓ Updated agent: ${agentId} (fields: ${Object.keys(updates).join(', ')})`);
|
|
2249
|
-
}
|
|
2250
|
-
|
|
2251
|
-
/**
|
|
2252
|
-
* Operation: publish - Publish a message to the bus
|
|
2253
|
-
* @private
|
|
2254
|
-
*/
|
|
2255
|
-
_opPublish(cluster, op, sender) {
|
|
2256
|
-
const { topic, content, metadata } = op;
|
|
2257
|
-
if (!topic) {
|
|
2258
|
-
throw new Error('publish operation missing topic');
|
|
2259
|
-
}
|
|
2260
|
-
|
|
2261
|
-
cluster.messageBus.publish({
|
|
2262
|
-
cluster_id: cluster.id,
|
|
2263
|
-
topic,
|
|
2264
|
-
sender: op.sender || sender,
|
|
2265
|
-
receiver: op.receiver || 'broadcast',
|
|
2266
|
-
content: content || {},
|
|
2267
|
-
metadata: metadata || null,
|
|
2268
|
-
});
|
|
2269
|
-
|
|
2270
|
-
this._log(` ✓ Published to topic: ${topic}`);
|
|
2271
|
-
}
|
|
2272
|
-
|
|
2273
|
-
/**
|
|
2274
|
-
* Operation: load_config - Load agents from a cluster config
|
|
2275
|
-
*
|
|
2276
|
-
* Supports two formats:
|
|
2277
|
-
* 1. Static config: { config: 'config-name' } - loads from cluster-templates/config-name.json
|
|
2278
|
-
* 2. Parameterized: { config: { base: 'template-name', params: {...} } } - resolves base template with params
|
|
2279
|
-
*
|
|
2280
|
-
* @private
|
|
2281
|
-
*/
|
|
2282
|
-
async _opLoadConfig(cluster, op, context) {
|
|
2283
|
-
const { config } = op;
|
|
2284
|
-
if (!config) {
|
|
2285
|
-
throw new Error('load_config operation missing config');
|
|
2286
|
-
}
|
|
2287
|
-
|
|
2288
|
-
const templatesDir = path.join(__dirname, '..', 'cluster-templates');
|
|
2289
|
-
let loadedConfig;
|
|
2290
|
-
|
|
2291
|
-
// Check if config is parameterized ({ base, params }) or static (string)
|
|
2292
|
-
if (typeof config === 'object' && config.base) {
|
|
2293
|
-
// Parameterized template - resolve with TemplateResolver
|
|
2294
|
-
const { base, params } = config;
|
|
2295
|
-
this._log(` Loading parameterized template: ${base}`);
|
|
2296
|
-
this._log(` Params: ${JSON.stringify(params)}`);
|
|
2297
|
-
|
|
2298
|
-
const resolver = new TemplateResolver(templatesDir);
|
|
2299
|
-
loadedConfig = resolver.resolve(base, params);
|
|
2300
|
-
|
|
2301
|
-
this._log(` ✓ Resolved template: ${base} → ${loadedConfig.agents?.length || 0} agent(s)`);
|
|
2302
|
-
} else if (typeof config === 'string') {
|
|
2303
|
-
// Static config - load directly from file
|
|
2304
|
-
const configPath = path.join(templatesDir, `${config}.json`);
|
|
2305
|
-
|
|
2306
|
-
if (!fs.existsSync(configPath)) {
|
|
2307
|
-
throw new Error(`Config not found: ${config} (looked in ${configPath})`);
|
|
2308
|
-
}
|
|
2309
|
-
|
|
2310
|
-
this._log(` Loading static config: ${config}`);
|
|
2311
|
-
|
|
2312
|
-
const configContent = fs.readFileSync(configPath, 'utf8');
|
|
2313
|
-
loadedConfig = JSON.parse(configContent);
|
|
2314
|
-
} else {
|
|
2315
|
-
throw new Error(
|
|
2316
|
-
`Invalid config format: expected string or {base, params}, got ${typeof config}`
|
|
2317
|
-
);
|
|
2318
|
-
}
|
|
2319
|
-
|
|
2320
|
-
if (!loadedConfig.agents || !Array.isArray(loadedConfig.agents)) {
|
|
2321
|
-
throw new Error(`Config has no agents array`);
|
|
2322
|
-
}
|
|
2323
|
-
|
|
2324
|
-
this._log(` Found ${loadedConfig.agents.length} agent(s)`);
|
|
2325
|
-
|
|
2326
|
-
// Add agents from loaded config (reuse existing add_agents logic)
|
|
2327
|
-
await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
|
|
2328
|
-
|
|
2329
|
-
this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
|
|
2330
|
-
|
|
2331
|
-
// Inject completion agent (templates don't include one - orchestrator controls termination)
|
|
2332
|
-
await this._injectCompletionAgent(cluster, context);
|
|
2333
|
-
}
|
|
2334
|
-
|
|
2335
|
-
/**
|
|
2336
|
-
* Inject appropriate completion agent based on mode
|
|
2337
|
-
* Templates define work, orchestrator controls termination strategy
|
|
2338
|
-
* @private
|
|
2339
|
-
*/
|
|
2340
|
-
async _injectCompletionAgent(cluster, context) {
|
|
2341
|
-
// Skip if completion agent already exists
|
|
2342
|
-
const hasCompletionAgent = cluster.agents.some(
|
|
2343
|
-
(a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
|
|
2344
|
-
);
|
|
2345
|
-
if (hasCompletionAgent) {
|
|
2346
|
-
return;
|
|
2347
|
-
}
|
|
2348
|
-
|
|
2349
|
-
const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
|
|
2350
|
-
|
|
2351
|
-
if (isPrMode) {
|
|
2352
|
-
// Detect platform from stored cluster metadata OR git context
|
|
2353
|
-
let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
|
|
2354
|
-
|
|
2355
|
-
// Fallback to git context detection if cluster metadata missing
|
|
2356
|
-
if (!platform) {
|
|
2357
|
-
const { getPlatformForPR } = require('./issue-providers');
|
|
2358
|
-
try {
|
|
2359
|
-
platform = getPlatformForPR(cluster.cwd || process.cwd());
|
|
2360
|
-
} catch (error) {
|
|
2361
|
-
throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
|
|
2362
|
-
}
|
|
2363
|
-
}
|
|
2364
|
-
|
|
2365
|
-
// Generate platform-specific git-pusher agent from template
|
|
2366
|
-
const {
|
|
2367
|
-
generateGitPusherAgent,
|
|
2368
|
-
isPlatformSupported,
|
|
2369
|
-
} = require('./agents/git-pusher-template');
|
|
2370
|
-
|
|
2371
|
-
if (!isPlatformSupported(platform)) {
|
|
2372
|
-
throw new Error(
|
|
2373
|
-
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
2374
|
-
);
|
|
2375
|
-
}
|
|
2376
|
-
|
|
2377
|
-
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
2378
|
-
|
|
2379
|
-
// Get issue context from ledger
|
|
2380
|
-
const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
|
|
2381
|
-
const issueNumber = issueMsg?.content?.data?.number || 'unknown';
|
|
2382
|
-
const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
|
|
2383
|
-
|
|
2384
|
-
// Inject issue context into prompt
|
|
2385
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
2386
|
-
.replace(/\{\{issue_number\}\}/g, issueNumber)
|
|
2387
|
-
.replace(/\{\{issue_title\}\}/g, issueTitle);
|
|
2388
|
-
|
|
2389
|
-
await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
|
|
2390
|
-
this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
|
|
2391
|
-
} else {
|
|
2392
|
-
// Default completion-detector
|
|
2393
|
-
const completionDetector = {
|
|
2394
|
-
id: 'completion-detector',
|
|
2395
|
-
role: 'orchestrator',
|
|
2396
|
-
model: 'haiku',
|
|
2397
|
-
timeout: 0,
|
|
2398
|
-
triggers: [
|
|
2399
|
-
{
|
|
2400
|
-
topic: 'VALIDATION_RESULT',
|
|
2401
|
-
logic: {
|
|
2402
|
-
engine: 'javascript',
|
|
2403
|
-
script: `const validators = cluster.getAgentsByRole('validator');
|
|
2404
|
-
const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });
|
|
2405
|
-
if (!lastPush) return false;
|
|
2406
|
-
if (validators.length === 0) return true;
|
|
2407
|
-
|
|
2408
|
-
const validatorIds = new Set(validators.map((v) => v.id));
|
|
2409
|
-
const results = ledger.query({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });
|
|
2410
|
-
|
|
2411
|
-
const latestByValidator = new Map();
|
|
2412
|
-
for (const msg of results) {
|
|
2413
|
-
if (!validatorIds.has(msg.sender)) continue;
|
|
2414
|
-
latestByValidator.set(msg.sender, msg);
|
|
2415
|
-
}
|
|
2416
|
-
|
|
2417
|
-
if (latestByValidator.size < validators.length) return false;
|
|
2418
|
-
|
|
2419
|
-
for (const validator of validators) {
|
|
2420
|
-
const msg = latestByValidator.get(validator.id);
|
|
2421
|
-
const approved = msg?.content?.data?.approved;
|
|
2422
|
-
if (!(approved === true || approved === 'true')) return false;
|
|
2423
|
-
}
|
|
2424
|
-
|
|
2425
|
-
return true;`,
|
|
2426
|
-
},
|
|
2427
|
-
action: 'stop_cluster',
|
|
2428
|
-
},
|
|
2429
|
-
],
|
|
2430
|
-
};
|
|
2431
|
-
|
|
2432
|
-
await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
|
|
2433
|
-
this._log(` Injected completion-detector agent`);
|
|
2434
|
-
}
|
|
2435
|
-
}
|
|
2436
|
-
|
|
2437
|
-
/**
|
|
2438
|
-
* Check if a process with given PID is running
|
|
2439
|
-
* @param {Number} pid - Process ID
|
|
2440
|
-
* @returns {Boolean} True if process exists
|
|
2441
|
-
* @private
|
|
2442
|
-
*/
|
|
2443
|
-
_isProcessRunning(pid) {
|
|
2444
|
-
if (!pid) return false;
|
|
2445
|
-
try {
|
|
2446
|
-
// Signal 0 doesn't kill, just checks if process exists
|
|
2447
|
-
process.kill(pid, 0);
|
|
2448
|
-
return true;
|
|
2449
|
-
} catch (e) {
|
|
2450
|
-
// ESRCH = No such process, EPERM = process exists but no permission
|
|
2451
|
-
return e.code === 'EPERM';
|
|
2452
|
-
}
|
|
2453
|
-
}
|
|
2454
|
-
|
|
2455
|
-
/**
|
|
2456
|
-
* Get cluster status
|
|
2457
|
-
* @param {String} clusterId - Cluster ID
|
|
2458
|
-
* @returns {Object} Cluster status
|
|
2459
|
-
*/
|
|
2460
|
-
getStatus(clusterId) {
|
|
2461
|
-
const cluster = this.clusters.get(clusterId);
|
|
2462
|
-
if (!cluster) {
|
|
2463
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
2464
|
-
}
|
|
2465
|
-
|
|
2466
|
-
// Detect zombie clusters: state=running but no backing process
|
|
2467
|
-
let state = cluster.state;
|
|
2468
|
-
let isZombie = false;
|
|
2469
|
-
if (state === 'running') {
|
|
2470
|
-
if (cluster.pid) {
|
|
2471
|
-
// PID recorded - check if process is running
|
|
2472
|
-
if (!this._isProcessRunning(cluster.pid)) {
|
|
2473
|
-
state = 'zombie';
|
|
2474
|
-
isZombie = true;
|
|
2475
|
-
this._log(
|
|
2476
|
-
`[Orchestrator] Detected zombie cluster ${clusterId} (PID ${cluster.pid} not running)`
|
|
2477
|
-
);
|
|
2478
|
-
}
|
|
2479
|
-
} else {
|
|
2480
|
-
// No PID recorded (legacy cluster or bug) - definitely a zombie
|
|
2481
|
-
// New code always records PID for running clusters
|
|
2482
|
-
state = 'zombie';
|
|
2483
|
-
isZombie = true;
|
|
2484
|
-
this._log(
|
|
2485
|
-
`[Orchestrator] Detected zombie cluster ${clusterId} (no PID recorded - legacy or killed cluster)`
|
|
2486
|
-
);
|
|
2487
|
-
}
|
|
2488
|
-
}
|
|
2489
|
-
|
|
2490
|
-
return {
|
|
2491
|
-
id: clusterId,
|
|
2492
|
-
state: state,
|
|
2493
|
-
isZombie: isZombie,
|
|
2494
|
-
pid: cluster.pid || null,
|
|
2495
|
-
createdAt: cluster.createdAt,
|
|
2496
|
-
agents: cluster.agents.map((a) => a.getState()),
|
|
2497
|
-
messageCount: cluster.messageBus.count({ cluster_id: clusterId }),
|
|
2498
|
-
};
|
|
2499
|
-
}
|
|
2500
|
-
|
|
2501
|
-
/**
|
|
2502
|
-
* List all clusters
|
|
2503
|
-
* @returns {Array} List of cluster summaries
|
|
2504
|
-
*/
|
|
2505
|
-
listClusters() {
|
|
2506
|
-
return Array.from(this.clusters.values()).map((cluster) => {
|
|
2507
|
-
// Detect zombie clusters (state=running but no backing process)
|
|
2508
|
-
let state = cluster.state;
|
|
2509
|
-
if (state === 'running') {
|
|
2510
|
-
if (cluster.pid) {
|
|
2511
|
-
if (!this._isProcessRunning(cluster.pid)) {
|
|
2512
|
-
state = 'zombie';
|
|
2513
|
-
}
|
|
2514
|
-
} else {
|
|
2515
|
-
// No PID recorded - definitely a zombie
|
|
2516
|
-
state = 'zombie';
|
|
2517
|
-
}
|
|
2518
|
-
}
|
|
2519
|
-
|
|
2520
|
-
return {
|
|
2521
|
-
id: cluster.id,
|
|
2522
|
-
state: state,
|
|
2523
|
-
createdAt: cluster.createdAt,
|
|
2524
|
-
agentCount: cluster.agents.length,
|
|
2525
|
-
messageCount: cluster.messageBus.getAll(cluster.id).length,
|
|
2526
|
-
};
|
|
2527
|
-
});
|
|
2528
|
-
}
|
|
2529
|
-
|
|
2530
|
-
/**
|
|
2531
|
-
* Get cluster object (for internal use)
|
|
2532
|
-
* @param {String} clusterId - Cluster ID
|
|
2533
|
-
* @returns {Object} Full cluster object
|
|
2534
|
-
*/
|
|
2535
|
-
getCluster(clusterId) {
|
|
2536
|
-
return this.clusters.get(clusterId);
|
|
2537
|
-
}
|
|
2538
|
-
|
|
2539
|
-
/**
|
|
2540
|
-
* Export cluster conversation
|
|
2541
|
-
* @param {String} clusterId - Cluster ID
|
|
2542
|
-
* @param {String} format - Export format ('json' or 'markdown')
|
|
2543
|
-
* @returns {String} Exported data
|
|
2544
|
-
*/
|
|
2545
|
-
export(clusterId, format = 'json') {
|
|
2546
|
-
const cluster = this.clusters.get(clusterId);
|
|
2547
|
-
if (!cluster) {
|
|
2548
|
-
throw new Error(`Cluster ${clusterId} not found`);
|
|
2549
|
-
}
|
|
2550
|
-
|
|
2551
|
-
const messages = cluster.messageBus.getAll(clusterId);
|
|
2552
|
-
|
|
2553
|
-
if (format === 'json') {
|
|
2554
|
-
return JSON.stringify(
|
|
2555
|
-
{
|
|
2556
|
-
cluster_id: clusterId,
|
|
2557
|
-
state: cluster.state,
|
|
2558
|
-
created_at: cluster.createdAt,
|
|
2559
|
-
agents: cluster.agents.map((a) => a.getState()),
|
|
2560
|
-
messages,
|
|
2561
|
-
},
|
|
2562
|
-
null,
|
|
2563
|
-
2
|
|
2564
|
-
);
|
|
2565
|
-
} else if (format === 'markdown') {
|
|
2566
|
-
return this._exportMarkdown(cluster, clusterId, messages);
|
|
2567
|
-
} else {
|
|
2568
|
-
throw new Error(`Unknown export format: ${format}`);
|
|
2569
|
-
}
|
|
2570
|
-
}
|
|
2571
|
-
|
|
2572
|
-
/**
|
|
2573
|
-
* Export cluster as nicely formatted markdown
|
|
2574
|
-
* @private
|
|
2575
|
-
*/
|
|
2576
|
-
_exportMarkdown(cluster, clusterId, messages) {
|
|
2577
|
-
const { parseProviderChunk } = require('./providers');
|
|
2578
|
-
|
|
2579
|
-
// Find task info
|
|
2580
|
-
const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
2581
|
-
const taskText = issueOpened?.content?.text || 'Unknown task';
|
|
2582
|
-
|
|
2583
|
-
// Calculate duration
|
|
2584
|
-
const firstMsg = messages[0];
|
|
2585
|
-
const lastMsg = messages[messages.length - 1];
|
|
2586
|
-
const durationMs = lastMsg ? lastMsg.timestamp - firstMsg.timestamp : 0;
|
|
2587
|
-
const durationMin = Math.round(durationMs / 60000);
|
|
2588
|
-
|
|
2589
|
-
// Header
|
|
2590
|
-
let md = `# Cluster: ${clusterId}\n\n`;
|
|
2591
|
-
md += `| Property | Value |\n|----------|-------|\n`;
|
|
2592
|
-
md += `| State | ${cluster.state} |\n`;
|
|
2593
|
-
md += `| Created | ${new Date(cluster.createdAt).toLocaleString()} |\n`;
|
|
2594
|
-
md += `| Duration | ${durationMin} minutes |\n`;
|
|
2595
|
-
md += `| Agents | ${cluster.agents.map((a) => a.id).join(', ')} |\n\n`;
|
|
2596
|
-
|
|
2597
|
-
// Task
|
|
2598
|
-
md += `## Task\n\n${taskText}\n\n`;
|
|
2599
|
-
|
|
2600
|
-
// Group messages by agent for cleaner output
|
|
2601
|
-
const agentOutputs = this._collectAgentOutputs(messages);
|
|
2602
|
-
|
|
2603
|
-
for (const [agentId, agentMsgs] of agentOutputs) {
|
|
2604
|
-
md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
|
|
2605
|
-
}
|
|
2606
|
-
|
|
2607
|
-
md += this._renderValidationMarkdown(messages);
|
|
2608
|
-
|
|
2609
|
-
// Final status
|
|
2610
|
-
const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
|
|
2611
|
-
if (clusterComplete) {
|
|
2612
|
-
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
2613
|
-
}
|
|
2614
|
-
|
|
2615
|
-
return md;
|
|
2616
|
-
}
|
|
2617
|
-
|
|
2618
|
-
_collectAgentOutputs(messages) {
|
|
2619
|
-
const agentOutputs = new Map();
|
|
2620
|
-
|
|
2621
|
-
for (const msg of messages) {
|
|
2622
|
-
if (msg.topic !== 'AGENT_OUTPUT') continue;
|
|
2623
|
-
if (!agentOutputs.has(msg.sender)) {
|
|
2624
|
-
agentOutputs.set(msg.sender, []);
|
|
2625
|
-
}
|
|
2626
|
-
agentOutputs.get(msg.sender).push(msg);
|
|
2627
|
-
}
|
|
2628
|
-
|
|
2629
|
-
return agentOutputs;
|
|
2630
|
-
}
|
|
2631
|
-
|
|
2632
|
-
_extractAgentOutput(agentMsgs, parseProviderChunk) {
|
|
2633
|
-
let text = '';
|
|
2634
|
-
const tools = [];
|
|
2635
|
-
|
|
2636
|
-
for (const msg of agentMsgs) {
|
|
2637
|
-
const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
|
|
2638
|
-
if (!content) continue;
|
|
2639
|
-
|
|
2640
|
-
const provider = normalizeProviderName(
|
|
2641
|
-
msg.content?.data?.provider || msg.sender_provider || 'claude'
|
|
2642
|
-
);
|
|
2643
|
-
const events = parseProviderChunk(provider, content);
|
|
2644
|
-
for (const event of events) {
|
|
2645
|
-
switch (event.type) {
|
|
2646
|
-
case 'text':
|
|
2647
|
-
text += event.text;
|
|
2648
|
-
break;
|
|
2649
|
-
case 'tool_call':
|
|
2650
|
-
tools.push({ name: event.toolName, input: event.input });
|
|
2651
|
-
break;
|
|
2652
|
-
case 'tool_result':
|
|
2653
|
-
if (tools.length > 0) {
|
|
2654
|
-
const lastTool = tools[tools.length - 1];
|
|
2655
|
-
lastTool.result = event.content;
|
|
2656
|
-
lastTool.isError = event.isError;
|
|
2657
|
-
}
|
|
2658
|
-
break;
|
|
2659
|
-
}
|
|
2660
|
-
}
|
|
2661
|
-
}
|
|
2662
|
-
|
|
2663
|
-
return { text, tools };
|
|
2664
|
-
}
|
|
2665
|
-
|
|
2666
|
-
_renderToolMarkdown(tools) {
|
|
2667
|
-
let md = `### Tools Used\n\n`;
|
|
2668
|
-
|
|
2669
|
-
for (const tool of tools) {
|
|
2670
|
-
const status = tool.isError ? '❌' : '✓';
|
|
2671
|
-
md += `- **${tool.name}** ${status}\n`;
|
|
2672
|
-
if (tool.input) {
|
|
2673
|
-
const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
|
|
2674
|
-
if (inputStr.length < 100) {
|
|
2675
|
-
md += ` - Input: \`${inputStr}\`\n`;
|
|
2676
|
-
}
|
|
2677
|
-
}
|
|
2678
|
-
}
|
|
2679
|
-
|
|
2680
|
-
return `${md}\n`;
|
|
2681
|
-
}
|
|
2682
|
-
|
|
2683
|
-
_renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
|
|
2684
|
-
let md = `## Agent: ${agentId}\n\n`;
|
|
2685
|
-
const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
|
|
2686
|
-
|
|
2687
|
-
if (text.trim()) {
|
|
2688
|
-
md += `### Output\n\n${text.trim()}\n\n`;
|
|
2689
|
-
}
|
|
2690
|
-
|
|
2691
|
-
if (tools.length > 0) {
|
|
2692
|
-
md += this._renderToolMarkdown(tools);
|
|
2693
|
-
}
|
|
2694
|
-
|
|
2695
|
-
return md;
|
|
2696
|
-
}
|
|
2697
|
-
|
|
2698
|
-
_renderCriteriaMarkdown(criteriaResults) {
|
|
2699
|
-
if (!Array.isArray(criteriaResults)) {
|
|
2700
|
-
return '';
|
|
2701
|
-
}
|
|
2702
|
-
|
|
2703
|
-
let md = '';
|
|
2704
|
-
const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
|
|
2705
|
-
if (cannotValidateYet.length > 0) {
|
|
2706
|
-
md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
|
|
2707
|
-
for (const cv of cannotValidateYet) {
|
|
2708
|
-
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2709
|
-
}
|
|
2710
|
-
md += '\n';
|
|
2711
|
-
}
|
|
2712
|
-
|
|
2713
|
-
const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
|
|
2714
|
-
if (cannotValidate.length > 0) {
|
|
2715
|
-
md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
|
|
2716
|
-
for (const cv of cannotValidate) {
|
|
2717
|
-
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2718
|
-
}
|
|
2719
|
-
md += '\n';
|
|
2720
|
-
}
|
|
2721
|
-
|
|
2722
|
-
return md;
|
|
2723
|
-
}
|
|
2724
|
-
|
|
2725
|
-
_renderValidationEntryMarkdown(validation) {
|
|
2726
|
-
const data = validation.content?.data || {};
|
|
2727
|
-
const approved = data.approved === true || data.approved === 'true';
|
|
2728
|
-
const icon = approved ? '✅' : '❌';
|
|
2729
|
-
let md = `### ${validation.sender} ${icon}\n\n`;
|
|
2730
|
-
|
|
2731
|
-
if (data.summary) {
|
|
2732
|
-
md += `${data.summary}\n\n`;
|
|
2733
|
-
}
|
|
2734
|
-
|
|
2735
|
-
if (!approved && data.issues) {
|
|
2736
|
-
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
2737
|
-
if (Array.isArray(issues) && issues.length > 0) {
|
|
2738
|
-
md += `**Issues:**\n`;
|
|
2739
|
-
for (const issue of issues) {
|
|
2740
|
-
md += `- ${issue}\n`;
|
|
2741
|
-
}
|
|
2742
|
-
md += '\n';
|
|
2743
|
-
}
|
|
2744
|
-
}
|
|
2745
|
-
|
|
2746
|
-
md += this._renderCriteriaMarkdown(data.criteriaResults);
|
|
2747
|
-
return md;
|
|
2748
|
-
}
|
|
2749
|
-
|
|
2750
|
-
_renderValidationMarkdown(messages) {
|
|
2751
|
-
const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
|
|
2752
|
-
if (validations.length === 0) {
|
|
2753
|
-
return '';
|
|
2754
|
-
}
|
|
2755
|
-
|
|
2756
|
-
let md = `## Validation Results\n\n`;
|
|
2757
|
-
for (const validation of validations) {
|
|
2758
|
-
md += this._renderValidationEntryMarkdown(validation);
|
|
2759
|
-
}
|
|
2760
|
-
|
|
2761
|
-
return md;
|
|
2762
|
-
}
|
|
2763
|
-
|
|
2764
|
-
/**
|
|
2765
|
-
* Validate cluster configuration (delegates to config-validator module)
|
|
2766
|
-
* @param {Object} config - Cluster configuration
|
|
2767
|
-
* @param {Object} options - Validation options
|
|
2768
|
-
* @param {boolean} options.strict - Treat warnings as errors (default: false)
|
|
2769
|
-
* @returns {Object} { valid: Boolean, errors: Array, warnings: Array }
|
|
2770
|
-
*/
|
|
2771
|
-
validateConfig(config, options = {}) {
|
|
2772
|
-
const result = configValidator.validateConfig(config);
|
|
2773
|
-
|
|
2774
|
-
// In strict mode, warnings become errors
|
|
2775
|
-
if (options.strict && result.warnings.length > 0) {
|
|
2776
|
-
result.errors.push(...result.warnings.map((w) => `[strict] ${w}`));
|
|
2777
|
-
result.valid = false;
|
|
2778
|
-
}
|
|
2779
|
-
|
|
2780
|
-
return result;
|
|
2781
|
-
}
|
|
2782
|
-
|
|
2783
|
-
/**
|
|
2784
|
-
* Load cluster configuration from file
|
|
2785
|
-
* @param {String} configPath - Path to config JSON file
|
|
2786
|
-
* @param {Object} options - Load options
|
|
2787
|
-
* @param {boolean} options.strict - Treat warnings as errors
|
|
2788
|
-
* @returns {Object} Parsed configuration
|
|
2789
|
-
*/
|
|
2790
|
-
loadConfig(configPath, options = {}) {
|
|
2791
|
-
const fullPath = path.resolve(configPath);
|
|
2792
|
-
const content = fs.readFileSync(fullPath, 'utf8');
|
|
2793
|
-
const config = JSON.parse(content);
|
|
2794
|
-
|
|
2795
|
-
const validation = this.validateConfig(config, options);
|
|
2796
|
-
|
|
2797
|
-
// Show warnings (but don't fail unless strict mode)
|
|
2798
|
-
if (validation.warnings && validation.warnings.length > 0 && !this.quiet) {
|
|
2799
|
-
console.warn('\n⚠️ Configuration warnings:');
|
|
2800
|
-
for (const warning of validation.warnings) {
|
|
2801
|
-
console.warn(` ${warning}`);
|
|
2802
|
-
}
|
|
2803
|
-
console.warn('');
|
|
2804
|
-
}
|
|
2805
|
-
|
|
2806
|
-
if (!validation.valid) {
|
|
2807
|
-
const errorMsg = validation.errors.join('\n ');
|
|
2808
|
-
throw new Error(`Invalid config:\n ${errorMsg}`);
|
|
2809
|
-
}
|
|
2810
|
-
|
|
2811
|
-
return config;
|
|
2812
|
-
}
|
|
2813
|
-
}
|
|
2814
|
-
|
|
2815
|
-
module.exports = Orchestrator;
|