@covibes/zeroshot 5.3.0 → 5.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +8 -385
  3. package/cli/index.js +108 -4615
  4. package/package.json +16 -125
  5. package/CHANGELOG.md +0 -537
  6. package/cli/commands/providers.js +0 -150
  7. package/cli/lib/first-run.js +0 -211
  8. package/cli/lib/update-checker.js +0 -281
  9. package/cli/message-formatter-utils.js +0 -75
  10. package/cli/message-formatters-normal.js +0 -253
  11. package/cli/message-formatters-watch.js +0 -181
  12. package/cluster-templates/base-templates/debug-workflow.json +0 -375
  13. package/cluster-templates/base-templates/full-workflow.json +0 -697
  14. package/cluster-templates/base-templates/single-worker.json +0 -63
  15. package/cluster-templates/base-templates/worker-validator.json +0 -152
  16. package/cluster-templates/conductor-bootstrap.json +0 -124
  17. package/docker/zeroshot-cluster/Dockerfile +0 -152
  18. package/docker/zeroshot-cluster/pre-baked-deps.json +0 -28
  19. package/lib/completion.js +0 -174
  20. package/lib/docker-config.js +0 -212
  21. package/lib/id-detector.js +0 -53
  22. package/lib/provider-detection.js +0 -59
  23. package/lib/provider-names.js +0 -56
  24. package/lib/settings.js +0 -405
  25. package/lib/stream-json-parser.js +0 -10
  26. package/scripts/fix-node-pty-permissions.js +0 -75
  27. package/scripts/record-demo.sh +0 -279
  28. package/scripts/test-install.sh +0 -40
  29. package/scripts/validate-templates.js +0 -100
  30. package/src/agent/agent-config.js +0 -193
  31. package/src/agent/agent-context-builder.js +0 -335
  32. package/src/agent/agent-hook-executor.js +0 -402
  33. package/src/agent/agent-lifecycle.js +0 -683
  34. package/src/agent/agent-stuck-detector.js +0 -256
  35. package/src/agent/agent-task-executor.js +0 -1431
  36. package/src/agent/agent-trigger-evaluator.js +0 -67
  37. package/src/agent/output-extraction.js +0 -200
  38. package/src/agent/output-reformatter.js +0 -175
  39. package/src/agent/schema-utils.js +0 -111
  40. package/src/agent-wrapper.js +0 -570
  41. package/src/agents/git-pusher-agent.json +0 -20
  42. package/src/attach/attach-client.js +0 -438
  43. package/src/attach/attach-server.js +0 -543
  44. package/src/attach/index.js +0 -35
  45. package/src/attach/protocol.js +0 -220
  46. package/src/attach/ring-buffer.js +0 -121
  47. package/src/attach/socket-discovery.js +0 -267
  48. package/src/claude-task-runner.js +0 -523
  49. package/src/config-router.js +0 -80
  50. package/src/config-validator.js +0 -1660
  51. package/src/copy-worker.js +0 -43
  52. package/src/github.js +0 -139
  53. package/src/isolation-manager.js +0 -1381
  54. package/src/ledger.js +0 -614
  55. package/src/logic-engine.js +0 -223
  56. package/src/message-bus-bridge.js +0 -139
  57. package/src/message-bus.js +0 -249
  58. package/src/name-generator.js +0 -232
  59. package/src/orchestrator.js +0 -2519
  60. package/src/preflight.js +0 -485
  61. package/src/process-metrics.js +0 -566
  62. package/src/providers/anthropic/cli-builder.js +0 -45
  63. package/src/providers/anthropic/index.js +0 -134
  64. package/src/providers/anthropic/models.js +0 -23
  65. package/src/providers/anthropic/output-parser.js +0 -159
  66. package/src/providers/base-provider.js +0 -181
  67. package/src/providers/capabilities.js +0 -51
  68. package/src/providers/google/cli-builder.js +0 -55
  69. package/src/providers/google/index.js +0 -116
  70. package/src/providers/google/models.js +0 -24
  71. package/src/providers/google/output-parser.js +0 -92
  72. package/src/providers/index.js +0 -75
  73. package/src/providers/openai/cli-builder.js +0 -122
  74. package/src/providers/openai/index.js +0 -135
  75. package/src/providers/openai/models.js +0 -21
  76. package/src/providers/openai/output-parser.js +0 -129
  77. package/src/schemas/sub-cluster.js +0 -156
  78. package/src/status-footer.js +0 -980
  79. package/src/sub-cluster-wrapper.js +0 -560
  80. package/src/task-runner.js +0 -30
  81. package/src/template-resolver.js +0 -369
  82. package/src/tui/CHANGES.txt +0 -133
  83. package/src/tui/LAYOUT.md +0 -261
  84. package/src/tui/README.txt +0 -192
  85. package/src/tui/TWO-LEVEL-NAVIGATION.md +0 -186
  86. package/src/tui/data-poller.js +0 -325
  87. package/src/tui/demo.js +0 -208
  88. package/src/tui/formatters.js +0 -128
  89. package/src/tui/index.js +0 -193
  90. package/src/tui/keybindings.js +0 -383
  91. package/src/tui/layout.js +0 -334
  92. package/src/tui/renderer.js +0 -194
  93. package/task-lib/attachable-watcher.js +0 -272
  94. package/task-lib/claude-recovery.js +0 -119
  95. package/task-lib/commands/clean.js +0 -50
  96. package/task-lib/commands/get-log-path.js +0 -23
  97. package/task-lib/commands/kill.js +0 -32
  98. package/task-lib/commands/list.js +0 -105
  99. package/task-lib/commands/logs.js +0 -411
  100. package/task-lib/commands/resume.js +0 -42
  101. package/task-lib/commands/run.js +0 -57
  102. package/task-lib/commands/schedule.js +0 -105
  103. package/task-lib/commands/scheduler-cmd.js +0 -96
  104. package/task-lib/commands/schedules.js +0 -98
  105. package/task-lib/commands/status.js +0 -44
  106. package/task-lib/commands/unschedule.js +0 -16
  107. package/task-lib/completion.js +0 -9
  108. package/task-lib/config.js +0 -10
  109. package/task-lib/name-generator.js +0 -230
  110. package/task-lib/package.json +0 -3
  111. package/task-lib/runner.js +0 -144
  112. package/task-lib/scheduler.js +0 -252
  113. package/task-lib/store.js +0 -230
  114. package/task-lib/tui/formatters.js +0 -166
  115. package/task-lib/tui/index.js +0 -197
  116. package/task-lib/tui/layout.js +0 -111
  117. package/task-lib/tui/renderer.js +0 -119
  118. package/task-lib/tui.js +0 -384
  119. package/task-lib/watcher.js +0 -188
@@ -1,2519 +0,0 @@
1
- /**
2
- * Orchestrator - Manages cluster lifecycle
3
- *
4
- * Provides:
5
- * - Cluster initialization and configuration
6
- * - Agent lifecycle management
7
- * - GitHub issue integration
8
- * - Cluster state tracking
9
- * - Crash recovery
10
- */
11
-
12
- const fs = require('fs');
13
- const path = require('path');
14
- const os = require('os');
15
- const lockfile = require('proper-lockfile');
16
-
17
- // Stale lock timeout in ms - if lock file is older than this, delete it
18
- const LOCK_STALE_MS = 5000;
19
-
20
- /**
21
- * Remove lock file if it's stale (older than LOCK_STALE_MS)
22
- * Handles crashes that leave orphaned lock files
23
- */
24
- function cleanStaleLock(lockPath) {
25
- try {
26
- if (fs.existsSync(lockPath)) {
27
- const age = Date.now() - fs.statSync(lockPath).mtimeMs;
28
- if (age > LOCK_STALE_MS) {
29
- fs.unlinkSync(lockPath);
30
- }
31
- }
32
- } catch {
33
- // Ignore - another process may have cleaned it
34
- }
35
- }
36
- const AgentWrapper = require('./agent-wrapper');
37
- const SubClusterWrapper = require('./sub-cluster-wrapper');
38
- const MessageBus = require('./message-bus');
39
- const Ledger = require('./ledger');
40
- const GitHub = require('./github');
41
- const IsolationManager = require('./isolation-manager');
42
- const { generateName } = require('./name-generator');
43
- const configValidator = require('./config-validator');
44
- const TemplateResolver = require('./template-resolver');
45
- const { loadSettings } = require('../lib/settings');
46
- const { normalizeProviderName } = require('../lib/provider-names');
47
- const crypto = require('crypto');
48
-
49
- function applyModelOverride(agentConfig, modelOverride) {
50
- if (!modelOverride) return;
51
-
52
- agentConfig.model = modelOverride;
53
- if (agentConfig.modelRules) {
54
- delete agentConfig.modelRules;
55
- }
56
- if (agentConfig.modelConfig) {
57
- delete agentConfig.modelConfig;
58
- }
59
- }
60
-
61
- /**
62
- * Operation Chain Schema
63
- * Conductor (or any agent) can publish CLUSTER_OPERATIONS to dynamically modify cluster
64
- *
65
- * Supported operations:
66
- * - add_agents: Spawn new agents with given configs
67
- * - remove_agents: Stop and remove agents by ID
68
- * - update_agent: Modify existing agent config
69
- * - publish: Publish a message to the bus
70
- * - load_config: Load agents from a named cluster config template
71
- */
72
- const VALID_OPERATIONS = ['add_agents', 'remove_agents', 'update_agent', 'publish', 'load_config'];
73
-
74
- /**
75
- * Workflow-triggering topics that indicate cluster state progression
76
- * These are the topics that MATTER for resume - not AGENT_OUTPUT noise
77
- */
78
- const WORKFLOW_TRIGGERS = Object.freeze([
79
- 'ISSUE_OPENED',
80
- 'PLAN_READY',
81
- 'IMPLEMENTATION_READY',
82
- 'VALIDATION_RESULT',
83
- 'CONDUCTOR_ESCALATE',
84
- ]);
85
-
86
- class Orchestrator {
87
- constructor(options = {}) {
88
- this.clusters = new Map(); // cluster_id -> cluster object
89
- this.quiet = options.quiet || false; // Suppress verbose logging
90
-
91
- // TaskRunner DI - allows injecting MockTaskRunner for testing
92
- // When set, passed to all AgentWrappers to control task execution
93
- this.taskRunner = options.taskRunner || null;
94
-
95
- // Set up persistent storage directory (can be overridden for testing)
96
- this.storageDir = options.storageDir || path.join(os.homedir(), '.zeroshot');
97
- if (!fs.existsSync(this.storageDir)) {
98
- fs.mkdirSync(this.storageDir, { recursive: true });
99
- }
100
-
101
- // Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
102
- this.closed = false;
103
-
104
- // Track if clusters are loaded (for lazy loading pattern)
105
- this._clustersLoaded = options.skipLoad === true;
106
- }
107
-
108
- /**
109
- * Factory method for async initialization
110
- * Use this instead of `new Orchestrator()` for proper async cluster loading
111
- * @param {Object} options - Same options as constructor
112
- * @returns {Promise<Orchestrator>}
113
- */
114
- static async create(options = {}) {
115
- const instance = new Orchestrator({ ...options, skipLoad: true });
116
- if (options.skipLoad !== true) {
117
- await instance._loadClusters();
118
- instance._clustersLoaded = true;
119
- }
120
- return instance;
121
- }
122
-
123
- /**
124
- * Log message (respects quiet mode)
125
- * @private
126
- */
127
- _log(...args) {
128
- if (!this.quiet) {
129
- console.log(...args);
130
- }
131
- }
132
-
133
- /**
134
- * Load clusters from persistent storage
135
- * Uses file locking for consistent reads
136
- * @private
137
- */
138
- async _loadClusters() {
139
- const clustersFile = path.join(this.storageDir, 'clusters.json');
140
- this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
141
-
142
- if (!fs.existsSync(clustersFile)) {
143
- this._log(`[Orchestrator] No clusters file found at ${clustersFile}`);
144
- return;
145
- }
146
-
147
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
148
- let release;
149
-
150
- try {
151
- // Clean stale locks from crashed processes
152
- cleanStaleLock(lockfilePath);
153
-
154
- // Acquire lock with async API (proper retries without CPU spin-wait)
155
- release = await lockfile.lock(clustersFile, {
156
- lockfilePath,
157
- stale: LOCK_STALE_MS,
158
- retries: {
159
- retries: 20,
160
- minTimeout: 100,
161
- maxTimeout: 200,
162
- randomize: true,
163
- },
164
- });
165
-
166
- const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
167
- const clusterIds = Object.keys(data);
168
- this._log(`[Orchestrator] Found ${clusterIds.length} clusters in file:`, clusterIds);
169
-
170
- // Track clusters to remove (missing .db files or 0 messages)
171
- const clustersToRemove = [];
172
- // Track clusters with 0 messages (corrupted from SIGINT race condition)
173
- const corruptedClusters = [];
174
-
175
- for (const [clusterId, clusterData] of Object.entries(data)) {
176
- // Skip clusters whose .db file doesn't exist (orphaned registry entries)
177
- const dbPath = path.join(this.storageDir, `${clusterId}.db`);
178
- if (!fs.existsSync(dbPath)) {
179
- console.warn(
180
- `[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
181
- );
182
- clustersToRemove.push(clusterId);
183
- continue;
184
- }
185
-
186
- this._log(`[Orchestrator] Loading cluster: ${clusterId}`);
187
- const cluster = this._loadSingleCluster(clusterId, clusterData);
188
-
189
- // VALIDATION: Detect 0-message clusters (corrupted from SIGINT during initialization)
190
- // These clusters were created before the initCompletePromise fix was applied
191
- if (cluster && cluster.messageBus) {
192
- const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
193
- if (messageCount === 0) {
194
- console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
195
- console.warn(
196
- `[Orchestrator] This likely occurred from SIGINT during initialization.`
197
- );
198
- console.warn(
199
- `[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
200
- );
201
- corruptedClusters.push(clusterId);
202
- // Mark cluster as corrupted for visibility in status/list commands
203
- cluster.state = 'corrupted';
204
- cluster.corruptedReason = 'SIGINT during initialization (0 messages in ledger)';
205
- }
206
- }
207
- }
208
-
209
- // Clean up orphaned entries from clusters.json
210
- if (clustersToRemove.length > 0) {
211
- for (const clusterId of clustersToRemove) {
212
- delete data[clusterId];
213
- }
214
- fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
215
- this._log(
216
- `[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
217
- );
218
- }
219
-
220
- // Log summary of corrupted clusters
221
- if (corruptedClusters.length > 0) {
222
- console.warn(
223
- `\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
224
- );
225
- for (const clusterId of corruptedClusters) {
226
- console.warn(` - ${clusterId}`);
227
- }
228
- console.warn(`[Orchestrator] Run 'zeroshot clear' to remove all corrupted clusters.\n`);
229
- }
230
-
231
- this._log(`[Orchestrator] Total clusters loaded: ${this.clusters.size}`);
232
- } catch (error) {
233
- console.error('[Orchestrator] Failed to load clusters:', error.message);
234
- console.error(error.stack);
235
- } finally {
236
- if (release) {
237
- await release();
238
- }
239
- }
240
- }
241
-
242
- /**
243
- * Load a single cluster from data
244
- * @private
245
- */
246
- _loadSingleCluster(clusterId, clusterData) {
247
- // Skip if already loaded
248
- if (this.clusters.has(clusterId)) {
249
- return this.clusters.get(clusterId);
250
- }
251
-
252
- // Restore ledger and message bus
253
- const dbPath = path.join(this.storageDir, `${clusterId}.db`);
254
- const ledger = new Ledger(dbPath);
255
- const messageBus = new MessageBus(ledger);
256
-
257
- // Restore isolation manager FIRST if cluster was running in isolation mode
258
- let isolation = clusterData.isolation || null;
259
- let isolationManager = null;
260
- if (isolation?.enabled && isolation.containerId) {
261
- isolationManager = new IsolationManager({ image: isolation.image });
262
- // Restore the container mapping so cleanup works
263
- isolationManager.containers.set(clusterId, isolation.containerId);
264
- // Restore isolated dir mapping for workspace preservation during cleanup
265
- if (isolation.workDir) {
266
- isolationManager.isolatedDirs.set(clusterId, {
267
- path: path.join(os.tmpdir(), 'zeroshot-isolated', clusterId),
268
- originalDir: isolation.workDir,
269
- });
270
- }
271
- isolation = {
272
- ...isolation,
273
- manager: isolationManager,
274
- };
275
- this._log(
276
- `[Orchestrator] Restored isolation manager for ${clusterId} (container: ${isolation.containerId}, workDir: ${isolation.workDir || 'unknown'})`
277
- );
278
- }
279
-
280
- // Reconstruct agent metadata from config (processes are ephemeral)
281
- // CRITICAL: Pass isolation context to agents if cluster was running in isolation
282
- const agents = [];
283
-
284
- // CRITICAL: Determine the correct cwd for agents (worktree > isolation > saved cwd)
285
- // This fixes clusters saved before the cwd injection bug was fixed
286
- const worktreePath = clusterData.worktree?.path;
287
- const isolationWorkDir = clusterData.isolation?.workDir;
288
- const agentCwd = worktreePath || isolationWorkDir || null;
289
-
290
- if (clusterData.config?.agents) {
291
- for (const agentConfig of clusterData.config.agents) {
292
- // Fix agents that were saved without cwd (pre-bugfix clusters)
293
- if (!agentConfig.cwd && agentCwd) {
294
- agentConfig.cwd = agentCwd;
295
- this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
296
- }
297
-
298
- if (clusterData.modelOverride) {
299
- applyModelOverride(agentConfig, clusterData.modelOverride);
300
- }
301
-
302
- const agentOptions = {
303
- id: clusterId,
304
- quiet: this.quiet,
305
- modelOverride: clusterData.modelOverride || null,
306
- };
307
-
308
- // Inject isolation context if enabled (MUST be done during agent creation)
309
- if (isolation?.enabled && isolationManager) {
310
- agentOptions.isolation = {
311
- enabled: true,
312
- manager: isolationManager,
313
- clusterId,
314
- };
315
- }
316
-
317
- // Create agent or subcluster wrapper based on type
318
- let agent;
319
- if (agentConfig.type === 'subcluster') {
320
- agent = new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
321
- } else {
322
- agent = new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
323
- }
324
-
325
- // Restore agent runtime state if available (for accurate status display)
326
- if (clusterData.agentStates) {
327
- const savedState = clusterData.agentStates.find((s) => s.id === agentConfig.id);
328
- if (savedState) {
329
- agent.state = savedState.state || 'idle';
330
- agent.iteration = savedState.iteration || 0;
331
- agent.currentTask = savedState.currentTask || false;
332
- agent.currentTaskId = savedState.currentTaskId || null;
333
- agent.processPid = savedState.processPid || null;
334
- }
335
- }
336
-
337
- agents.push(agent);
338
- }
339
- }
340
-
341
- const cluster = {
342
- ...clusterData,
343
- ledger,
344
- messageBus,
345
- agents,
346
- isolation,
347
- autoPr: clusterData.autoPr || false,
348
- };
349
-
350
- this.clusters.set(clusterId, cluster);
351
- this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
352
-
353
- return cluster;
354
- }
355
-
356
- /**
357
- * Ensure clusters file exists (required for file locking)
358
- * @private
359
- */
360
- _ensureClustersFile() {
361
- const clustersFile = path.join(this.storageDir, 'clusters.json');
362
- if (!fs.existsSync(clustersFile)) {
363
- fs.writeFileSync(clustersFile, '{}');
364
- }
365
- return clustersFile;
366
- }
367
-
368
- /**
369
- * Save clusters to persistent storage
370
- * Uses file locking to prevent race conditions with other processes
371
- * @private
372
- */
373
- async _saveClusters() {
374
- // Skip saving if orchestrator is closed (prevents race conditions during cleanup)
375
- if (this.closed) {
376
- return;
377
- }
378
-
379
- const clustersFile = this._ensureClustersFile();
380
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
381
- let release;
382
-
383
- try {
384
- // Clean stale locks from crashed processes
385
- cleanStaleLock(lockfilePath);
386
-
387
- // Acquire exclusive lock with async API (proper retries without CPU spin-wait)
388
- release = await lockfile.lock(clustersFile, {
389
- lockfilePath,
390
- stale: LOCK_STALE_MS,
391
- retries: {
392
- retries: 50,
393
- minTimeout: 100,
394
- maxTimeout: 300,
395
- randomize: true,
396
- },
397
- });
398
-
399
- // Read existing clusters from file (other processes may have added clusters)
400
- let existingClusters = {};
401
- try {
402
- const content = fs.readFileSync(clustersFile, 'utf8');
403
- existingClusters = JSON.parse(content);
404
- } catch (error) {
405
- console.error('[Orchestrator] Failed to read existing clusters:', error.message);
406
- }
407
-
408
- // Merge: update/add clusters from this process
409
- for (const [clusterId, cluster] of this.clusters.entries()) {
410
- // CRITICAL: Only update clusters this process actually owns or has modified
411
- // A process owns a cluster if: it started it (pid matches) OR it explicitly stopped/killed it
412
- const isOwnedByThisProcess = cluster.pid === process.pid;
413
- const wasModifiedByThisProcess = cluster.state === 'stopped' || cluster.state === 'killed';
414
-
415
- // Skip clusters we don't own and haven't modified - prevents race condition
416
- // where a running cluster overwrites another process's stop/kill operation
417
- if (!isOwnedByThisProcess && !wasModifiedByThisProcess) {
418
- // Preserve existing state from file for clusters we don't own
419
- continue;
420
- }
421
-
422
- // CRITICAL: Killed clusters are DELETED from disk, not persisted
423
- // This ensures they can't be accidentally resumed
424
- if (cluster.state === 'killed') {
425
- delete existingClusters[clusterId];
426
- continue;
427
- }
428
-
429
- existingClusters[clusterId] = {
430
- id: cluster.id,
431
- config: cluster.config,
432
- state: cluster.state,
433
- createdAt: cluster.createdAt,
434
- // Track PID for zombie detection (null if cluster is stopped/killed)
435
- pid: cluster.state === 'running' ? cluster.pid : null,
436
- // Persist failure info for resume capability
437
- failureInfo: cluster.failureInfo || null,
438
- // Persist PR mode for completion agent selection
439
- autoPr: cluster.autoPr || false,
440
- // Persist model override for consistent agent spawning on resume
441
- modelOverride: cluster.modelOverride || null,
442
- // Persist isolation info (excluding manager instance which can't be serialized)
443
- // CRITICAL: workDir is required for resume() to recreate container with same workspace
444
- isolation: cluster.isolation
445
- ? {
446
- enabled: cluster.isolation.enabled,
447
- containerId: cluster.isolation.containerId,
448
- image: cluster.isolation.image,
449
- workDir: cluster.isolation.workDir, // Required for resume
450
- }
451
- : null,
452
- // Persist agent runtime states for accurate status display from other processes
453
- agentStates: cluster.agents
454
- ? cluster.agents.map((a) => ({
455
- id: a.id,
456
- state: a.state,
457
- iteration: a.iteration,
458
- currentTask: a.currentTask ? true : false,
459
- currentTaskId: a.currentTaskId,
460
- processPid: a.processPid,
461
- }))
462
- : null,
463
- };
464
- }
465
-
466
- // Write merged data
467
- fs.writeFileSync(clustersFile, JSON.stringify(existingClusters, null, 2));
468
- this._log(
469
- `[Orchestrator] Saved ${this.clusters.size} cluster(s), file now has ${Object.keys(existingClusters).length} total`
470
- );
471
- } finally {
472
- // Always release lock
473
- if (release) {
474
- await release();
475
- }
476
- }
477
- }
478
-
479
- /**
480
- * Watch for new clusters and call callback when found
481
- * Polls the clusters file for changes with file locking
482
- * @param {Function} onNewCluster - Callback(cluster) for each new cluster
483
- * @param {Number} intervalMs - Poll interval in ms (default: 2000)
484
- * @returns {Function} Stop function to cancel watching
485
- */
486
- watchForNewClusters(onNewCluster, intervalMs = 2000) {
487
- const clustersFile = path.join(this.storageDir, 'clusters.json');
488
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
489
- const knownClusterIds = new Set(this.clusters.keys());
490
-
491
- const intervalId = setInterval(() => {
492
- let release;
493
- try {
494
- if (!fs.existsSync(clustersFile)) return;
495
-
496
- // Clean stale locks from crashed processes
497
- cleanStaleLock(lockfilePath);
498
-
499
- // Try to acquire lock once (polling is best-effort, will retry on next cycle)
500
- try {
501
- release = lockfile.lockSync(clustersFile, {
502
- lockfilePath,
503
- stale: LOCK_STALE_MS,
504
- });
505
- } catch (lockErr) {
506
- // Lock busy - skip this poll cycle, try again next interval
507
- if (lockErr.code === 'ELOCKED') return;
508
- throw lockErr;
509
- }
510
-
511
- const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
512
-
513
- for (const [clusterId, clusterData] of Object.entries(data)) {
514
- if (!knownClusterIds.has(clusterId)) {
515
- // New cluster found
516
- knownClusterIds.add(clusterId);
517
- const cluster = this._loadSingleCluster(clusterId, clusterData);
518
- if (cluster && onNewCluster) {
519
- onNewCluster(cluster);
520
- }
521
- }
522
- }
523
- } catch (error) {
524
- // File access during polling can fail transiently - log and continue
525
- console.error(`[Orchestrator] watchForNewClusters error (will retry): ${error.message}`);
526
- } finally {
527
- if (release) {
528
- release();
529
- }
530
- }
531
- }, intervalMs);
532
-
533
- // Return stop function
534
- return () => clearInterval(intervalId);
535
- }
536
-
537
- /**
538
- * Start a new cluster with mocked agent executors (TESTING ONLY)
539
- *
540
- * CRITICAL: This method PREVENTS real Claude API calls.
541
- * All agent behaviors must be defined in mockExecutor.
542
- *
543
- * @param {Object} config - Cluster configuration
544
- * @param {Object} input - Input source { issue, text, or config }
545
- * @param {MockAgentExecutor} mockExecutor - Mock executor with agent behaviors
546
- * @returns {Object} Cluster object
547
- */
548
- startWithMock(config, input, mockExecutor) {
549
- if (!mockExecutor) {
550
- throw new Error('Orchestrator.startWithMock: mockExecutor is required');
551
- }
552
-
553
- // Validate all agents that execute tasks have mock behaviors defined
554
- // Orchestrator agents (action: 'stop_cluster') don't execute tasks, so don't need mocks
555
- for (const agentConfig of config.agents) {
556
- const agentId = agentConfig.id;
557
-
558
- // Check if agent has any triggers that execute tasks
559
- const executesTask = agentConfig.triggers?.some(
560
- (trigger) => !trigger.action || trigger.action === 'execute_task'
561
- );
562
-
563
- if (executesTask && !mockExecutor.behaviors[agentId]) {
564
- throw new Error(
565
- `Orchestrator.startWithMock: No behavior defined for agent '${agentId}'. ` +
566
- `This would cause real Claude API calls. ABORTING.\n` +
567
- `Available behaviors: ${Object.keys(mockExecutor.behaviors).join(', ')}`
568
- );
569
- }
570
- }
571
-
572
- return this._startInternal(config, input, {
573
- mockExecutor,
574
- testMode: true,
575
- });
576
- }
577
-
578
- /**
579
- * Start a new cluster
580
- * @param {Object} config - Cluster configuration
581
- * @param {Object} input - Input source { issue, text, or config }
582
- * @param {Object} options - Start options
583
- * @param {boolean} options.isolation - Run in Docker container
584
- * @param {string} options.isolationImage - Docker image to use
585
- * @param {boolean} options.worktree - Run in git worktree isolation (lightweight, no Docker)
586
- * @returns {Object} Cluster object
587
- */
588
- start(config, input = {}, options = {}) {
589
- return this._startInternal(config, input, {
590
- testMode: false,
591
- cwd: options.cwd || process.cwd(), // Target working directory for agents
592
- isolation: options.isolation || false,
593
- isolationImage: options.isolationImage,
594
- worktree: options.worktree || false,
595
- autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
596
- modelOverride: options.modelOverride, // Model override for all agents
597
- clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
598
- });
599
- }
600
-
601
- /**
602
- * Internal start implementation (shared by start and startWithMock)
603
- * @private
604
- */
605
- async _startInternal(config, input = {}, options = {}) {
606
- // Generate a unique cluster ID for this process call.
607
- // IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
608
- // - test harnesses may set it globally (breaking multi-start tests)
609
- // - callers may start multiple clusters in one process
610
- // Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
611
- const clusterId = this._generateUniqueClusterId(
612
- options.clusterId || null,
613
- config?.dbPath || null
614
- );
615
-
616
- // Create ledger and message bus with persistent storage
617
- const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
618
- const ledger = new Ledger(dbPath);
619
- const messageBus = new MessageBus(ledger);
620
-
621
- // Handle isolation mode (Docker container OR git worktree)
622
- let isolationManager = null;
623
- let containerId = null;
624
- let worktreeInfo = null;
625
-
626
- if (options.isolation) {
627
- // Check Docker availability
628
- if (!IsolationManager.isDockerAvailable()) {
629
- throw new Error('Docker is not available. Install Docker to use --docker mode.');
630
- }
631
-
632
- // Ensure image exists (auto-build if missing)
633
- const image = options.isolationImage || 'zeroshot-cluster-base';
634
- await IsolationManager.ensureImage(image);
635
-
636
- isolationManager = new IsolationManager({ image });
637
- this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
638
-
639
- // Create container with workspace mounted
640
- // CRITICAL: Use options.cwd (git repo root) instead of process.cwd()
641
- const workDir = options.cwd || process.cwd();
642
- const providerName = normalizeProviderName(
643
- config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
644
- );
645
- containerId = await isolationManager.createContainer(clusterId, {
646
- workDir,
647
- image,
648
- // Mount configuration (CLI overrides)
649
- noMounts: options.noMounts,
650
- mounts: options.mounts,
651
- containerHome: options.containerHome,
652
- provider: providerName,
653
- });
654
- this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
655
- } else if (options.worktree) {
656
- // Worktree isolation: lightweight git-based isolation (no Docker required)
657
- const workDir = options.cwd || process.cwd();
658
-
659
- isolationManager = new IsolationManager({});
660
- worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
661
-
662
- this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
663
- this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
664
- this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
665
- }
666
-
667
- // Build cluster object
668
- // CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
669
- // This prevents 0-message clusters from SIGINT during async initialization
670
- let resolveInitComplete;
671
- const initCompletePromise = new Promise((resolve) => {
672
- resolveInitComplete = resolve;
673
- });
674
-
675
- const cluster = {
676
- id: clusterId,
677
- config,
678
- state: 'initializing',
679
- messageBus,
680
- ledger,
681
- agents: [],
682
- createdAt: Date.now(),
683
- // Track PID for zombie detection (this process owns the cluster)
684
- pid: process.pid,
685
- // Initialization completion tracking (for safe SIGINT handling)
686
- initCompletePromise,
687
- _resolveInitComplete: resolveInitComplete,
688
- autoPr: options.autoPr || false,
689
- // Model override for all agents (applied to dynamically added agents)
690
- modelOverride: options.modelOverride || null,
691
- // Isolation state (only if enabled)
692
- // CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
693
- isolation: options.isolation
694
- ? {
695
- enabled: true,
696
- containerId,
697
- image: options.isolationImage || 'zeroshot-cluster-base',
698
- manager: isolationManager,
699
- workDir: options.cwd || process.cwd(), // Persisted for resume
700
- }
701
- : null,
702
- // Worktree isolation state (lightweight alternative to Docker)
703
- worktree: options.worktree
704
- ? {
705
- enabled: true,
706
- path: worktreeInfo.path,
707
- branch: worktreeInfo.branch,
708
- repoRoot: worktreeInfo.repoRoot,
709
- manager: isolationManager,
710
- workDir: options.cwd || process.cwd(), // Persisted for resume
711
- }
712
- : null,
713
- };
714
-
715
- this.clusters.set(clusterId, cluster);
716
-
717
- try {
718
- // Fetch input (GitHub issue, file, or text)
719
- let inputData;
720
- if (input.issue) {
721
- inputData = await GitHub.fetchIssue(input.issue);
722
- // Log clickable issue link
723
- if (inputData.url) {
724
- this._log(`[Orchestrator] Issue: ${inputData.url}`);
725
- }
726
- } else if (input.file) {
727
- inputData = GitHub.createFileInput(input.file);
728
- this._log(`[Orchestrator] File: ${input.file}`);
729
- } else if (input.text) {
730
- inputData = GitHub.createTextInput(input.text);
731
- } else {
732
- throw new Error('Either issue, file, or text input is required');
733
- }
734
-
735
- // Inject git-pusher agent if --pr is set (replaces completion-detector)
736
- if (options.autoPr) {
737
- // Remove completion-detector by ID (git-pusher handles completion + PR)
738
- config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
739
-
740
- // Load and configure git-pusher agent (use fs.readFileSync to avoid require cache)
741
- const gitPusherPath = path.join(__dirname, 'agents', 'git-pusher-agent.json');
742
- const gitPusherConfig = JSON.parse(fs.readFileSync(gitPusherPath, 'utf8'));
743
-
744
- // Inject issue context placeholders
745
- gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
746
- /\{\{issue_number\}\}/g,
747
- inputData.number || 'unknown'
748
- );
749
- gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
750
- /\{\{issue_title\}\}/g,
751
- inputData.title || 'Implementation'
752
- );
753
-
754
- config.agents.push(gitPusherConfig);
755
- this._log(`[Orchestrator] Injected git-pusher agent (creates PR and auto-merges)`);
756
- }
757
-
758
- // Inject workers instruction if --workers explicitly provided and > 1
759
- const workersCount = process.env.ZEROSHOT_WORKERS
760
- ? parseInt(process.env.ZEROSHOT_WORKERS)
761
- : 0;
762
- if (workersCount > 1) {
763
- const workerAgent = config.agents.find((a) => a.id === 'worker');
764
- if (workerAgent) {
765
- const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
766
-
767
- if (!workerAgent.prompt) {
768
- workerAgent.prompt = instruction;
769
- } else if (typeof workerAgent.prompt === 'string') {
770
- workerAgent.prompt = instruction + workerAgent.prompt;
771
- } else if (workerAgent.prompt.system) {
772
- workerAgent.prompt.system = instruction + workerAgent.prompt.system;
773
- }
774
- this._log(
775
- `[Orchestrator] Injected parallelization instruction (workers=${workersCount})`
776
- );
777
- }
778
- }
779
-
780
- // Initialize agents with optional mock injection
781
- // Check agent type: regular agent or subcluster
782
- // CRITICAL: Inject cwd into each agent config for proper working directory
783
- // In worktree mode, agents run in the worktree path (not original cwd)
784
- const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
785
- for (const agentConfig of config.agents) {
786
- // Inject cwd if not already set (config may override)
787
- if (!agentConfig.cwd) {
788
- agentConfig.cwd = agentCwd;
789
- }
790
-
791
- // Apply model override if set (for consistency across all agents)
792
- if (options.modelOverride) {
793
- applyModelOverride(agentConfig, options.modelOverride);
794
- this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
795
- }
796
-
797
- const agentOptions = {
798
- testMode: options.testMode || !!this.taskRunner, // Enable testMode if taskRunner provided
799
- quiet: this.quiet,
800
- modelOverride: options.modelOverride || null,
801
- };
802
-
803
- // Inject mock spawn function if provided (legacy mockExecutor API)
804
- if (options.mockExecutor) {
805
- agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
806
- }
807
-
808
- // TaskRunner DI - new pattern for mocking task execution
809
- // Creates a mockSpawnFn wrapper that delegates to the taskRunner
810
- if (this.taskRunner) {
811
- // CRITICAL: agent is a closure variable capturing the AgentWrapper instance
812
- // We cannot access agent._selectModel() here because agent doesn't exist yet
813
- // Solution: Pass a factory function that will be called when agent is available
814
- agentOptions.taskRunner = this.taskRunner;
815
- }
816
-
817
- // Pass isolation context if enabled
818
- if (cluster.isolation) {
819
- agentOptions.isolation = {
820
- enabled: true,
821
- manager: isolationManager,
822
- clusterId,
823
- };
824
- }
825
-
826
- // Pass worktree context if enabled (lightweight isolation without Docker)
827
- if (cluster.worktree) {
828
- agentOptions.worktree = {
829
- enabled: true,
830
- path: cluster.worktree.path,
831
- branch: cluster.worktree.branch,
832
- repoRoot: cluster.worktree.repoRoot,
833
- };
834
- }
835
-
836
- // Create agent or subcluster wrapper based on type
837
- let agent;
838
- if (agentConfig.type === 'subcluster') {
839
- agent = new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions);
840
- } else {
841
- agent = new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
842
- }
843
-
844
- cluster.agents.push(agent);
845
- }
846
-
847
- // ========================================================================
848
- // CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
849
- // ========================================================================
850
- // MUST register ALL subscriptions BEFORE starting agents.
851
- //
852
- // WHY: EventEmitter is synchronous and doesn't replay past events to new
853
- // listeners. If an agent completes and publishes CLUSTER_COMPLETE before
854
- // we register the subscription, the orchestrator will never receive the
855
- // message and the cluster will appear stuck in 'executing_task' forever.
856
- //
857
- // ORDER:
858
- // 1. Register subscriptions (lines below)
859
- // 2. Start agents
860
- // 3. Publish ISSUE_OPENED
861
- //
862
- // DO NOT move subscriptions after agent.start() - this will reintroduce
863
- // the race condition fixed in issue #31.
864
- // ========================================================================
865
-
866
- // Helper: Subscribe to cluster-scoped topic
867
- const subscribeToClusterTopic = (topic, handler) => {
868
- messageBus.subscribe((message) => {
869
- if (message.topic === topic && message.cluster_id === clusterId) {
870
- handler(message);
871
- }
872
- });
873
- };
874
-
875
- // Watch for CLUSTER_COMPLETE message to auto-stop
876
- subscribeToClusterTopic('CLUSTER_COMPLETE', (message) => {
877
- this._log(`\n${'='.repeat(80)}`);
878
- this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
879
- this._log(`${'='.repeat(80)}`);
880
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
881
- this._log(`Initiated by: ${message.sender}`);
882
- this._log(`${'='.repeat(80)}\n`);
883
-
884
- // Auto-stop cluster
885
- this.stop(clusterId).catch((err) => {
886
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
887
- });
888
- });
889
-
890
- // Watch for CLUSTER_FAILED message to auto-stop
891
- subscribeToClusterTopic('CLUSTER_FAILED', (message) => {
892
- this._log(`\n${'='.repeat(80)}`);
893
- this._log(`❌ CLUSTER FAILED: ${clusterId}`);
894
- this._log(`${'='.repeat(80)}`);
895
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
896
- this._log(`Agent: ${message.sender}`);
897
- if (message.content?.text) {
898
- this._log(`Details: ${message.content.text}`);
899
- }
900
- this._log(`${'='.repeat(80)}\n`);
901
-
902
- // Auto-stop cluster
903
- this.stop(clusterId).catch((err) => {
904
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
905
- });
906
- });
907
-
908
- // Watch for AGENT_ERROR - if critical agent fails, stop cluster
909
- subscribeToClusterTopic('AGENT_ERROR', async (message) => {
910
- const agentRole = message.content?.data?.role;
911
- const attempts = message.content?.data?.attempts || 1;
912
-
913
- // Save cluster state to persist failureInfo
914
- await await this._saveClusters();
915
-
916
- // Only stop cluster if non-validator agent exhausted retries
917
- if (agentRole === 'implementation' && attempts >= 3) {
918
- this._log(`\n${'='.repeat(80)}`);
919
- this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
920
- this._log(`${'='.repeat(80)}`);
921
- this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
922
- this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
923
- this._log(`Stopping cluster - worker cannot continue`);
924
- this._log(`${'='.repeat(80)}\n`);
925
-
926
- // Auto-stop cluster
927
- this.stop(clusterId).catch((err) => {
928
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
929
- });
930
- }
931
- });
932
-
933
- // Persist agent state changes for accurate status display
934
- messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
935
- const event = message.content?.data?.event;
936
- // Save on key state transitions that affect status display
937
- if (
938
- [
939
- 'TASK_STARTED',
940
- 'TASK_COMPLETED',
941
- 'PROCESS_SPAWNED',
942
- 'TASK_ID_ASSIGNED',
943
- 'STARTED',
944
- ].includes(event)
945
- ) {
946
- await await this._saveClusters();
947
- }
948
- });
949
-
950
- // Watch for stale agent detection (informational only)
951
- messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
952
- if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
953
-
954
- const agentId = message.content?.data?.agent;
955
- const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
956
- const analysis = message.content?.data?.analysis || 'No analysis available';
957
-
958
- this._log(
959
- `⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
960
- );
961
- this._log(` Analysis: ${analysis}`);
962
- this._log(
963
- ` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
964
- );
965
- });
966
-
967
- // CONDUCTOR WATCHDOG: If conductor completes but CLUSTER_OPERATIONS never arrives, FAIL FAST
968
- // This catches the silent failure where conductor outputs result but hook fails to publish
969
- const CONDUCTOR_WATCHDOG_TIMEOUT_MS = 30000; // 30 seconds
970
- let conductorWatchdogTimer = null;
971
- let conductorCompletedAt = null;
972
-
973
- // Start watchdog when conductor completes
974
- subscribeToClusterTopic('AGENT_LIFECYCLE', (message) => {
975
- const event = message.content?.data?.event;
976
- const role = message.content?.data?.role;
977
-
978
- // Conductor completed - start watchdog
979
- if (event === 'TASK_COMPLETED' && role === 'conductor') {
980
- conductorCompletedAt = Date.now();
981
- this._log(
982
- `⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${CONDUCTOR_WATCHDOG_TIMEOUT_MS / 1000}s`
983
- );
984
-
985
- conductorWatchdogTimer = setTimeout(() => {
986
- // Check if CLUSTER_OPERATIONS was received
987
- const clusterOps = messageBus.query({ topic: 'CLUSTER_OPERATIONS', limit: 1 });
988
- if (clusterOps.length === 0) {
989
- console.error(`\n${'='.repeat(80)}`);
990
- console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
991
- console.error(`${'='.repeat(80)}`);
992
- console.error(
993
- `Conductor completed ${CONDUCTOR_WATCHDOG_TIMEOUT_MS / 1000}s ago but no CLUSTER_OPERATIONS`
994
- );
995
- console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
996
- console.error(
997
- `Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
998
- );
999
- console.error(`${'='.repeat(80)}\n`);
1000
-
1001
- // Publish CLUSTER_FAILED to stop the cluster
1002
- messageBus.publish({
1003
- cluster_id: clusterId,
1004
- topic: 'CLUSTER_FAILED',
1005
- sender: 'orchestrator',
1006
- content: {
1007
- text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
1008
- data: {
1009
- reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
1010
- conductorCompletedAt,
1011
- timeoutMs: CONDUCTOR_WATCHDOG_TIMEOUT_MS,
1012
- },
1013
- },
1014
- });
1015
- }
1016
- }, CONDUCTOR_WATCHDOG_TIMEOUT_MS);
1017
- }
1018
- });
1019
-
1020
- // Watch for CLUSTER_OPERATIONS - dynamic agent spawn/removal/update
1021
- subscribeToClusterTopic('CLUSTER_OPERATIONS', (message) => {
1022
- // Clear conductor watchdog - CLUSTER_OPERATIONS received successfully
1023
- if (conductorWatchdogTimer) {
1024
- clearTimeout(conductorWatchdogTimer);
1025
- conductorWatchdogTimer = null;
1026
- const elapsed = conductorCompletedAt ? Date.now() - conductorCompletedAt : 0;
1027
- this._log(
1028
- `✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
1029
- );
1030
- }
1031
- let operations = message.content?.data?.operations;
1032
-
1033
- // Parse operations if they came as a JSON string
1034
- if (typeof operations === 'string') {
1035
- try {
1036
- operations = JSON.parse(operations);
1037
- } catch (e) {
1038
- this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
1039
- return;
1040
- }
1041
- }
1042
-
1043
- if (!operations || !Array.isArray(operations)) {
1044
- this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
1045
- return;
1046
- }
1047
-
1048
- this._log(`\n${'='.repeat(80)}`);
1049
- this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
1050
- this._log(`${'='.repeat(80)}`);
1051
- if (message.content?.data?.reasoning) {
1052
- this._log(`Reasoning: ${message.content.data.reasoning}`);
1053
- }
1054
- this._log(`Operations: ${operations.length}`);
1055
- this._log(`${'='.repeat(80)}\n`);
1056
-
1057
- // Execute operation chain
1058
- this._handleOperations(clusterId, operations, message.sender, {
1059
- isolationManager,
1060
- containerId,
1061
- }).catch((err) => {
1062
- console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
1063
- // Publish failure message
1064
- messageBus.publish({
1065
- cluster_id: clusterId,
1066
- topic: 'CLUSTER_OPERATIONS_FAILED',
1067
- sender: 'orchestrator',
1068
- content: {
1069
- text: `Operation chain failed: ${err.message}`,
1070
- data: {
1071
- error: err.message,
1072
- operations: operations,
1073
- },
1074
- },
1075
- });
1076
-
1077
- // CRITICAL: Stop cluster on operation failure
1078
- this._log(`\n${'='.repeat(80)}`);
1079
- this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
1080
- this._log(`${'='.repeat(80)}`);
1081
- this._log(`Error: ${err.message}`);
1082
- this._log(`${'='.repeat(80)}\n`);
1083
-
1084
- this.stop(clusterId).catch((stopErr) => {
1085
- console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
1086
- });
1087
- });
1088
- });
1089
-
1090
- // Start all agents
1091
- for (const agent of cluster.agents) {
1092
- await agent.start();
1093
- }
1094
-
1095
- cluster.state = 'running';
1096
-
1097
- // Publish ISSUE_OPENED message to bootstrap workflow
1098
- messageBus.publish({
1099
- cluster_id: clusterId,
1100
- topic: 'ISSUE_OPENED',
1101
- sender: 'system',
1102
- receiver: 'broadcast',
1103
- content: {
1104
- text: inputData.context,
1105
- data: {
1106
- issue_number: inputData.number,
1107
- title: inputData.title,
1108
- },
1109
- },
1110
- metadata: {
1111
- source: input.issue ? 'github' : input.file ? 'file' : 'text',
1112
- },
1113
- });
1114
-
1115
- // CRITICAL: Mark initialization complete AFTER ISSUE_OPENED is published
1116
- // This ensures stop() waits for at least 1 message before stopping
1117
- if (cluster._resolveInitComplete) {
1118
- cluster._resolveInitComplete();
1119
- }
1120
-
1121
- this._log(`Cluster ${clusterId} started with ${cluster.agents.length} agents`);
1122
-
1123
- // DISABLED: Idle timeout auto-stop mechanism
1124
- // WHY DISABLED: Clusters should only stop on explicit signals:
1125
- // - User `zeroshot kill` command
1126
- // - CLUSTER_COMPLETE message (successful completion)
1127
- // - CLUSTER_FAILED message (failure/abort)
1128
- // Being "idle" is NOT a reason to auto-stop - agents may be legitimately
1129
- // waiting for external events, user input (in interactive mode), or
1130
- // processing that doesn't show as "executing" (e.g., polling, monitoring).
1131
- //
1132
- // Previous behavior: Stopped cluster after 2 minutes of all agents idle
1133
- // Result: Clusters were killed while legitimately waiting, causing confusion
1134
- //
1135
- // cluster.idleCheckInterval = setInterval(() => { ... }, 30000);
1136
- // ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
1137
-
1138
- // Save cluster to disk
1139
- await this._saveClusters();
1140
-
1141
- return {
1142
- id: clusterId,
1143
- state: cluster.state,
1144
- agents: cluster.agents.map((a) => a.getState()),
1145
- ledger: cluster.ledger, // Expose ledger for testing
1146
- messageBus: cluster.messageBus, // Expose messageBus for testing
1147
- };
1148
- } catch (error) {
1149
- cluster.state = 'failed';
1150
- // CRITICAL: Resolve the promise on failure too, so stop() doesn't hang
1151
- if (cluster._resolveInitComplete) {
1152
- cluster._resolveInitComplete();
1153
- }
1154
- console.error(`Cluster ${clusterId} failed to start:`, error);
1155
- throw error;
1156
- }
1157
- }
1158
-
1159
- /**
1160
- * Generate a unique cluster ID, safe for concurrent starts in-process.
1161
- * If an explicit ID is provided, uses it as a base and suffixes on collision.
1162
- * @private
1163
- */
1164
- _generateUniqueClusterId(explicitId, explicitDbPath) {
1165
- const baseId = explicitId || generateName('cluster');
1166
- const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
1167
-
1168
- // Fast path: base is unused.
1169
- if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
1170
- return baseId;
1171
- }
1172
-
1173
- // Collision: suffix with random bytes to avoid race conditions under concurrency.
1174
- for (let attempt = 0; attempt < 50; attempt++) {
1175
- const suffix = crypto.randomBytes(3).toString('hex');
1176
- const candidateId = `${baseId}-${suffix}`;
1177
- const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1178
- if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1179
- return candidateId;
1180
- }
1181
- }
1182
-
1183
- // Last resort: new generated name (should never happen).
1184
- for (let attempt = 0; attempt < 50; attempt++) {
1185
- const candidateId = generateName('cluster');
1186
- const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1187
- if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1188
- return candidateId;
1189
- }
1190
- }
1191
-
1192
- throw new Error('Failed to generate unique cluster ID after many attempts');
1193
- }
1194
-
1195
- /**
1196
- * Stop a cluster
1197
- * @param {String} clusterId - Cluster ID
1198
- */
1199
- async stop(clusterId) {
1200
- const cluster = this.clusters.get(clusterId);
1201
- if (!cluster) {
1202
- throw new Error(`Cluster ${clusterId} not found`);
1203
- }
1204
-
1205
- // CRITICAL: Wait for initialization to complete before stopping
1206
- // This ensures ISSUE_OPENED is published, preventing 0-message clusters
1207
- // Timeout after 30s to prevent infinite hang if init truly fails
1208
- if (cluster.initCompletePromise && cluster.state === 'initializing') {
1209
- this._log(`[Orchestrator] Waiting for initialization to complete before stopping...`);
1210
- await Promise.race([
1211
- cluster.initCompletePromise,
1212
- new Promise((resolve) => setTimeout(resolve, 30000)),
1213
- ]);
1214
- }
1215
-
1216
- cluster.state = 'stopping';
1217
-
1218
- // Stop all agents (including subclusters which handle their own children)
1219
- for (const agent of cluster.agents) {
1220
- await agent.stop();
1221
- }
1222
-
1223
- // Clean up isolation container if enabled
1224
- // CRITICAL: Preserve workspace for resume capability - only delete on kill()
1225
- if (cluster.isolation?.manager) {
1226
- this._log(
1227
- `[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
1228
- );
1229
- await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
1230
- this._log(`[Orchestrator] Container stopped, workspace preserved`);
1231
- }
1232
-
1233
- // Worktree cleanup on stop: preserve for resume capability
1234
- // Branch stays, worktree stays - can resume work later
1235
- if (cluster.worktree?.manager) {
1236
- this._log(`[Orchestrator] Worktree preserved at ${cluster.worktree.path} for resume`);
1237
- this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1238
- // Don't cleanup worktree - it will be reused on resume
1239
- }
1240
-
1241
- cluster.state = 'stopped';
1242
- cluster.pid = null; // Clear PID - cluster is no longer running
1243
- this._log(`Cluster ${clusterId} stopped`);
1244
-
1245
- // Save updated state
1246
- await this._saveClusters();
1247
- }
1248
-
1249
- /**
1250
- * Kill a cluster (force stop)
1251
- * @param {String} clusterId - Cluster ID
1252
- */
1253
- async kill(clusterId) {
1254
- const cluster = this.clusters.get(clusterId);
1255
- if (!cluster) {
1256
- throw new Error(`Cluster ${clusterId} not found`);
1257
- }
1258
-
1259
- cluster.state = 'stopping';
1260
-
1261
- // Force stop all agents
1262
- for (const agent of cluster.agents) {
1263
- await agent.stop();
1264
- }
1265
-
1266
- // Force remove isolation container AND workspace (full cleanup, no resume)
1267
- if (cluster.isolation?.manager) {
1268
- this._log(
1269
- `[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
1270
- );
1271
- await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
1272
- this._log(`[Orchestrator] Container and workspace removed`);
1273
- }
1274
-
1275
- // Force remove worktree (full cleanup, no resume)
1276
- // Note: Branch is preserved for potential PR creation / inspection
1277
- if (cluster.worktree?.manager) {
1278
- this._log(`[Orchestrator] Force removing worktree for ${clusterId}...`);
1279
- cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
1280
- this._log(`[Orchestrator] Worktree removed, branch ${cluster.worktree.branch} preserved`);
1281
- }
1282
-
1283
- // Close message bus and ledger
1284
- cluster.messageBus.close();
1285
-
1286
- cluster.state = 'killed';
1287
- cluster.pid = null; // Clear PID - cluster is no longer running
1288
- // DON'T delete from memory - keep it so it gets saved with 'killed' state
1289
- // this.clusters.delete(clusterId);
1290
-
1291
- this._log(`Cluster ${clusterId} killed`);
1292
-
1293
- // Save updated state (will be marked as 'killed' in file)
1294
- await this._saveClusters();
1295
-
1296
- // Now remove from memory after persisting
1297
- this.clusters.delete(clusterId);
1298
- }
1299
-
1300
- /**
1301
- * Kill all running clusters
1302
- * @returns {Object} { killed: Array<string>, errors: Array<{id, error}> }
1303
- */
1304
- async killAll() {
1305
- const results = { killed: [], errors: [] };
1306
- const runningClusters = Array.from(this.clusters.values()).filter(
1307
- (c) => c.state === 'running' || c.state === 'initializing'
1308
- );
1309
-
1310
- for (const cluster of runningClusters) {
1311
- try {
1312
- await this.kill(cluster.id);
1313
- results.killed.push(cluster.id);
1314
- } catch (error) {
1315
- results.errors.push({ id: cluster.id, error: error.message });
1316
- }
1317
- }
1318
-
1319
- return results;
1320
- }
1321
-
1322
- /**
1323
- * Close the orchestrator (prevents further _saveClusters operations)
1324
- * Call before deleting storageDir to prevent ENOENT race conditions during cleanup
1325
- */
1326
- close() {
1327
- this.closed = true;
1328
- }
1329
-
1330
- /**
1331
- * Find the last workflow-triggering message in the ledger
1332
- * Workflow triggers indicate cluster state progression (not AGENT_OUTPUT noise)
1333
- * @param {Array} messages - All messages from ledger
1334
- * @returns {Object|null} - Last workflow trigger message or null
1335
- * @private
1336
- */
1337
- _findLastWorkflowTrigger(messages) {
1338
- for (let i = messages.length - 1; i >= 0; i--) {
1339
- if (WORKFLOW_TRIGGERS.includes(messages[i].topic)) {
1340
- return messages[i];
1341
- }
1342
- }
1343
- return null;
1344
- }
1345
-
1346
- /**
1347
- * Resume a stopped cluster from where it left off
1348
- * Handles both failed clusters (with error context) and cleanly stopped clusters
1349
- * @param {String} clusterId - Cluster ID
1350
- * @param {String} prompt - Optional custom resume prompt
1351
- * @returns {Object} Resumed cluster info
1352
- */
1353
- async resume(clusterId, prompt) {
1354
- const cluster = this.clusters.get(clusterId);
1355
- if (!cluster) {
1356
- throw new Error(`Cluster not found: ${clusterId}`);
1357
- }
1358
-
1359
- if (cluster.state === 'running') {
1360
- throw new Error(
1361
- `Cluster ${clusterId} is still running. Use 'zeroshot stop' first if you want to restart it.`
1362
- );
1363
- }
1364
-
1365
- // Get failure info - either from saved state or from ledger
1366
- let failureInfo = cluster.failureInfo;
1367
-
1368
- if (!failureInfo) {
1369
- // Query ledger for AGENT_ERROR messages to find failed agent
1370
- const errors = cluster.messageBus.query({
1371
- cluster_id: clusterId,
1372
- topic: 'AGENT_ERROR',
1373
- limit: 10,
1374
- order: 'desc',
1375
- });
1376
-
1377
- if (errors.length > 0) {
1378
- // Use the first error found
1379
- const firstError = errors[0];
1380
- failureInfo = {
1381
- agentId: firstError.sender,
1382
- taskId: firstError.content?.data?.taskId || null,
1383
- iteration: firstError.content?.data?.iteration || 0,
1384
- error: firstError.content?.data?.error || firstError.content?.text,
1385
- timestamp: firstError.timestamp,
1386
- };
1387
- this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
1388
- }
1389
- }
1390
-
1391
- // CRITICAL: Recreate isolation container if needed
1392
- if (cluster.isolation?.enabled) {
1393
- const { spawn } = require('child_process');
1394
- const oldContainerId = cluster.isolation.containerId;
1395
-
1396
- // Check if container exists
1397
- const checkContainer = spawn('docker', ['inspect', oldContainerId], {
1398
- stdio: 'ignore',
1399
- });
1400
- const containerExists = await new Promise((resolve) => {
1401
- checkContainer.on('close', (code) => resolve(code === 0));
1402
- });
1403
-
1404
- if (!containerExists) {
1405
- this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
1406
-
1407
- // Create new container using saved workDir (CRITICAL for isolation mode resume)
1408
- // The isolated workspace at /tmp/zeroshot-isolated/{clusterId} was preserved by stop()
1409
- const workDir = cluster.isolation.workDir;
1410
- if (!workDir) {
1411
- throw new Error(
1412
- `Cannot resume cluster ${clusterId}: workDir not saved in isolation state`
1413
- );
1414
- }
1415
-
1416
- // Check if isolated workspace still exists (it should, if stop() was used)
1417
- const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
1418
- if (!fs.existsSync(isolatedPath)) {
1419
- throw new Error(
1420
- `Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
1421
- `Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
1422
- );
1423
- }
1424
-
1425
- const providerName = normalizeProviderName(
1426
- cluster.config?.forceProvider ||
1427
- cluster.config?.defaultProvider ||
1428
- loadSettings().defaultProvider ||
1429
- 'claude'
1430
- );
1431
- const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
1432
- workDir, // Use saved workDir, NOT process.cwd()
1433
- image: cluster.isolation.image,
1434
- reuseExistingWorkspace: true, // CRITICAL: Don't wipe existing work
1435
- provider: providerName,
1436
- });
1437
-
1438
- this._log(`[Orchestrator] New container created: ${newContainerId}`);
1439
-
1440
- // Update cluster isolation state
1441
- cluster.isolation.containerId = newContainerId;
1442
-
1443
- // CRITICAL: Update all agents' isolation context with new container ID
1444
- for (const agent of cluster.agents) {
1445
- if (agent.isolation?.enabled) {
1446
- agent.isolation.containerId = newContainerId;
1447
- agent.isolation.manager = cluster.isolation.manager;
1448
- }
1449
- }
1450
-
1451
- this._log(`[Orchestrator] All agents updated with new container ID`);
1452
- } else {
1453
- this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
1454
- }
1455
- }
1456
-
1457
- // Verify worktree still exists for worktree isolation mode
1458
- if (cluster.worktree?.enabled) {
1459
- const worktreePath = cluster.worktree.path;
1460
- if (!fs.existsSync(worktreePath)) {
1461
- throw new Error(
1462
- `Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
1463
- `Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
1464
- );
1465
- }
1466
-
1467
- this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
1468
- this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1469
- }
1470
-
1471
- // Restart all agents
1472
- cluster.state = 'running';
1473
- for (const agent of cluster.agents) {
1474
- if (!agent.running) {
1475
- await agent.start();
1476
- }
1477
- }
1478
-
1479
- // Query recent messages from ledger to provide context
1480
- const recentMessages = cluster.messageBus
1481
- .query({
1482
- cluster_id: clusterId,
1483
- limit: 50,
1484
- order: 'desc',
1485
- })
1486
- .reverse();
1487
-
1488
- // CASE 1: Failed cluster - Resume the failed agent with error context
1489
- if (failureInfo) {
1490
- const { agentId, iteration, error } = failureInfo;
1491
- this._log(
1492
- `[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
1493
- );
1494
- this._log(`[Orchestrator] Previous error: ${error}`);
1495
-
1496
- // Find the failed agent
1497
- const failedAgent = cluster.agents.find((a) => a.id === agentId);
1498
- if (!failedAgent) {
1499
- throw new Error(`Failed agent '${agentId}' not found in cluster`);
1500
- }
1501
-
1502
- // Build failure resume context
1503
- const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1504
- let context = `You are resuming from a previous failed attempt.\n\n`;
1505
- context += `Previous error: ${error}\n\n`;
1506
- context += `## Recent Context\n\n`;
1507
-
1508
- for (const msg of recentMessages.slice(-10)) {
1509
- if (msg.topic === 'AGENT_OUTPUT' || msg.topic === 'VALIDATION_RESULT') {
1510
- context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1511
- }
1512
- }
1513
-
1514
- context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1515
-
1516
- // Clear failure info since we're resuming
1517
- cluster.failureInfo = null;
1518
-
1519
- // Save updated state
1520
- await this._saveClusters();
1521
-
1522
- // Resume the failed agent
1523
- failedAgent.resume(context).catch((err) => {
1524
- console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
1525
- });
1526
-
1527
- this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
1528
-
1529
- return {
1530
- id: clusterId,
1531
- state: cluster.state,
1532
- resumeType: 'failure',
1533
- resumedAgent: agentId,
1534
- previousError: error,
1535
- };
1536
- }
1537
-
1538
- // CASE 2: Cleanly stopped cluster - Resume by re-triggering based on ledger state
1539
- this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
1540
-
1541
- // Build generic resume context
1542
- const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1543
- let context = `Resuming cluster from previous session.\n\n`;
1544
- context += `## Recent Context\n\n`;
1545
-
1546
- for (const msg of recentMessages.slice(-10)) {
1547
- if (
1548
- msg.topic === 'AGENT_OUTPUT' ||
1549
- msg.topic === 'VALIDATION_RESULT' ||
1550
- msg.topic === 'ISSUE_OPENED'
1551
- ) {
1552
- context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1553
- }
1554
- }
1555
-
1556
- context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1557
-
1558
- // Find the LAST workflow trigger - not arbitrary last 5 messages
1559
- // This is the message that indicates current workflow state
1560
- const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
1561
- const agentsToResume = [];
1562
-
1563
- if (lastTrigger) {
1564
- this._log(
1565
- `[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
1566
- );
1567
-
1568
- for (const agent of cluster.agents) {
1569
- if (!agent.config.triggers) continue;
1570
-
1571
- const matchingTrigger = agent.config.triggers.find((trigger) => {
1572
- // Exact match
1573
- if (trigger.topic === lastTrigger.topic) return true;
1574
- // Wildcard match
1575
- if (trigger.topic === '*') return true;
1576
- // Prefix match (e.g., "VALIDATION_*")
1577
- if (trigger.topic.endsWith('*')) {
1578
- const prefix = trigger.topic.slice(0, -1);
1579
- return lastTrigger.topic.startsWith(prefix);
1580
- }
1581
- return false;
1582
- });
1583
-
1584
- if (matchingTrigger) {
1585
- // Evaluate logic script if present
1586
- if (matchingTrigger.logic?.script) {
1587
- const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
1588
- if (!shouldTrigger) continue;
1589
- }
1590
- agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
1591
- }
1592
- }
1593
- } else {
1594
- this._log(`[Orchestrator] No workflow triggers found in ledger`);
1595
- }
1596
-
1597
- if (agentsToResume.length === 0) {
1598
- if (!lastTrigger) {
1599
- // No workflow activity - cluster never really started
1600
- this._log(
1601
- `[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
1602
- );
1603
- this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
1604
-
1605
- // Re-publish the original issue if we have it
1606
- const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
1607
- if (issueMessage) {
1608
- cluster.messageBus.publish({
1609
- cluster_id: clusterId,
1610
- topic: 'ISSUE_OPENED',
1611
- sender: 'system',
1612
- receiver: 'broadcast',
1613
- content: issueMessage.content,
1614
- metadata: { _resumed: true, _originalId: issueMessage.id },
1615
- });
1616
- } else {
1617
- throw new Error(
1618
- `Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
1619
- `The cluster may not have started properly. Try: zeroshot run <issue> instead.`
1620
- );
1621
- }
1622
- } else {
1623
- // Had a trigger but no agents matched - something is wrong with agent configs
1624
- throw new Error(
1625
- `Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
1626
- `Check agent trigger configurations.`
1627
- );
1628
- }
1629
- } else {
1630
- // Resume agents that should run based on ledger state
1631
- this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
1632
- for (const { agent, message } of agentsToResume) {
1633
- this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
1634
- agent.resume(context).catch((err) => {
1635
- console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
1636
- });
1637
- }
1638
- }
1639
-
1640
- // Save updated state
1641
- await this._saveClusters();
1642
-
1643
- this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
1644
-
1645
- return {
1646
- id: clusterId,
1647
- state: cluster.state,
1648
- resumeType: 'clean',
1649
- resumedAgents: agentsToResume.map((a) => a.agent.id),
1650
- };
1651
- }
1652
-
1653
- /**
1654
- * Force restart a stale agent with imperative prompt injection
1655
- * @param {string} clusterId - Cluster ID
1656
- * @param {string} agentId - Agent to restart
1657
- * @param {number} staleDuration - How long agent was stale (ms)
1658
- * @private
1659
- */
1660
- async _forceRestartAgent(clusterId, agentId, staleDuration) {
1661
- const cluster = this.clusters.get(clusterId);
1662
- if (!cluster) {
1663
- throw new Error(`Cluster ${clusterId} not found`);
1664
- }
1665
-
1666
- const agent = cluster.agents.find((a) => a.id === agentId);
1667
- if (!agent) {
1668
- throw new Error(`Agent ${agentId} not found in cluster ${clusterId}`);
1669
- }
1670
-
1671
- // Kill current task
1672
- try {
1673
- agent._killTask();
1674
- } catch (err) {
1675
- this._log(`⚠️ Failed to kill agent ${agentId} task:`, err.message);
1676
- }
1677
-
1678
- // Build imperative restart context
1679
- const staleMinutes = Math.round(staleDuration / 60000);
1680
- const imperativePrompt = `
1681
- 🔴 CRITICAL: Your previous session STOPPED PRODUCING OUTPUT for ${staleMinutes} minutes and was detected as STUCK.
1682
-
1683
- ## What Happened
1684
- - Last output timestamp: ${new Date(Date.now() - staleDuration).toISOString()}
1685
- - Detected as stale after ${staleMinutes} minutes of silence
1686
- - Process was forcefully restarted
1687
-
1688
- ## Your Instructions
1689
- You MUST complete your current task. DO NOT STOP until you either:
1690
- 1. Successfully complete the task and publish your completion message, OR
1691
- 2. Explicitly state WHY you cannot complete the task (missing files, impossible requirements, etc.)
1692
-
1693
- If you discovered that files you need to modify don't exist:
1694
- - CREATE them from scratch with the expected implementation
1695
- - DO NOT silently give up
1696
- - DO NOT stop working without explicit explanation
1697
-
1698
- If you are stuck in an impossible situation:
1699
- - EXPLAIN the problem clearly
1700
- - PROPOSE alternative solutions
1701
- - WAIT for guidance - do not exit
1702
-
1703
- ## Resume Your Work
1704
- Continue from where you left off. Review your previous output to understand what you were working on.
1705
- `.trim();
1706
-
1707
- // Get recent context from ledger
1708
- const recentMessages = cluster.messageBus
1709
- .query({
1710
- cluster_id: cluster.id,
1711
- limit: 10,
1712
- order: 'desc',
1713
- })
1714
- .reverse();
1715
-
1716
- const contextText = recentMessages
1717
- .map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
1718
- .join('\n\n');
1719
-
1720
- const fullContext = `${imperativePrompt}\n\n## Recent Context\n${contextText}`;
1721
-
1722
- // Resume agent with imperative prompt
1723
- this._log(
1724
- `🔄 Restarting agent ${agentId} with imperative prompt (${imperativePrompt.length} chars)`
1725
- );
1726
-
1727
- try {
1728
- await agent.resume(fullContext);
1729
- this._log(`✅ Agent ${agentId} successfully restarted`);
1730
- } catch (err) {
1731
- this._log(`❌ Failed to resume agent ${agentId}:`, err.message);
1732
- throw err;
1733
- }
1734
- }
1735
-
1736
- /**
1737
- * Handle operation chain from CLUSTER_OPERATIONS message
1738
- * Executes operations sequentially: add_agents, remove_agents, update_agent, publish
1739
- *
1740
- * Validation strategy:
1741
- * 1. Pre-validate all agent configs before executing any operations
1742
- * 2. Build a mock cluster config with proposed changes
1743
- * 3. Run config-validator on the mock to catch structural issues
1744
- * 4. Only execute operations if validation passes
1745
- *
1746
- * @param {string} clusterId - Cluster ID
1747
- * @param {Array} operations - Array of operation objects
1748
- * @param {string} sender - Who sent the operations (for attribution)
1749
- * @param {Object} context - Isolation context { isolationManager, containerId }
1750
- * @private
1751
- */
1752
- async _handleOperations(clusterId, operations, sender, context = {}) {
1753
- const cluster = this.clusters.get(clusterId);
1754
- if (!cluster) {
1755
- throw new Error(`Cluster ${clusterId} not found`);
1756
- }
1757
-
1758
- this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
1759
-
1760
- // Phase 1: Pre-validate operation structure
1761
- const validationErrors = [];
1762
- for (let i = 0; i < operations.length; i++) {
1763
- const op = operations[i];
1764
- if (!op.action) {
1765
- validationErrors.push(`Operation ${i}: missing 'action' field`);
1766
- continue;
1767
- }
1768
- if (!VALID_OPERATIONS.includes(op.action)) {
1769
- validationErrors.push(
1770
- `Operation ${i}: unknown action '${op.action}'. Valid: ${VALID_OPERATIONS.join(', ')}`
1771
- );
1772
- }
1773
- }
1774
-
1775
- if (validationErrors.length > 0) {
1776
- const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
1777
- this._log(`[Orchestrator] ❌ ${errorMsg}`);
1778
- throw new Error(errorMsg);
1779
- }
1780
-
1781
- // Phase 2: Build mock cluster config with proposed agents
1782
- // Collect all agents that would exist after operations complete
1783
- const existingAgentConfigs = cluster.config.agents || [];
1784
- const proposedAgentConfigs = [...existingAgentConfigs];
1785
-
1786
- for (const op of operations) {
1787
- if (op.action === 'add_agents' && op.agents) {
1788
- for (const agentConfig of op.agents) {
1789
- // Check for duplicate before adding
1790
- const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
1791
- if (existingIdx === -1) {
1792
- proposedAgentConfigs.push(agentConfig);
1793
- }
1794
- }
1795
- } else if (op.action === 'remove_agents' && op.agentIds) {
1796
- for (const agentId of op.agentIds) {
1797
- const idx = proposedAgentConfigs.findIndex((a) => a.id === agentId);
1798
- if (idx !== -1) {
1799
- proposedAgentConfigs.splice(idx, 1);
1800
- }
1801
- }
1802
- } else if (op.action === 'update_agent' && op.agentId && op.updates) {
1803
- const agentConfig = proposedAgentConfigs.find((a) => a.id === op.agentId);
1804
- if (agentConfig) {
1805
- Object.assign(agentConfig, op.updates);
1806
- }
1807
- }
1808
- }
1809
-
1810
- // Phase 3: Validate proposed cluster config
1811
- const mockConfig = { agents: proposedAgentConfigs };
1812
- const validation = configValidator.validateConfig(mockConfig);
1813
-
1814
- if (!validation.valid) {
1815
- const errorMsg = `Proposed cluster configuration is invalid:\n - ${validation.errors.join('\n - ')}`;
1816
- this._log(`[Orchestrator] ❌ ${errorMsg}`);
1817
-
1818
- // Publish validation failure for conductor to see and retry
1819
- cluster.messageBus.publish({
1820
- cluster_id: clusterId,
1821
- topic: 'CLUSTER_OPERATIONS_VALIDATION_FAILED',
1822
- sender: 'orchestrator',
1823
- content: {
1824
- text: 'Operation chain would create invalid cluster configuration',
1825
- data: {
1826
- errors: validation.errors,
1827
- warnings: validation.warnings,
1828
- operations: operations,
1829
- },
1830
- },
1831
- });
1832
-
1833
- throw new Error(errorMsg);
1834
- }
1835
-
1836
- // Log warnings but proceed
1837
- if (validation.warnings.length > 0) {
1838
- this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
1839
- for (const warning of validation.warnings) {
1840
- this._log(` - ${warning}`);
1841
- }
1842
- }
1843
-
1844
- // Phase 4: Execute validated operations
1845
- this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
1846
-
1847
- for (let i = 0; i < operations.length; i++) {
1848
- const op = operations[i];
1849
- this._log(` [${i + 1}/${operations.length}] ${op.action}`);
1850
-
1851
- switch (op.action) {
1852
- case 'add_agents':
1853
- await this._opAddAgents(cluster, op, context);
1854
- break;
1855
-
1856
- case 'remove_agents':
1857
- await this._opRemoveAgents(cluster, op);
1858
- break;
1859
-
1860
- case 'update_agent':
1861
- this._opUpdateAgent(cluster, op);
1862
- break;
1863
-
1864
- case 'publish':
1865
- this._opPublish(cluster, op, sender);
1866
- break;
1867
-
1868
- case 'load_config':
1869
- await this._opLoadConfig(cluster, op, context);
1870
- break;
1871
- }
1872
- }
1873
-
1874
- this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
1875
-
1876
- // Publish success notification
1877
- cluster.messageBus.publish({
1878
- cluster_id: clusterId,
1879
- topic: 'CLUSTER_OPERATIONS_SUCCESS',
1880
- sender: 'orchestrator',
1881
- content: {
1882
- text: `Executed ${operations.length} operation(s)`,
1883
- data: {
1884
- operationCount: operations.length,
1885
- agentCount: cluster.agents.length,
1886
- },
1887
- },
1888
- });
1889
-
1890
- // Save updated cluster state to disk
1891
- await this._saveClusters();
1892
- }
1893
-
1894
- /**
1895
- * Operation: add_agents - Spawn new agents dynamically
1896
- * @private
1897
- */
1898
- async _opAddAgents(cluster, op, context) {
1899
- const agents = op.agents;
1900
- if (!agents || !Array.isArray(agents)) {
1901
- throw new Error('add_agents operation missing agents array');
1902
- }
1903
-
1904
- // CRITICAL: Inject cluster's worktree cwd into dynamically added agents
1905
- // Without this, template agents (planning, implementation, validator) run in
1906
- // original directory instead of worktree, causing --ship mode to pollute main
1907
- const agentCwd = cluster.worktree?.path || cluster.isolation?.workDir || process.cwd();
1908
-
1909
- for (const agentConfig of agents) {
1910
- // Inject cwd if not already set (same as startCluster does for initial agents)
1911
- if (!agentConfig.cwd) {
1912
- agentConfig.cwd = agentCwd;
1913
- this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
1914
- }
1915
-
1916
- // Apply model override if set (for consistency with initial agents)
1917
- if (cluster.modelOverride) {
1918
- applyModelOverride(agentConfig, cluster.modelOverride);
1919
- this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
1920
- }
1921
- // Validate agent config has required fields
1922
- if (!agentConfig.id) {
1923
- throw new Error('Agent config missing required field: id');
1924
- }
1925
-
1926
- // Check for duplicate agent ID
1927
- const existingAgent = cluster.agents.find((a) => a.id === agentConfig.id);
1928
- if (existingAgent) {
1929
- this._log(` ⚠️ Agent ${agentConfig.id} already exists, skipping`);
1930
- continue;
1931
- }
1932
-
1933
- // Add to config agents array (for persistence)
1934
- if (!cluster.config.agents) {
1935
- cluster.config.agents = [];
1936
- }
1937
- cluster.config.agents.push(agentConfig);
1938
-
1939
- // Build agent options
1940
- const agentOptions = {
1941
- testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
1942
- quiet: this.quiet,
1943
- modelOverride: cluster.modelOverride || null,
1944
- };
1945
-
1946
- // TaskRunner DI - propagate to dynamically spawned agents
1947
- if (this.taskRunner) {
1948
- agentOptions.taskRunner = this.taskRunner;
1949
- }
1950
-
1951
- // Pass isolation context if cluster is running in isolation mode
1952
- if (cluster.isolation?.enabled && context.isolationManager) {
1953
- agentOptions.isolation = {
1954
- enabled: true,
1955
- manager: context.isolationManager,
1956
- clusterId: cluster.id,
1957
- };
1958
- }
1959
-
1960
- // Create and start agent
1961
- const agent = new AgentWrapper(agentConfig, cluster.messageBus, cluster, agentOptions);
1962
- cluster.agents.push(agent);
1963
- await agent.start();
1964
-
1965
- this._log(
1966
- ` ✓ Added agent: ${agentConfig.id} (role: ${agentConfig.role || 'unspecified'})`
1967
- );
1968
- }
1969
- }
1970
-
1971
- /**
1972
- * Operation: remove_agents - Stop and remove agents by ID
1973
- * @private
1974
- */
1975
- async _opRemoveAgents(cluster, op) {
1976
- const agentIds = op.agentIds;
1977
- if (!agentIds || !Array.isArray(agentIds)) {
1978
- throw new Error('remove_agents operation missing agentIds array');
1979
- }
1980
-
1981
- for (const agentId of agentIds) {
1982
- const agentIndex = cluster.agents.findIndex((a) => a.id === agentId);
1983
- if (agentIndex === -1) {
1984
- this._log(` ⚠️ Agent ${agentId} not found, skipping removal`);
1985
- continue;
1986
- }
1987
-
1988
- const agent = cluster.agents[agentIndex];
1989
- await agent.stop();
1990
-
1991
- // Remove from cluster.agents
1992
- cluster.agents.splice(agentIndex, 1);
1993
-
1994
- // Remove from config.agents
1995
- if (cluster.config.agents) {
1996
- const configIndex = cluster.config.agents.findIndex((a) => a.id === agentId);
1997
- if (configIndex !== -1) {
1998
- cluster.config.agents.splice(configIndex, 1);
1999
- }
2000
- }
2001
-
2002
- this._log(` ✓ Removed agent: ${agentId}`);
2003
- }
2004
- }
2005
-
2006
- /**
2007
- * Operation: update_agent - Modify existing agent config at runtime
2008
- * Note: Some updates may require agent restart to take effect
2009
- * @private
2010
- */
2011
- _opUpdateAgent(cluster, op) {
2012
- const { agentId, updates } = op;
2013
- if (!agentId) {
2014
- throw new Error('update_agent operation missing agentId');
2015
- }
2016
- if (!updates || typeof updates !== 'object') {
2017
- throw new Error('update_agent operation missing updates object');
2018
- }
2019
-
2020
- const agent = cluster.agents.find((a) => a.id === agentId);
2021
- if (!agent) {
2022
- throw new Error(`update_agent: Agent ${agentId} not found`);
2023
- }
2024
-
2025
- // Apply updates to agent config
2026
- Object.assign(agent.config, updates);
2027
-
2028
- // Also update in cluster.config.agents for persistence
2029
- if (cluster.config.agents) {
2030
- const configAgent = cluster.config.agents.find((a) => a.id === agentId);
2031
- if (configAgent) {
2032
- Object.assign(configAgent, updates);
2033
- }
2034
- }
2035
-
2036
- this._log(` ✓ Updated agent: ${agentId} (fields: ${Object.keys(updates).join(', ')})`);
2037
- }
2038
-
2039
- /**
2040
- * Operation: publish - Publish a message to the bus
2041
- * @private
2042
- */
2043
- _opPublish(cluster, op, sender) {
2044
- const { topic, content, metadata } = op;
2045
- if (!topic) {
2046
- throw new Error('publish operation missing topic');
2047
- }
2048
-
2049
- cluster.messageBus.publish({
2050
- cluster_id: cluster.id,
2051
- topic,
2052
- sender: op.sender || sender,
2053
- receiver: op.receiver || 'broadcast',
2054
- content: content || {},
2055
- metadata: metadata || null,
2056
- });
2057
-
2058
- this._log(` ✓ Published to topic: ${topic}`);
2059
- }
2060
-
2061
- /**
2062
- * Operation: load_config - Load agents from a cluster config
2063
- *
2064
- * Supports two formats:
2065
- * 1. Static config: { config: 'config-name' } - loads from cluster-templates/config-name.json
2066
- * 2. Parameterized: { config: { base: 'template-name', params: {...} } } - resolves base template with params
2067
- *
2068
- * @private
2069
- */
2070
- async _opLoadConfig(cluster, op, context) {
2071
- const { config } = op;
2072
- if (!config) {
2073
- throw new Error('load_config operation missing config');
2074
- }
2075
-
2076
- const templatesDir = path.join(__dirname, '..', 'cluster-templates');
2077
- let loadedConfig;
2078
-
2079
- // Check if config is parameterized ({ base, params }) or static (string)
2080
- if (typeof config === 'object' && config.base) {
2081
- // Parameterized template - resolve with TemplateResolver
2082
- const { base, params } = config;
2083
- this._log(` Loading parameterized template: ${base}`);
2084
- this._log(` Params: ${JSON.stringify(params)}`);
2085
-
2086
- const resolver = new TemplateResolver(templatesDir);
2087
- loadedConfig = resolver.resolve(base, params);
2088
-
2089
- this._log(` ✓ Resolved template: ${base} → ${loadedConfig.agents?.length || 0} agent(s)`);
2090
- } else if (typeof config === 'string') {
2091
- // Static config - load directly from file
2092
- const configPath = path.join(templatesDir, `${config}.json`);
2093
-
2094
- if (!fs.existsSync(configPath)) {
2095
- throw new Error(`Config not found: ${config} (looked in ${configPath})`);
2096
- }
2097
-
2098
- this._log(` Loading static config: ${config}`);
2099
-
2100
- const configContent = fs.readFileSync(configPath, 'utf8');
2101
- loadedConfig = JSON.parse(configContent);
2102
- } else {
2103
- throw new Error(
2104
- `Invalid config format: expected string or {base, params}, got ${typeof config}`
2105
- );
2106
- }
2107
-
2108
- if (!loadedConfig.agents || !Array.isArray(loadedConfig.agents)) {
2109
- throw new Error(`Config has no agents array`);
2110
- }
2111
-
2112
- this._log(` Found ${loadedConfig.agents.length} agent(s)`);
2113
-
2114
- // Add agents from loaded config (reuse existing add_agents logic)
2115
- await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
2116
-
2117
- this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
2118
-
2119
- // Inject completion agent (templates don't include one - orchestrator controls termination)
2120
- await this._injectCompletionAgent(cluster, context);
2121
- }
2122
-
2123
- /**
2124
- * Inject appropriate completion agent based on mode
2125
- * Templates define work, orchestrator controls termination strategy
2126
- * @private
2127
- */
2128
- async _injectCompletionAgent(cluster, context) {
2129
- // Skip if completion agent already exists
2130
- const hasCompletionAgent = cluster.agents.some(
2131
- (a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
2132
- );
2133
- if (hasCompletionAgent) {
2134
- return;
2135
- }
2136
-
2137
- const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
2138
-
2139
- if (isPrMode) {
2140
- // Load git-pusher for PR mode
2141
- const gitPusherPath = path.join(__dirname, 'agents', 'git-pusher-agent.json');
2142
- const gitPusherConfig = JSON.parse(fs.readFileSync(gitPusherPath, 'utf8'));
2143
-
2144
- // Get issue context from ledger
2145
- const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
2146
- const issueNumber = issueMsg?.content?.data?.number || 'unknown';
2147
- const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
2148
-
2149
- // Inject placeholders
2150
- gitPusherConfig.prompt = gitPusherConfig.prompt
2151
- .replace(/\{\{issue_number\}\}/g, issueNumber)
2152
- .replace(/\{\{issue_title\}\}/g, issueTitle);
2153
-
2154
- await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
2155
- this._log(` [--pr mode] Injected git-pusher agent`);
2156
- } else {
2157
- // Default completion-detector
2158
- const completionDetector = {
2159
- id: 'completion-detector',
2160
- role: 'orchestrator',
2161
- model: 'haiku',
2162
- timeout: 0,
2163
- triggers: [
2164
- {
2165
- topic: 'VALIDATION_RESULT',
2166
- logic: {
2167
- engine: 'javascript',
2168
- script: `const validators = cluster.getAgentsByRole('validator');
2169
- const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });
2170
- if (!lastPush) return false;
2171
- if (validators.length === 0) return true;
2172
-
2173
- const validatorIds = new Set(validators.map((v) => v.id));
2174
- const results = ledger.query({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });
2175
-
2176
- const latestByValidator = new Map();
2177
- for (const msg of results) {
2178
- if (!validatorIds.has(msg.sender)) continue;
2179
- latestByValidator.set(msg.sender, msg);
2180
- }
2181
-
2182
- if (latestByValidator.size < validators.length) return false;
2183
-
2184
- for (const validator of validators) {
2185
- const msg = latestByValidator.get(validator.id);
2186
- const approved = msg?.content?.data?.approved;
2187
- if (!(approved === true || approved === 'true')) return false;
2188
- }
2189
-
2190
- return true;`,
2191
- },
2192
- action: 'stop_cluster',
2193
- },
2194
- ],
2195
- };
2196
-
2197
- await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
2198
- this._log(` Injected completion-detector agent`);
2199
- }
2200
- }
2201
-
2202
- /**
2203
- * Check if a process with given PID is running
2204
- * @param {Number} pid - Process ID
2205
- * @returns {Boolean} True if process exists
2206
- * @private
2207
- */
2208
- _isProcessRunning(pid) {
2209
- if (!pid) return false;
2210
- try {
2211
- // Signal 0 doesn't kill, just checks if process exists
2212
- process.kill(pid, 0);
2213
- return true;
2214
- } catch (e) {
2215
- // ESRCH = No such process, EPERM = process exists but no permission
2216
- return e.code === 'EPERM';
2217
- }
2218
- }
2219
-
2220
- /**
2221
- * Get cluster status
2222
- * @param {String} clusterId - Cluster ID
2223
- * @returns {Object} Cluster status
2224
- */
2225
- getStatus(clusterId) {
2226
- const cluster = this.clusters.get(clusterId);
2227
- if (!cluster) {
2228
- throw new Error(`Cluster ${clusterId} not found`);
2229
- }
2230
-
2231
- // Detect zombie clusters: state=running but no backing process
2232
- let state = cluster.state;
2233
- let isZombie = false;
2234
- if (state === 'running') {
2235
- if (cluster.pid) {
2236
- // PID recorded - check if process is running
2237
- if (!this._isProcessRunning(cluster.pid)) {
2238
- state = 'zombie';
2239
- isZombie = true;
2240
- this._log(
2241
- `[Orchestrator] Detected zombie cluster ${clusterId} (PID ${cluster.pid} not running)`
2242
- );
2243
- }
2244
- } else {
2245
- // No PID recorded (legacy cluster or bug) - definitely a zombie
2246
- // New code always records PID for running clusters
2247
- state = 'zombie';
2248
- isZombie = true;
2249
- this._log(
2250
- `[Orchestrator] Detected zombie cluster ${clusterId} (no PID recorded - legacy or killed cluster)`
2251
- );
2252
- }
2253
- }
2254
-
2255
- return {
2256
- id: clusterId,
2257
- state: state,
2258
- isZombie: isZombie,
2259
- pid: cluster.pid || null,
2260
- createdAt: cluster.createdAt,
2261
- agents: cluster.agents.map((a) => a.getState()),
2262
- messageCount: cluster.messageBus.count({ cluster_id: clusterId }),
2263
- };
2264
- }
2265
-
2266
- /**
2267
- * List all clusters
2268
- * @returns {Array} List of cluster summaries
2269
- */
2270
- listClusters() {
2271
- return Array.from(this.clusters.values()).map((cluster) => {
2272
- // Detect zombie clusters (state=running but no backing process)
2273
- let state = cluster.state;
2274
- if (state === 'running') {
2275
- if (cluster.pid) {
2276
- if (!this._isProcessRunning(cluster.pid)) {
2277
- state = 'zombie';
2278
- }
2279
- } else {
2280
- // No PID recorded - definitely a zombie
2281
- state = 'zombie';
2282
- }
2283
- }
2284
-
2285
- return {
2286
- id: cluster.id,
2287
- state: state,
2288
- createdAt: cluster.createdAt,
2289
- agentCount: cluster.agents.length,
2290
- messageCount: cluster.messageBus.getAll(cluster.id).length,
2291
- };
2292
- });
2293
- }
2294
-
2295
- /**
2296
- * Get cluster object (for internal use)
2297
- * @param {String} clusterId - Cluster ID
2298
- * @returns {Object} Full cluster object
2299
- */
2300
- getCluster(clusterId) {
2301
- return this.clusters.get(clusterId);
2302
- }
2303
-
2304
- /**
2305
- * Export cluster conversation
2306
- * @param {String} clusterId - Cluster ID
2307
- * @param {String} format - Export format ('json' or 'markdown')
2308
- * @returns {String} Exported data
2309
- */
2310
- export(clusterId, format = 'json') {
2311
- const cluster = this.clusters.get(clusterId);
2312
- if (!cluster) {
2313
- throw new Error(`Cluster ${clusterId} not found`);
2314
- }
2315
-
2316
- const messages = cluster.messageBus.getAll(clusterId);
2317
-
2318
- if (format === 'json') {
2319
- return JSON.stringify(
2320
- {
2321
- cluster_id: clusterId,
2322
- state: cluster.state,
2323
- created_at: cluster.createdAt,
2324
- agents: cluster.agents.map((a) => a.getState()),
2325
- messages,
2326
- },
2327
- null,
2328
- 2
2329
- );
2330
- } else if (format === 'markdown') {
2331
- return this._exportMarkdown(cluster, clusterId, messages);
2332
- } else {
2333
- throw new Error(`Unknown export format: ${format}`);
2334
- }
2335
- }
2336
-
2337
- /**
2338
- * Export cluster as nicely formatted markdown
2339
- * @private
2340
- */
2341
- _exportMarkdown(cluster, clusterId, messages) {
2342
- const { parseProviderChunk } = require('./providers');
2343
-
2344
- // Find task info
2345
- const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
2346
- const taskText = issueOpened?.content?.text || 'Unknown task';
2347
-
2348
- // Calculate duration
2349
- const firstMsg = messages[0];
2350
- const lastMsg = messages[messages.length - 1];
2351
- const durationMs = lastMsg ? lastMsg.timestamp - firstMsg.timestamp : 0;
2352
- const durationMin = Math.round(durationMs / 60000);
2353
-
2354
- // Header
2355
- let md = `# Cluster: ${clusterId}\n\n`;
2356
- md += `| Property | Value |\n|----------|-------|\n`;
2357
- md += `| State | ${cluster.state} |\n`;
2358
- md += `| Created | ${new Date(cluster.createdAt).toLocaleString()} |\n`;
2359
- md += `| Duration | ${durationMin} minutes |\n`;
2360
- md += `| Agents | ${cluster.agents.map((a) => a.id).join(', ')} |\n\n`;
2361
-
2362
- // Task
2363
- md += `## Task\n\n${taskText}\n\n`;
2364
-
2365
- // Group messages by agent for cleaner output
2366
- const agentOutputs = new Map();
2367
-
2368
- for (const msg of messages) {
2369
- if (msg.topic === 'AGENT_OUTPUT') {
2370
- if (!agentOutputs.has(msg.sender)) {
2371
- agentOutputs.set(msg.sender, []);
2372
- }
2373
- agentOutputs.get(msg.sender).push(msg);
2374
- }
2375
- }
2376
-
2377
- // Agent sections
2378
- for (const [agentId, agentMsgs] of agentOutputs) {
2379
- md += `## Agent: ${agentId}\n\n`;
2380
-
2381
- let text = '';
2382
- let tools = [];
2383
-
2384
- for (const msg of agentMsgs) {
2385
- const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
2386
- if (!content) continue;
2387
-
2388
- const provider = normalizeProviderName(
2389
- msg.content?.data?.provider || msg.sender_provider || 'claude'
2390
- );
2391
- const events = parseProviderChunk(provider, content);
2392
- for (const event of events) {
2393
- switch (event.type) {
2394
- case 'text':
2395
- text += event.text;
2396
- break;
2397
- case 'tool_call':
2398
- tools.push({ name: event.toolName, input: event.input });
2399
- break;
2400
- case 'tool_result':
2401
- if (tools.length > 0) {
2402
- const lastTool = tools[tools.length - 1];
2403
- lastTool.result = event.content;
2404
- lastTool.isError = event.isError;
2405
- }
2406
- break;
2407
- }
2408
- }
2409
- }
2410
-
2411
- // Output text
2412
- if (text.trim()) {
2413
- md += `### Output\n\n${text.trim()}\n\n`;
2414
- }
2415
-
2416
- // Tools used
2417
- if (tools.length > 0) {
2418
- md += `### Tools Used\n\n`;
2419
- for (const tool of tools) {
2420
- const status = tool.isError ? '❌' : '✓';
2421
- md += `- **${tool.name}** ${status}\n`;
2422
- if (tool.input) {
2423
- const inputStr =
2424
- typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
2425
- if (inputStr.length < 100) {
2426
- md += ` - Input: \`${inputStr}\`\n`;
2427
- }
2428
- }
2429
- }
2430
- md += '\n';
2431
- }
2432
- }
2433
-
2434
- // Validation results
2435
- const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
2436
- if (validations.length > 0) {
2437
- md += `## Validation Results\n\n`;
2438
- for (const v of validations) {
2439
- const data = v.content?.data || {};
2440
- const approved = data.approved === true || data.approved === 'true';
2441
- const icon = approved ? '✅' : '❌';
2442
- md += `### ${v.sender} ${icon}\n\n`;
2443
- if (data.summary) {
2444
- md += `${data.summary}\n\n`;
2445
- }
2446
- if (!approved && data.issues) {
2447
- const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
2448
- if (Array.isArray(issues) && issues.length > 0) {
2449
- md += `**Issues:**\n`;
2450
- for (const issue of issues) {
2451
- md += `- ${issue}\n`;
2452
- }
2453
- md += '\n';
2454
- }
2455
- }
2456
- }
2457
- }
2458
-
2459
- // Final status
2460
- const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
2461
- if (clusterComplete) {
2462
- md += `## Result\n\n✅ **Cluster completed successfully**\n`;
2463
- }
2464
-
2465
- return md;
2466
- }
2467
-
2468
- /**
2469
- * Validate cluster configuration (delegates to config-validator module)
2470
- * @param {Object} config - Cluster configuration
2471
- * @param {Object} options - Validation options
2472
- * @param {boolean} options.strict - Treat warnings as errors (default: false)
2473
- * @returns {Object} { valid: Boolean, errors: Array, warnings: Array }
2474
- */
2475
- validateConfig(config, options = {}) {
2476
- const result = configValidator.validateConfig(config);
2477
-
2478
- // In strict mode, warnings become errors
2479
- if (options.strict && result.warnings.length > 0) {
2480
- result.errors.push(...result.warnings.map((w) => `[strict] ${w}`));
2481
- result.valid = false;
2482
- }
2483
-
2484
- return result;
2485
- }
2486
-
2487
- /**
2488
- * Load cluster configuration from file
2489
- * @param {String} configPath - Path to config JSON file
2490
- * @param {Object} options - Load options
2491
- * @param {boolean} options.strict - Treat warnings as errors
2492
- * @returns {Object} Parsed configuration
2493
- */
2494
- loadConfig(configPath, options = {}) {
2495
- const fullPath = path.resolve(configPath);
2496
- const content = fs.readFileSync(fullPath, 'utf8');
2497
- const config = JSON.parse(content);
2498
-
2499
- const validation = this.validateConfig(config, options);
2500
-
2501
- // Show warnings (but don't fail unless strict mode)
2502
- if (validation.warnings && validation.warnings.length > 0 && !this.quiet) {
2503
- console.warn('\n⚠️ Configuration warnings:');
2504
- for (const warning of validation.warnings) {
2505
- console.warn(` ${warning}`);
2506
- }
2507
- console.warn('');
2508
- }
2509
-
2510
- if (!validation.valid) {
2511
- const errorMsg = validation.errors.join('\n ');
2512
- throw new Error(`Invalid config:\n ${errorMsg}`);
2513
- }
2514
-
2515
- return config;
2516
- }
2517
- }
2518
-
2519
- module.exports = Orchestrator;