@covibes/zeroshot 5.4.0 → 5.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/LICENSE +1 -1
  2. package/README.md +8 -465
  3. package/cli/index.js +101 -5231
  4. package/package.json +16 -130
  5. package/CHANGELOG.md +0 -537
  6. package/cli/commands/providers.js +0 -149
  7. package/cli/lib/first-run.js +0 -211
  8. package/cli/lib/update-checker.js +0 -281
  9. package/cli/message-formatter-utils.js +0 -75
  10. package/cli/message-formatters-normal.js +0 -275
  11. package/cli/message-formatters-watch.js +0 -181
  12. package/cluster-templates/base-templates/debug-workflow.json +0 -375
  13. package/cluster-templates/base-templates/full-workflow.json +0 -581
  14. package/cluster-templates/base-templates/single-worker.json +0 -63
  15. package/cluster-templates/base-templates/worker-validator.json +0 -208
  16. package/cluster-templates/conductor-bootstrap.json +0 -124
  17. package/docker/zeroshot-cluster/Dockerfile +0 -152
  18. package/docker/zeroshot-cluster/pre-baked-deps.json +0 -28
  19. package/lib/completion.js +0 -174
  20. package/lib/docker-config.js +0 -220
  21. package/lib/git-remote-utils.js +0 -165
  22. package/lib/id-detector.js +0 -56
  23. package/lib/provider-defaults.js +0 -62
  24. package/lib/provider-detection.js +0 -59
  25. package/lib/provider-names.js +0 -57
  26. package/lib/settings/claude-auth.js +0 -78
  27. package/lib/settings.js +0 -503
  28. package/lib/stream-json-parser.js +0 -10
  29. package/scripts/fix-node-pty-permissions.js +0 -75
  30. package/scripts/record-demo.sh +0 -279
  31. package/scripts/setup-merge-queue.sh +0 -170
  32. package/scripts/test-install.sh +0 -40
  33. package/scripts/validate-templates.js +0 -100
  34. package/src/agent/agent-config.js +0 -246
  35. package/src/agent/agent-context-builder.js +0 -444
  36. package/src/agent/agent-hook-executor.js +0 -599
  37. package/src/agent/agent-lifecycle.js +0 -743
  38. package/src/agent/agent-stuck-detector.js +0 -256
  39. package/src/agent/agent-task-executor.js +0 -1690
  40. package/src/agent/agent-trigger-evaluator.js +0 -67
  41. package/src/agent/output-extraction.js +0 -217
  42. package/src/agent/output-reformatter.js +0 -175
  43. package/src/agent/schema-utils.js +0 -146
  44. package/src/agent-wrapper.js +0 -579
  45. package/src/agents/git-pusher-template.js +0 -285
  46. package/src/attach/attach-client.js +0 -438
  47. package/src/attach/attach-server.js +0 -543
  48. package/src/attach/index.js +0 -35
  49. package/src/attach/protocol.js +0 -220
  50. package/src/attach/ring-buffer.js +0 -121
  51. package/src/attach/socket-discovery.js +0 -267
  52. package/src/claude-task-runner.js +0 -574
  53. package/src/config-router.js +0 -80
  54. package/src/config-validator.js +0 -1925
  55. package/src/copy-worker.js +0 -43
  56. package/src/input-helpers.js +0 -65
  57. package/src/isolation-manager.js +0 -1471
  58. package/src/issue-providers/README.md +0 -305
  59. package/src/issue-providers/azure-devops-provider.js +0 -273
  60. package/src/issue-providers/base-provider.js +0 -232
  61. package/src/issue-providers/github-provider.js +0 -179
  62. package/src/issue-providers/gitlab-provider.js +0 -241
  63. package/src/issue-providers/index.js +0 -196
  64. package/src/issue-providers/jira-provider.js +0 -239
  65. package/src/ledger.js +0 -631
  66. package/src/lib/safe-exec.js +0 -88
  67. package/src/logic-engine.js +0 -223
  68. package/src/message-bus-bridge.js +0 -139
  69. package/src/message-bus.js +0 -249
  70. package/src/name-generator.js +0 -232
  71. package/src/orchestrator.js +0 -2815
  72. package/src/preflight.js +0 -639
  73. package/src/process-metrics.js +0 -608
  74. package/src/providers/anthropic/cli-builder.js +0 -73
  75. package/src/providers/anthropic/index.js +0 -204
  76. package/src/providers/anthropic/models.js +0 -23
  77. package/src/providers/anthropic/output-parser.js +0 -177
  78. package/src/providers/base-provider.js +0 -251
  79. package/src/providers/capabilities.js +0 -60
  80. package/src/providers/google/cli-builder.js +0 -55
  81. package/src/providers/google/index.js +0 -116
  82. package/src/providers/google/models.js +0 -24
  83. package/src/providers/google/output-parser.js +0 -101
  84. package/src/providers/index.js +0 -91
  85. package/src/providers/openai/cli-builder.js +0 -133
  86. package/src/providers/openai/index.js +0 -136
  87. package/src/providers/openai/models.js +0 -21
  88. package/src/providers/openai/output-parser.js +0 -143
  89. package/src/providers/opencode/cli-builder.js +0 -42
  90. package/src/providers/opencode/index.js +0 -103
  91. package/src/providers/opencode/models.js +0 -55
  92. package/src/providers/opencode/output-parser.js +0 -122
  93. package/src/schemas/sub-cluster.js +0 -185
  94. package/src/status-footer.js +0 -1026
  95. package/src/sub-cluster-wrapper.js +0 -601
  96. package/src/task-runner.js +0 -30
  97. package/src/template-resolver.js +0 -372
  98. package/src/tui/CHANGES.txt +0 -133
  99. package/src/tui/LAYOUT.md +0 -261
  100. package/src/tui/README.txt +0 -192
  101. package/src/tui/TWO-LEVEL-NAVIGATION.md +0 -186
  102. package/src/tui/data-poller.js +0 -349
  103. package/src/tui/demo.js +0 -208
  104. package/src/tui/formatters.js +0 -129
  105. package/src/tui/index.js +0 -193
  106. package/src/tui/keybindings.js +0 -324
  107. package/src/tui/layout.js +0 -334
  108. package/src/tui/renderer.js +0 -184
  109. package/task-lib/attachable-watcher.js +0 -309
  110. package/task-lib/claude-recovery.js +0 -156
  111. package/task-lib/commands/clean.js +0 -50
  112. package/task-lib/commands/episodes.js +0 -105
  113. package/task-lib/commands/get-log-path.js +0 -23
  114. package/task-lib/commands/kill.js +0 -32
  115. package/task-lib/commands/list.js +0 -105
  116. package/task-lib/commands/logs.js +0 -453
  117. package/task-lib/commands/resume.js +0 -42
  118. package/task-lib/commands/run.js +0 -57
  119. package/task-lib/commands/schedule.js +0 -105
  120. package/task-lib/commands/scheduler-cmd.js +0 -96
  121. package/task-lib/commands/schedules.js +0 -148
  122. package/task-lib/commands/status.js +0 -44
  123. package/task-lib/commands/unschedule.js +0 -16
  124. package/task-lib/completion.js +0 -9
  125. package/task-lib/config.js +0 -8
  126. package/task-lib/name-generator.js +0 -230
  127. package/task-lib/package.json +0 -3
  128. package/task-lib/runner.js +0 -201
  129. package/task-lib/scheduler.js +0 -252
  130. package/task-lib/store.js +0 -529
  131. package/task-lib/tui/formatters.js +0 -215
  132. package/task-lib/tui/index.js +0 -197
  133. package/task-lib/tui/layout.js +0 -111
  134. package/task-lib/tui/renderer.js +0 -129
  135. package/task-lib/tui.js +0 -425
  136. package/task-lib/watcher.js +0 -226
@@ -1,2815 +0,0 @@
1
- /**
2
- * Orchestrator - Manages cluster lifecycle
3
- *
4
- * Provides:
5
- * - Cluster initialization and configuration
6
- * - Agent lifecycle management
7
- * - GitHub issue integration
8
- * - Cluster state tracking
9
- * - Crash recovery
10
- */
11
-
12
- const fs = require('fs');
13
- const path = require('path');
14
- const os = require('os');
15
- const lockfile = require('proper-lockfile');
16
-
17
- // Stale lock timeout in ms - if lock file is older than this, delete it
18
- const LOCK_STALE_MS = 5000;
19
-
20
- /**
21
- * Remove lock file if it's stale (older than LOCK_STALE_MS)
22
- * Handles crashes that leave orphaned lock files
23
- */
24
- function cleanStaleLock(lockPath) {
25
- try {
26
- if (fs.existsSync(lockPath)) {
27
- const age = Date.now() - fs.statSync(lockPath).mtimeMs;
28
- if (age > LOCK_STALE_MS) {
29
- fs.unlinkSync(lockPath);
30
- }
31
- }
32
- } catch {
33
- // Ignore - another process may have cleaned it
34
- }
35
- }
36
- const AgentWrapper = require('./agent-wrapper');
37
- const SubClusterWrapper = require('./sub-cluster-wrapper');
38
- const MessageBus = require('./message-bus');
39
- const Ledger = require('./ledger');
40
- const InputHelpers = require('./input-helpers');
41
- const { detectProvider } = require('./issue-providers');
42
- const IsolationManager = require('./isolation-manager');
43
- const { generateName } = require('./name-generator');
44
- const configValidator = require('./config-validator');
45
- const TemplateResolver = require('./template-resolver');
46
- const { loadSettings } = require('../lib/settings');
47
- const { normalizeProviderName } = require('../lib/provider-names');
48
- const crypto = require('crypto');
49
-
50
- function applyModelOverride(agentConfig, modelOverride) {
51
- if (!modelOverride) return;
52
-
53
- agentConfig.model = modelOverride;
54
- if (agentConfig.modelRules) {
55
- delete agentConfig.modelRules;
56
- }
57
- if (agentConfig.modelConfig) {
58
- delete agentConfig.modelConfig;
59
- }
60
- }
61
-
62
- /**
63
- * Operation Chain Schema
64
- * Conductor (or any agent) can publish CLUSTER_OPERATIONS to dynamically modify cluster
65
- *
66
- * Supported operations:
67
- * - add_agents: Spawn new agents with given configs
68
- * - remove_agents: Stop and remove agents by ID
69
- * - update_agent: Modify existing agent config
70
- * - publish: Publish a message to the bus
71
- * - load_config: Load agents from a named cluster config template
72
- */
73
- const VALID_OPERATIONS = ['add_agents', 'remove_agents', 'update_agent', 'publish', 'load_config'];
74
-
75
- /**
76
- * Workflow-triggering topics that indicate cluster state progression
77
- * These are the topics that MATTER for resume - not AGENT_OUTPUT noise
78
- */
79
- const WORKFLOW_TRIGGERS = Object.freeze([
80
- 'ISSUE_OPENED',
81
- 'PLAN_READY',
82
- 'IMPLEMENTATION_READY',
83
- 'VALIDATION_RESULT',
84
- 'CONDUCTOR_ESCALATE',
85
- ]);
86
-
87
- class Orchestrator {
88
- constructor(options = {}) {
89
- this.clusters = new Map(); // cluster_id -> cluster object
90
- this.quiet = options.quiet || false; // Suppress verbose logging
91
-
92
- // TaskRunner DI - allows injecting MockTaskRunner for testing
93
- // When set, passed to all AgentWrappers to control task execution
94
- this.taskRunner = options.taskRunner || null;
95
-
96
- // Set up persistent storage directory (can be overridden for testing)
97
- this.storageDir = options.storageDir || path.join(os.homedir(), '.zeroshot');
98
- if (!fs.existsSync(this.storageDir)) {
99
- fs.mkdirSync(this.storageDir, { recursive: true });
100
- }
101
-
102
- // Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
103
- this.closed = false;
104
-
105
- // Track if clusters are loaded (for lazy loading pattern)
106
- this._clustersLoaded = options.skipLoad === true;
107
- }
108
-
109
- /**
110
- * Factory method for async initialization
111
- * Use this instead of `new Orchestrator()` for proper async cluster loading
112
- * @param {Object} options - Same options as constructor
113
- * @returns {Promise<Orchestrator>}
114
- */
115
- static async create(options = {}) {
116
- const instance = new Orchestrator({ ...options, skipLoad: true });
117
- if (options.skipLoad !== true) {
118
- await instance._loadClusters();
119
- instance._clustersLoaded = true;
120
- }
121
- return instance;
122
- }
123
-
124
- /**
125
- * Log message (respects quiet mode)
126
- * @private
127
- */
128
- _log(...args) {
129
- if (!this.quiet) {
130
- console.log(...args);
131
- }
132
- }
133
-
134
- /**
135
- * Get input source type for metadata
136
- * @param {Object} input - Input object
137
- * @returns {string} Source type: 'github', 'file', or 'text'
138
- * @private
139
- */
140
- _getInputSource(input) {
141
- if (input.issue) return 'github';
142
- if (input.file) return 'file';
143
- return 'text';
144
- }
145
-
146
- /**
147
- * Load clusters from persistent storage
148
- * Uses file locking for consistent reads
149
- * @private
150
- */
151
- async _loadClusters() {
152
- const clustersFile = path.join(this.storageDir, 'clusters.json');
153
- this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
154
-
155
- if (!fs.existsSync(clustersFile)) {
156
- this._log(`[Orchestrator] No clusters file found at ${clustersFile}`);
157
- return;
158
- }
159
-
160
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
161
- let release;
162
-
163
- try {
164
- // Clean stale locks from crashed processes
165
- cleanStaleLock(lockfilePath);
166
-
167
- // Acquire lock with async API (proper retries without CPU spin-wait)
168
- release = await lockfile.lock(clustersFile, {
169
- lockfilePath,
170
- stale: LOCK_STALE_MS,
171
- retries: {
172
- retries: 20,
173
- minTimeout: 100,
174
- maxTimeout: 200,
175
- randomize: true,
176
- },
177
- });
178
-
179
- const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
180
- const clusterIds = Object.keys(data);
181
- this._log(`[Orchestrator] Found ${clusterIds.length} clusters in file:`, clusterIds);
182
-
183
- // Track clusters to remove (missing .db files or 0 messages)
184
- const clustersToRemove = [];
185
- // Track clusters with 0 messages (corrupted from SIGINT race condition)
186
- const corruptedClusters = [];
187
-
188
- for (const [clusterId, clusterData] of Object.entries(data)) {
189
- // Skip clusters whose .db file doesn't exist (orphaned registry entries)
190
- const dbPath = path.join(this.storageDir, `${clusterId}.db`);
191
- if (!fs.existsSync(dbPath)) {
192
- console.warn(
193
- `[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
194
- );
195
- clustersToRemove.push(clusterId);
196
- continue;
197
- }
198
-
199
- this._log(`[Orchestrator] Loading cluster: ${clusterId}`);
200
- const cluster = this._loadSingleCluster(clusterId, clusterData);
201
-
202
- // VALIDATION: Detect 0-message clusters (corrupted from SIGINT during initialization)
203
- // These clusters were created before the initCompletePromise fix was applied
204
- if (cluster && cluster.messageBus) {
205
- const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
206
- if (messageCount === 0) {
207
- console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
208
- console.warn(
209
- `[Orchestrator] This likely occurred from SIGINT during initialization.`
210
- );
211
- console.warn(
212
- `[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
213
- );
214
- corruptedClusters.push(clusterId);
215
- // Mark cluster as corrupted for visibility in status/list commands
216
- cluster.state = 'corrupted';
217
- cluster.corruptedReason = 'SIGINT during initialization (0 messages in ledger)';
218
- }
219
- }
220
- }
221
-
222
- // Clean up orphaned entries from clusters.json
223
- if (clustersToRemove.length > 0) {
224
- for (const clusterId of clustersToRemove) {
225
- delete data[clusterId];
226
- }
227
- fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
228
- this._log(
229
- `[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
230
- );
231
- }
232
-
233
- // Log summary of corrupted clusters
234
- if (corruptedClusters.length > 0) {
235
- console.warn(
236
- `\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
237
- );
238
- for (const clusterId of corruptedClusters) {
239
- console.warn(` - ${clusterId}`);
240
- }
241
- console.warn(`[Orchestrator] Run 'zeroshot clear' to remove all corrupted clusters.\n`);
242
- }
243
-
244
- this._log(`[Orchestrator] Total clusters loaded: ${this.clusters.size}`);
245
- } catch (error) {
246
- console.error('[Orchestrator] Failed to load clusters:', error.message);
247
- console.error(error.stack);
248
- } finally {
249
- if (release) {
250
- await release();
251
- }
252
- }
253
- }
254
-
255
- /**
256
- * Load a single cluster from data
257
- * @private
258
- */
259
- _loadSingleCluster(clusterId, clusterData) {
260
- // Skip if already loaded
261
- if (this.clusters.has(clusterId)) {
262
- return this.clusters.get(clusterId);
263
- }
264
-
265
- // Restore ledger and message bus
266
- const dbPath = path.join(this.storageDir, `${clusterId}.db`);
267
- const ledger = new Ledger(dbPath);
268
- const messageBus = new MessageBus(ledger);
269
-
270
- // Restore isolation manager FIRST if cluster was running in isolation mode
271
- const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
272
-
273
- // Reconstruct agent metadata from config (processes are ephemeral)
274
- // CRITICAL: Pass isolation context to agents if cluster was running in isolation
275
- const agents = this._rebuildClusterAgents(
276
- clusterId,
277
- clusterData,
278
- messageBus,
279
- isolation,
280
- isolationManager
281
- );
282
-
283
- const cluster = {
284
- ...clusterData,
285
- ledger,
286
- messageBus,
287
- agents,
288
- isolation,
289
- autoPr: clusterData.autoPr || false,
290
- };
291
-
292
- this.clusters.set(clusterId, cluster);
293
- this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
294
-
295
- return cluster;
296
- }
297
-
298
- _restoreClusterIsolation(clusterId, clusterData) {
299
- let isolation = clusterData.isolation || null;
300
- let isolationManager = null;
301
-
302
- if (isolation?.enabled && isolation.containerId) {
303
- isolationManager = new IsolationManager({ image: isolation.image });
304
- // Restore the container mapping so cleanup works
305
- isolationManager.containers.set(clusterId, isolation.containerId);
306
- // Restore isolated dir mapping for workspace preservation during cleanup
307
- if (isolation.workDir) {
308
- isolationManager.isolatedDirs.set(clusterId, {
309
- path: path.join(os.tmpdir(), 'zeroshot-isolated', clusterId),
310
- originalDir: isolation.workDir,
311
- });
312
- }
313
- isolation = {
314
- ...isolation,
315
- manager: isolationManager,
316
- };
317
- this._log(
318
- `[Orchestrator] Restored isolation manager for ${clusterId} (container: ${isolation.containerId}, workDir: ${isolation.workDir || 'unknown'})`
319
- );
320
- }
321
-
322
- return { isolation, isolationManager };
323
- }
324
-
325
- _resolveAgentCwd(clusterData) {
326
- const worktreePath = clusterData.worktree?.path;
327
- const isolationWorkDir = clusterData.isolation?.workDir;
328
- return worktreePath || isolationWorkDir || null;
329
- }
330
-
331
- _buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
332
- const agentOptions = {
333
- id: clusterId,
334
- quiet: this.quiet,
335
- modelOverride: clusterData.modelOverride || null,
336
- };
337
-
338
- if (isolation?.enabled && isolationManager) {
339
- agentOptions.isolation = {
340
- enabled: true,
341
- manager: isolationManager,
342
- clusterId,
343
- };
344
- }
345
-
346
- return agentOptions;
347
- }
348
-
349
- _instantiateAgent(agentConfig, messageBus, clusterId, agentOptions) {
350
- if (agentConfig.type === 'subcluster') {
351
- return new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
352
- }
353
-
354
- return new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
355
- }
356
-
357
- _restoreAgentState(agent, agentConfig, clusterData) {
358
- if (!clusterData.agentStates) return;
359
-
360
- const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
361
- if (!savedState) return;
362
-
363
- agent.state = savedState.state || 'idle';
364
- agent.iteration = savedState.iteration || 0;
365
- agent.currentTask = savedState.currentTask || false;
366
- agent.currentTaskId = savedState.currentTaskId || null;
367
- agent.processPid = savedState.processPid || null;
368
- }
369
-
370
- _rebuildClusterAgents(clusterId, clusterData, messageBus, isolation, isolationManager) {
371
- const agents = [];
372
- const agentCwd = this._resolveAgentCwd(clusterData);
373
-
374
- if (!clusterData.config?.agents) {
375
- return agents;
376
- }
377
-
378
- for (const agentConfig of clusterData.config.agents) {
379
- if (!agentConfig.cwd && agentCwd) {
380
- agentConfig.cwd = agentCwd;
381
- this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
382
- }
383
-
384
- if (clusterData.modelOverride) {
385
- applyModelOverride(agentConfig, clusterData.modelOverride);
386
- }
387
-
388
- const agentOptions = this._buildAgentOptions(
389
- clusterId,
390
- clusterData,
391
- isolation,
392
- isolationManager
393
- );
394
- const agent = this._instantiateAgent(agentConfig, messageBus, clusterId, agentOptions);
395
- this._restoreAgentState(agent, agentConfig, clusterData);
396
- agents.push(agent);
397
- }
398
-
399
- return agents;
400
- }
401
-
402
- /**
403
- * Ensure clusters file exists (required for file locking)
404
- * @private
405
- */
406
- _ensureClustersFile() {
407
- const clustersFile = path.join(this.storageDir, 'clusters.json');
408
- if (!fs.existsSync(clustersFile)) {
409
- fs.writeFileSync(clustersFile, '{}');
410
- }
411
- return clustersFile;
412
- }
413
-
414
- /**
415
- * Save clusters to persistent storage
416
- * Uses file locking to prevent race conditions with other processes
417
- * @private
418
- */
419
- async _saveClusters() {
420
- // Skip saving if orchestrator is closed (prevents race conditions during cleanup)
421
- if (this.closed) {
422
- return;
423
- }
424
-
425
- const clustersFile = this._ensureClustersFile();
426
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
427
- let release;
428
-
429
- try {
430
- // Clean stale locks from crashed processes
431
- cleanStaleLock(lockfilePath);
432
-
433
- // Acquire exclusive lock with async API (proper retries without CPU spin-wait)
434
- release = await lockfile.lock(clustersFile, {
435
- lockfilePath,
436
- stale: LOCK_STALE_MS,
437
- retries: {
438
- retries: 50,
439
- minTimeout: 100,
440
- maxTimeout: 300,
441
- randomize: true,
442
- },
443
- });
444
-
445
- // Read existing clusters from file (other processes may have added clusters)
446
- let existingClusters = {};
447
- try {
448
- const content = fs.readFileSync(clustersFile, 'utf8');
449
- existingClusters = JSON.parse(content);
450
- } catch (error) {
451
- console.error('[Orchestrator] Failed to read existing clusters:', error.message);
452
- }
453
-
454
- // Merge: update/add clusters from this process
455
- for (const [clusterId, cluster] of this.clusters.entries()) {
456
- // CRITICAL: Only update clusters this process actually owns or has modified
457
- // A process owns a cluster if: it started it (pid matches) OR it explicitly stopped/killed it
458
- const isOwnedByThisProcess = cluster.pid === process.pid;
459
- const wasModifiedByThisProcess = cluster.state === 'stopped' || cluster.state === 'killed';
460
-
461
- // Skip clusters we don't own and haven't modified - prevents race condition
462
- // where a running cluster overwrites another process's stop/kill operation
463
- if (!isOwnedByThisProcess && !wasModifiedByThisProcess) {
464
- // Preserve existing state from file for clusters we don't own
465
- continue;
466
- }
467
-
468
- // CRITICAL: Killed clusters are DELETED from disk, not persisted
469
- // This ensures they can't be accidentally resumed
470
- if (cluster.state === 'killed') {
471
- delete existingClusters[clusterId];
472
- continue;
473
- }
474
-
475
- existingClusters[clusterId] = {
476
- id: cluster.id,
477
- config: cluster.config,
478
- state: cluster.state,
479
- createdAt: cluster.createdAt,
480
- // Track PID for zombie detection (null if cluster is stopped/killed)
481
- pid: cluster.state === 'running' ? cluster.pid : null,
482
- // Persist failure info for resume capability
483
- failureInfo: cluster.failureInfo || null,
484
- // Persist PR mode for completion agent selection
485
- autoPr: cluster.autoPr || false,
486
- // Persist model override for consistent agent spawning on resume
487
- modelOverride: cluster.modelOverride || null,
488
- // Persist isolation info (excluding manager instance which can't be serialized)
489
- // CRITICAL: workDir is required for resume() to recreate container with same workspace
490
- isolation: cluster.isolation
491
- ? {
492
- enabled: cluster.isolation.enabled,
493
- containerId: cluster.isolation.containerId,
494
- image: cluster.isolation.image,
495
- workDir: cluster.isolation.workDir, // Required for resume
496
- }
497
- : null,
498
- // Persist agent runtime states for accurate status display from other processes
499
- agentStates: cluster.agents
500
- ? cluster.agents.map((a) => ({
501
- id: a.id,
502
- state: a.state,
503
- iteration: a.iteration,
504
- currentTask: a.currentTask ? true : false,
505
- currentTaskId: a.currentTaskId,
506
- processPid: a.processPid,
507
- }))
508
- : null,
509
- };
510
- }
511
-
512
- // Write merged data
513
- fs.writeFileSync(clustersFile, JSON.stringify(existingClusters, null, 2));
514
- this._log(
515
- `[Orchestrator] Saved ${this.clusters.size} cluster(s), file now has ${Object.keys(existingClusters).length} total`
516
- );
517
- } finally {
518
- // Always release lock
519
- if (release) {
520
- await release();
521
- }
522
- }
523
- }
524
-
525
- /**
526
- * Watch for new clusters and call callback when found
527
- * Polls the clusters file for changes with file locking
528
- * @param {Function} onNewCluster - Callback(cluster) for each new cluster
529
- * @param {Number} intervalMs - Poll interval in ms (default: 2000)
530
- * @returns {Function} Stop function to cancel watching
531
- */
532
- watchForNewClusters(onNewCluster, intervalMs = 2000) {
533
- const clustersFile = path.join(this.storageDir, 'clusters.json');
534
- const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
535
- const knownClusterIds = new Set(this.clusters.keys());
536
-
537
- const intervalId = setInterval(() => {
538
- let release;
539
- try {
540
- if (!fs.existsSync(clustersFile)) return;
541
-
542
- // Clean stale locks from crashed processes
543
- cleanStaleLock(lockfilePath);
544
-
545
- // Try to acquire lock once (polling is best-effort, will retry on next cycle)
546
- try {
547
- release = lockfile.lockSync(clustersFile, {
548
- lockfilePath,
549
- stale: LOCK_STALE_MS,
550
- });
551
- } catch (lockErr) {
552
- // Lock busy - skip this poll cycle, try again next interval
553
- if (lockErr.code === 'ELOCKED') return;
554
- throw lockErr;
555
- }
556
-
557
- const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
558
-
559
- for (const [clusterId, clusterData] of Object.entries(data)) {
560
- if (!knownClusterIds.has(clusterId)) {
561
- // New cluster found
562
- knownClusterIds.add(clusterId);
563
- const cluster = this._loadSingleCluster(clusterId, clusterData);
564
- if (cluster && onNewCluster) {
565
- onNewCluster(cluster);
566
- }
567
- }
568
- }
569
- } catch (error) {
570
- // File access during polling can fail transiently - log and continue
571
- console.error(`[Orchestrator] watchForNewClusters error (will retry): ${error.message}`);
572
- } finally {
573
- if (release) {
574
- release();
575
- }
576
- }
577
- }, intervalMs);
578
-
579
- // Return stop function
580
- return () => clearInterval(intervalId);
581
- }
582
-
583
- /**
584
- * Start a new cluster with mocked agent executors (TESTING ONLY)
585
- *
586
- * CRITICAL: This method PREVENTS real Claude API calls.
587
- * All agent behaviors must be defined in mockExecutor.
588
- *
589
- * @param {Object} config - Cluster configuration
590
- * @param {Object} input - Input source { issue, text, or config }
591
- * @param {MockAgentExecutor} mockExecutor - Mock executor with agent behaviors
592
- * @returns {Object} Cluster object
593
- */
594
- startWithMock(config, input, mockExecutor) {
595
- if (!mockExecutor) {
596
- throw new Error('Orchestrator.startWithMock: mockExecutor is required');
597
- }
598
-
599
- // Validate all agents that execute tasks have mock behaviors defined
600
- // Orchestrator agents (action: 'stop_cluster') don't execute tasks, so don't need mocks
601
- for (const agentConfig of config.agents) {
602
- const agentId = agentConfig.id;
603
-
604
- // Check if agent has any triggers that execute tasks
605
- const executesTask = agentConfig.triggers?.some(
606
- (trigger) => !trigger.action || trigger.action === 'execute_task'
607
- );
608
-
609
- if (executesTask && !mockExecutor.behaviors[agentId]) {
610
- throw new Error(
611
- `Orchestrator.startWithMock: No behavior defined for agent '${agentId}'. ` +
612
- `This would cause real Claude API calls. ABORTING.\n` +
613
- `Available behaviors: ${Object.keys(mockExecutor.behaviors).join(', ')}`
614
- );
615
- }
616
- }
617
-
618
- return this._startInternal(config, input, {
619
- mockExecutor,
620
- testMode: true,
621
- });
622
- }
623
-
624
- /**
625
- * Start a new cluster
626
- * @param {Object} config - Cluster configuration
627
- * @param {Object} input - Input source { issue, text, or config }
628
- * @param {Object} options - Start options
629
- * @param {boolean} options.isolation - Run in Docker container
630
- * @param {string} options.isolationImage - Docker image to use
631
- * @param {boolean} options.worktree - Run in git worktree isolation (lightweight, no Docker)
632
- * @returns {Object} Cluster object
633
- */
634
- start(config, input = {}, options = {}) {
635
- return this._startInternal(config, input, {
636
- testMode: false,
637
- cwd: options.cwd || process.cwd(), // Target working directory for agents
638
- isolation: options.isolation || false,
639
- isolationImage: options.isolationImage,
640
- worktree: options.worktree || false,
641
- autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
642
- modelOverride: options.modelOverride, // Model override for all agents
643
- clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
644
- settings: options.settings, // User settings for issue provider detection
645
- forceProvider: options.forceProvider, // Force specific issue provider
646
- });
647
- }
648
-
649
- /**
650
- * Internal start implementation (shared by start and startWithMock)
651
- * @private
652
- */
653
- async _startInternal(config, input = {}, options = {}) {
654
- // Generate a unique cluster ID for this process call.
655
- // IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
656
- // - test harnesses may set it globally (breaking multi-start tests)
657
- // - callers may start multiple clusters in one process
658
- // Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
659
- const clusterId = this._generateUniqueClusterId(
660
- options.clusterId || null,
661
- config?.dbPath || null
662
- );
663
-
664
- // Create ledger and message bus with persistent storage
665
- const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
666
- const ledger = new Ledger(dbPath);
667
- const messageBus = new MessageBus(ledger);
668
-
669
- // Handle isolation mode (Docker container OR git worktree)
670
- const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
671
- options,
672
- config,
673
- clusterId
674
- );
675
-
676
- // Build cluster object
677
- // CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
678
- // This prevents 0-message clusters from SIGINT during async initialization
679
- let resolveInitComplete;
680
- const initCompletePromise = new Promise((resolve) => {
681
- resolveInitComplete = resolve;
682
- });
683
-
684
- const cluster = {
685
- id: clusterId,
686
- config,
687
- state: 'initializing',
688
- messageBus,
689
- ledger,
690
- agents: [],
691
- createdAt: Date.now(),
692
- // Track PID for zombie detection (this process owns the cluster)
693
- pid: process.pid,
694
- // Initialization completion tracking (for safe SIGINT handling)
695
- initCompletePromise,
696
- _resolveInitComplete: resolveInitComplete,
697
- autoPr: options.autoPr || false,
698
- // Model override for all agents (applied to dynamically added agents)
699
- modelOverride: options.modelOverride || null,
700
- // Issue provider tracking (where issue was fetched from)
701
- issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
702
- // Git platform tracking (where PR/MR will be created - independent of issue provider)
703
- gitPlatform: null, // Set when --pr mode is active
704
- // Isolation state (only if enabled)
705
- // CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
706
- isolation: options.isolation
707
- ? {
708
- enabled: true,
709
- containerId,
710
- image: options.isolationImage || 'zeroshot-cluster-base',
711
- manager: isolationManager,
712
- workDir: options.cwd || process.cwd(), // Persisted for resume
713
- }
714
- : null,
715
- // Worktree isolation state (lightweight alternative to Docker)
716
- worktree: options.worktree
717
- ? {
718
- enabled: true,
719
- path: worktreeInfo.path,
720
- branch: worktreeInfo.branch,
721
- repoRoot: worktreeInfo.repoRoot,
722
- manager: isolationManager,
723
- workDir: options.cwd || process.cwd(), // Persisted for resume
724
- }
725
- : null,
726
- };
727
-
728
- this.clusters.set(clusterId, cluster);
729
-
730
- try {
731
- // Fetch input (issue from provider, file, or text)
732
- let inputData;
733
- if (input.issue) {
734
- // Detect provider and fetch issue
735
- const ProviderClass = detectProvider(
736
- input.issue,
737
- options.settings || {},
738
- options.forceProvider
739
- );
740
- if (!ProviderClass) {
741
- throw new Error(`No issue provider matched input: ${input.issue}`);
742
- }
743
-
744
- const provider = new ProviderClass();
745
- inputData = await provider.fetchIssue(input.issue, options.settings || {});
746
-
747
- // Store issue provider for logging/debugging and cross-provider workflows
748
- cluster.issueProvider = ProviderClass.id;
749
-
750
- // Log clickable issue link
751
- if (inputData.url) {
752
- this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
753
- }
754
- } else if (input.file) {
755
- inputData = InputHelpers.createFileInput(input.file);
756
- this._log(`[Orchestrator] File: ${input.file}`);
757
- } else if (input.text) {
758
- inputData = InputHelpers.createTextInput(input.text);
759
- } else {
760
- throw new Error('Either issue, file, or text input is required');
761
- }
762
-
763
- // Detect git platform for --pr mode (independent of issue provider)
764
- if (options.autoPr) {
765
- const { detectGitContext } = require('../lib/git-remote-utils');
766
- const gitContext = detectGitContext(options.cwd);
767
- cluster.gitPlatform = gitContext?.provider || null;
768
-
769
- if (cluster.gitPlatform) {
770
- this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
771
- }
772
- }
773
-
774
- // Inject git-pusher agent if --pr is set (replaces completion-detector)
775
- this._applyAutoPrConfig(config, inputData, options);
776
-
777
- // Inject workers instruction if --workers explicitly provided and > 1
778
- this._applyWorkerInstruction(config);
779
-
780
- // Initialize agents with optional mock injection
781
- // Check agent type: regular agent or subcluster
782
- // CRITICAL: Inject cwd into each agent config for proper working directory
783
- // In worktree mode, agents run in the worktree path (not original cwd)
784
- this._initializeClusterAgents({
785
- config,
786
- cluster,
787
- messageBus,
788
- options,
789
- isolationManager,
790
- clusterId,
791
- });
792
-
793
- // ========================================================================
794
- // CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
795
- // ========================================================================
796
- // MUST register ALL subscriptions BEFORE starting agents.
797
- //
798
- // WHY: EventEmitter is synchronous and doesn't replay past events to new
799
- // listeners. If an agent completes and publishes CLUSTER_COMPLETE before
800
- // we register the subscription, the orchestrator will never receive the
801
- // message and the cluster will appear stuck in 'executing_task' forever.
802
- //
803
- // ORDER:
804
- // 1. Register subscriptions (lines below)
805
- // 2. Start agents
806
- // 3. Publish ISSUE_OPENED
807
- //
808
- // DO NOT move subscriptions after agent.start() - this will reintroduce
809
- // the race condition fixed in issue #31.
810
- // ========================================================================
811
- this._registerClusterSubscriptions({
812
- messageBus,
813
- clusterId,
814
- isolationManager,
815
- containerId,
816
- });
817
-
818
- // Start all agents
819
- for (const agent of cluster.agents) {
820
- await agent.start();
821
- }
822
-
823
- cluster.state = 'running';
824
-
825
- // Publish ISSUE_OPENED message to bootstrap workflow
826
- messageBus.publish({
827
- cluster_id: clusterId,
828
- topic: 'ISSUE_OPENED',
829
- sender: 'system',
830
- receiver: 'broadcast',
831
- content: {
832
- text: inputData.context,
833
- data: {
834
- issue_number: inputData.number,
835
- title: inputData.title,
836
- },
837
- },
838
- metadata: {
839
- source: this._getInputSource(input),
840
- },
841
- });
842
-
843
- // CRITICAL: Mark initialization complete AFTER ISSUE_OPENED is published
844
- // This ensures stop() waits for at least 1 message before stopping
845
- if (cluster._resolveInitComplete) {
846
- cluster._resolveInitComplete();
847
- }
848
-
849
- this._log(`Cluster ${clusterId} started with ${cluster.agents.length} agents`);
850
-
851
- // DISABLED: Idle timeout auto-stop mechanism
852
- // WHY DISABLED: Clusters should only stop on explicit signals:
853
- // - User `zeroshot kill` command
854
- // - CLUSTER_COMPLETE message (successful completion)
855
- // - CLUSTER_FAILED message (failure/abort)
856
- // Being "idle" is NOT a reason to auto-stop - agents may be legitimately
857
- // waiting for external events, user input (in interactive mode), or
858
- // processing that doesn't show as "executing" (e.g., polling, monitoring).
859
- //
860
- // Previous behavior: Stopped cluster after 2 minutes of all agents idle
861
- // Result: Clusters were killed while legitimately waiting, causing confusion
862
- //
863
- // cluster.idleCheckInterval = setInterval(() => { ... }, 30000);
864
- // ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
865
-
866
- // Save cluster to disk
867
- await this._saveClusters();
868
-
869
- return {
870
- id: clusterId,
871
- state: cluster.state,
872
- agents: cluster.agents.map((a) => a.getState()),
873
- ledger: cluster.ledger, // Expose ledger for testing
874
- messageBus: cluster.messageBus, // Expose messageBus for testing
875
- };
876
- } catch (error) {
877
- cluster.state = 'failed';
878
- // CRITICAL: Resolve the promise on failure too, so stop() doesn't hang
879
- if (cluster._resolveInitComplete) {
880
- cluster._resolveInitComplete();
881
- }
882
- console.error(`Cluster ${clusterId} failed to start:`, error);
883
- throw error;
884
- }
885
- }
886
-
887
- _initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
888
- const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
889
-
890
- for (const agentConfig of config.agents) {
891
- if (!agentConfig.cwd) {
892
- agentConfig.cwd = agentCwd;
893
- }
894
-
895
- if (options.modelOverride) {
896
- applyModelOverride(agentConfig, options.modelOverride);
897
- this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
898
- }
899
-
900
- const agentOptions = {
901
- testMode: options.testMode || !!this.taskRunner,
902
- quiet: this.quiet,
903
- modelOverride: options.modelOverride || null,
904
- };
905
-
906
- if (options.mockExecutor) {
907
- agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
908
- }
909
-
910
- if (this.taskRunner) {
911
- agentOptions.taskRunner = this.taskRunner;
912
- }
913
-
914
- if (cluster.isolation) {
915
- agentOptions.isolation = {
916
- enabled: true,
917
- manager: isolationManager,
918
- clusterId,
919
- };
920
- }
921
-
922
- if (cluster.worktree) {
923
- agentOptions.worktree = {
924
- enabled: true,
925
- path: cluster.worktree.path,
926
- branch: cluster.worktree.branch,
927
- repoRoot: cluster.worktree.repoRoot,
928
- };
929
- }
930
-
931
- const agent =
932
- agentConfig.type === 'subcluster'
933
- ? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
934
- : new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
935
-
936
- cluster.agents.push(agent);
937
- }
938
- }
939
-
940
- _subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
941
- messageBus.subscribe((message) => {
942
- if (message.topic === topic && message.cluster_id === clusterId) {
943
- handler(message);
944
- }
945
- });
946
- }
947
-
948
- _registerClusterCompletionHandlers(messageBus, clusterId) {
949
- this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
950
- this._log(`\n${'='.repeat(80)}`);
951
- this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
952
- this._log(`${'='.repeat(80)}`);
953
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
954
- this._log(`Initiated by: ${message.sender}`);
955
- this._log(`${'='.repeat(80)}\n`);
956
-
957
- this.stop(clusterId).catch((err) => {
958
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
959
- });
960
- });
961
-
962
- this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
963
- this._log(`\n${'='.repeat(80)}`);
964
- this._log(`❌ CLUSTER FAILED: ${clusterId}`);
965
- this._log(`${'='.repeat(80)}`);
966
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
967
- this._log(`Agent: ${message.sender}`);
968
- if (message.content?.text) {
969
- this._log(`Details: ${message.content.text}`);
970
- }
971
- this._log(`${'='.repeat(80)}\n`);
972
-
973
- this.stop(clusterId).catch((err) => {
974
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
975
- });
976
- });
977
- }
978
-
979
- _registerAgentErrorHandler(messageBus, clusterId) {
980
- this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
981
- const agentRole = message.content?.data?.role;
982
- const attempts = message.content?.data?.attempts || 1;
983
-
984
- await this._saveClusters();
985
-
986
- if (agentRole === 'implementation' && attempts >= 3) {
987
- this._log(`\n${'='.repeat(80)}`);
988
- this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
989
- this._log(`${'='.repeat(80)}`);
990
- this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
991
- this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
992
- this._log(`Stopping cluster - worker cannot continue`);
993
- this._log(`${'='.repeat(80)}\n`);
994
-
995
- this.stop(clusterId).catch((err) => {
996
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
997
- });
998
- }
999
- });
1000
- }
1001
-
1002
- _registerAgentLifecycleHandlers(messageBus, clusterId) {
1003
- messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
1004
- const event = message.content?.data?.event;
1005
- if (
1006
- [
1007
- 'TASK_STARTED',
1008
- 'TASK_COMPLETED',
1009
- 'PROCESS_SPAWNED',
1010
- 'TASK_ID_ASSIGNED',
1011
- 'STARTED',
1012
- ].includes(event)
1013
- ) {
1014
- await this._saveClusters();
1015
- }
1016
- });
1017
-
1018
- messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
1019
- if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
1020
-
1021
- const agentId = message.content?.data?.agent;
1022
- const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
1023
- const analysis = message.content?.data?.analysis || 'No analysis available';
1024
-
1025
- this._log(
1026
- `⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
1027
- );
1028
- this._log(` Analysis: ${analysis}`);
1029
- this._log(
1030
- ` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
1031
- );
1032
- });
1033
- }
1034
-
1035
- _registerConductorWatchdog(messageBus, clusterId) {
1036
- const timeoutMs = 30000;
1037
- let watchdogTimer = null;
1038
- let completedAt = null;
1039
-
1040
- this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
1041
- const event = message.content?.data?.event;
1042
- const role = message.content?.data?.role;
1043
-
1044
- if (event === 'TASK_COMPLETED' && role === 'conductor') {
1045
- completedAt = Date.now();
1046
- this._log(
1047
- `⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
1048
- );
1049
-
1050
- watchdogTimer = setTimeout(() => {
1051
- const clusterOps = messageBus.query({
1052
- cluster_id: clusterId,
1053
- topic: 'CLUSTER_OPERATIONS',
1054
- limit: 1,
1055
- });
1056
- if (clusterOps.length === 0) {
1057
- console.error(`\n${'='.repeat(80)}`);
1058
- console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
1059
- console.error(`${'='.repeat(80)}`);
1060
- console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
1061
- console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
1062
- console.error(
1063
- `Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
1064
- );
1065
- console.error(`${'='.repeat(80)}\n`);
1066
-
1067
- messageBus.publish({
1068
- cluster_id: clusterId,
1069
- topic: 'CLUSTER_FAILED',
1070
- sender: 'orchestrator',
1071
- content: {
1072
- text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
1073
- data: {
1074
- reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
1075
- conductorCompletedAt: completedAt,
1076
- timeoutMs: timeoutMs,
1077
- },
1078
- },
1079
- });
1080
- }
1081
- }, timeoutMs);
1082
- }
1083
- });
1084
-
1085
- return {
1086
- clear: () => {
1087
- if (!watchdogTimer) {
1088
- return;
1089
- }
1090
- clearTimeout(watchdogTimer);
1091
- watchdogTimer = null;
1092
- const elapsed = completedAt ? Date.now() - completedAt : 0;
1093
- this._log(
1094
- `✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
1095
- );
1096
- },
1097
- };
1098
- }
1099
-
1100
- _registerClusterOperationsHandler(
1101
- messageBus,
1102
- clusterId,
1103
- isolationManager,
1104
- containerId,
1105
- watchdog
1106
- ) {
1107
- this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
1108
- watchdog?.clear?.();
1109
-
1110
- let operations = message.content?.data?.operations;
1111
- if (typeof operations === 'string') {
1112
- try {
1113
- operations = JSON.parse(operations);
1114
- } catch (e) {
1115
- this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
1116
- return;
1117
- }
1118
- }
1119
-
1120
- if (!operations || !Array.isArray(operations)) {
1121
- this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
1122
- return;
1123
- }
1124
-
1125
- this._log(`\n${'='.repeat(80)}`);
1126
- this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
1127
- this._log(`${'='.repeat(80)}`);
1128
- if (message.content?.data?.reasoning) {
1129
- this._log(`Reasoning: ${message.content.data.reasoning}`);
1130
- }
1131
- this._log(`Operations: ${operations.length}`);
1132
- this._log(`${'='.repeat(80)}\n`);
1133
-
1134
- this._handleOperations(clusterId, operations, message.sender, {
1135
- isolationManager,
1136
- containerId,
1137
- }).catch((err) => {
1138
- console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
1139
- messageBus.publish({
1140
- cluster_id: clusterId,
1141
- topic: 'CLUSTER_OPERATIONS_FAILED',
1142
- sender: 'orchestrator',
1143
- content: {
1144
- text: `Operation chain failed: ${err.message}`,
1145
- data: {
1146
- error: err.message,
1147
- operations: operations,
1148
- },
1149
- },
1150
- });
1151
-
1152
- this._log(`\n${'='.repeat(80)}`);
1153
- this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
1154
- this._log(`${'='.repeat(80)}`);
1155
- this._log(`Error: ${err.message}`);
1156
- this._log(`${'='.repeat(80)}\n`);
1157
-
1158
- this.stop(clusterId).catch((stopErr) => {
1159
- console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
1160
- });
1161
- });
1162
- });
1163
- }
1164
-
1165
- _registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
1166
- this._registerClusterCompletionHandlers(messageBus, clusterId);
1167
- this._registerAgentErrorHandler(messageBus, clusterId);
1168
- this._registerAgentLifecycleHandlers(messageBus, clusterId);
1169
-
1170
- const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
1171
- this._registerClusterOperationsHandler(
1172
- messageBus,
1173
- clusterId,
1174
- isolationManager,
1175
- containerId,
1176
- watchdog
1177
- );
1178
- }
1179
-
1180
- async _initializeIsolation(options, config, clusterId) {
1181
- let isolationManager = null;
1182
- let containerId = null;
1183
- let worktreeInfo = null;
1184
-
1185
- if (options.isolation) {
1186
- if (!IsolationManager.isDockerAvailable()) {
1187
- throw new Error('Docker is not available. Install Docker to use --docker mode.');
1188
- }
1189
-
1190
- const image = options.isolationImage || 'zeroshot-cluster-base';
1191
- await IsolationManager.ensureImage(image);
1192
-
1193
- isolationManager = new IsolationManager({ image });
1194
- this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
1195
-
1196
- const workDir = options.cwd || process.cwd();
1197
- const providerName = normalizeProviderName(
1198
- config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
1199
- );
1200
- containerId = await isolationManager.createContainer(clusterId, {
1201
- workDir,
1202
- image,
1203
- noMounts: options.noMounts,
1204
- mounts: options.mounts,
1205
- containerHome: options.containerHome,
1206
- provider: providerName,
1207
- });
1208
- this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
1209
- } else if (options.worktree) {
1210
- const workDir = options.cwd || process.cwd();
1211
-
1212
- isolationManager = new IsolationManager({});
1213
- worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
1214
-
1215
- this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
1216
- this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
1217
- this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
1218
- }
1219
-
1220
- return { isolationManager, containerId, worktreeInfo };
1221
- }
1222
-
1223
- _applyAutoPrConfig(config, inputData, options) {
1224
- if (!options.autoPr) {
1225
- return;
1226
- }
1227
-
1228
- config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
1229
-
1230
- // Detect git platform (independent of issue provider)
1231
- const { getPlatformForPR } = require('./issue-providers');
1232
- const { detectGitContext } = require('../lib/git-remote-utils');
1233
-
1234
- let platform;
1235
- try {
1236
- // ALWAYS use git context, regardless of issue provider
1237
- // This allows Jira issues in GitHub repos, etc.
1238
- platform = getPlatformForPR(options.cwd);
1239
- } catch (error) {
1240
- throw new Error(`--pr mode failed: ${error.message}`);
1241
- }
1242
-
1243
- // Check for repo mismatch (issue URL repo vs current git remote)
1244
- // This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
1245
- // from a different repo directory
1246
- let skipCloseIssue = false;
1247
- const gitContext = detectGitContext(options.cwd);
1248
- if (inputData.url && gitContext) {
1249
- const issueHost = this._extractHostFromUrl(inputData.url);
1250
- const gitHost = gitContext.host;
1251
-
1252
- // Compare hosts - if different platforms, warn and don't close issue
1253
- if (issueHost && gitHost && issueHost !== gitHost) {
1254
- this._log(
1255
- `[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
1256
- );
1257
- this._log(
1258
- `[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
1259
- );
1260
- this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
1261
- skipCloseIssue = true;
1262
- }
1263
- }
1264
-
1265
- // Generate platform-specific git-pusher agent from template
1266
- const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
1267
-
1268
- if (!isPlatformSupported(platform)) {
1269
- throw new Error(
1270
- `Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
1271
- );
1272
- }
1273
-
1274
- const gitPusherConfig = generateGitPusherAgent(platform);
1275
-
1276
- // Template replacement for issue context
1277
- const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
1278
- gitPusherConfig.prompt = gitPusherConfig.prompt
1279
- .replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
1280
- .replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
1281
- .replace(/Closes #\{\{issue_number\}\}/g, issueRef);
1282
-
1283
- config.agents.push(gitPusherConfig);
1284
- this._log(
1285
- `[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
1286
- );
1287
- }
1288
-
1289
- /**
1290
- * Extract hostname from a URL
1291
- * @private
1292
- */
1293
- _extractHostFromUrl(url) {
1294
- try {
1295
- const match = url.match(/https?:\/\/([^/]+)/);
1296
- return match ? match[1] : null;
1297
- } catch {
1298
- return null;
1299
- }
1300
- }
1301
-
1302
- _applyWorkerInstruction(config) {
1303
- const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
1304
- if (workersCount <= 1) {
1305
- return;
1306
- }
1307
-
1308
- const workerAgent = config.agents.find((a) => a.id === 'worker');
1309
- if (!workerAgent) {
1310
- return;
1311
- }
1312
-
1313
- const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
1314
-
1315
- if (!workerAgent.prompt) {
1316
- workerAgent.prompt = instruction;
1317
- } else if (typeof workerAgent.prompt === 'string') {
1318
- workerAgent.prompt = instruction + workerAgent.prompt;
1319
- } else if (workerAgent.prompt.system) {
1320
- workerAgent.prompt.system = instruction + workerAgent.prompt.system;
1321
- }
1322
-
1323
- this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
1324
- }
1325
-
1326
- /**
1327
- * Generate a unique cluster ID, safe for concurrent starts in-process.
1328
- * If an explicit ID is provided, uses it as a base and suffixes on collision.
1329
- * @private
1330
- */
1331
- _generateUniqueClusterId(explicitId, explicitDbPath) {
1332
- const baseId = explicitId || generateName('cluster');
1333
- const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
1334
-
1335
- // Fast path: base is unused.
1336
- if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
1337
- return baseId;
1338
- }
1339
-
1340
- // Collision: suffix with random bytes to avoid race conditions under concurrency.
1341
- for (let attempt = 0; attempt < 50; attempt++) {
1342
- const suffix = crypto.randomBytes(3).toString('hex');
1343
- const candidateId = `${baseId}-${suffix}`;
1344
- const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1345
- if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1346
- return candidateId;
1347
- }
1348
- }
1349
-
1350
- // Last resort: new generated name (should never happen).
1351
- for (let attempt = 0; attempt < 50; attempt++) {
1352
- const candidateId = generateName('cluster');
1353
- const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1354
- if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1355
- return candidateId;
1356
- }
1357
- }
1358
-
1359
- throw new Error('Failed to generate unique cluster ID after many attempts');
1360
- }
1361
-
1362
- /**
1363
- * Stop a cluster
1364
- * @param {String} clusterId - Cluster ID
1365
- */
1366
- async stop(clusterId) {
1367
- const cluster = this.clusters.get(clusterId);
1368
- if (!cluster) {
1369
- throw new Error(`Cluster ${clusterId} not found`);
1370
- }
1371
-
1372
- // CRITICAL: Wait for initialization to complete before stopping
1373
- // This ensures ISSUE_OPENED is published, preventing 0-message clusters
1374
- // Timeout after 30s to prevent infinite hang if init truly fails
1375
- if (cluster.initCompletePromise && cluster.state === 'initializing') {
1376
- this._log(`[Orchestrator] Waiting for initialization to complete before stopping...`);
1377
- await Promise.race([
1378
- cluster.initCompletePromise,
1379
- new Promise((resolve) => setTimeout(resolve, 30000)),
1380
- ]);
1381
- }
1382
-
1383
- cluster.state = 'stopping';
1384
-
1385
- // Stop all agents (including subclusters which handle their own children)
1386
- for (const agent of cluster.agents) {
1387
- await agent.stop();
1388
- }
1389
-
1390
- // Clean up isolation container if enabled
1391
- // CRITICAL: Preserve workspace for resume capability - only delete on kill()
1392
- if (cluster.isolation?.manager) {
1393
- this._log(
1394
- `[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
1395
- );
1396
- await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
1397
- this._log(`[Orchestrator] Container stopped, workspace preserved`);
1398
- }
1399
-
1400
- // Worktree cleanup on stop: preserve for resume capability
1401
- // Branch stays, worktree stays - can resume work later
1402
- if (cluster.worktree?.manager) {
1403
- this._log(`[Orchestrator] Worktree preserved at ${cluster.worktree.path} for resume`);
1404
- this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1405
- // Don't cleanup worktree - it will be reused on resume
1406
- }
1407
-
1408
- cluster.state = 'stopped';
1409
- cluster.pid = null; // Clear PID - cluster is no longer running
1410
- this._log(`Cluster ${clusterId} stopped`);
1411
-
1412
- // Save updated state
1413
- await this._saveClusters();
1414
- }
1415
-
1416
- /**
1417
- * Kill a cluster (force stop)
1418
- * @param {String} clusterId - Cluster ID
1419
- */
1420
- async kill(clusterId) {
1421
- const cluster = this.clusters.get(clusterId);
1422
- if (!cluster) {
1423
- throw new Error(`Cluster ${clusterId} not found`);
1424
- }
1425
-
1426
- cluster.state = 'stopping';
1427
-
1428
- // Force stop all agents
1429
- for (const agent of cluster.agents) {
1430
- await agent.stop();
1431
- }
1432
-
1433
- // Force remove isolation container AND workspace (full cleanup, no resume)
1434
- if (cluster.isolation?.manager) {
1435
- this._log(
1436
- `[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
1437
- );
1438
- await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
1439
- this._log(`[Orchestrator] Container and workspace removed`);
1440
- }
1441
-
1442
- // Force remove worktree (full cleanup, no resume)
1443
- // Note: Branch is preserved for potential PR creation / inspection
1444
- if (cluster.worktree?.manager) {
1445
- this._log(`[Orchestrator] Force removing worktree for ${clusterId}...`);
1446
- cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
1447
- this._log(`[Orchestrator] Worktree removed, branch ${cluster.worktree.branch} preserved`);
1448
- }
1449
-
1450
- // Close message bus and ledger
1451
- cluster.messageBus.close();
1452
-
1453
- cluster.state = 'killed';
1454
- cluster.pid = null; // Clear PID - cluster is no longer running
1455
- // DON'T delete from memory - keep it so it gets saved with 'killed' state
1456
- // this.clusters.delete(clusterId);
1457
-
1458
- this._log(`Cluster ${clusterId} killed`);
1459
-
1460
- // Save updated state (will be marked as 'killed' in file)
1461
- await this._saveClusters();
1462
-
1463
- // Now remove from memory after persisting
1464
- this.clusters.delete(clusterId);
1465
- }
1466
-
1467
- /**
1468
- * Kill all running clusters
1469
- * @returns {Object} { killed: Array<string>, errors: Array<{id, error}> }
1470
- */
1471
- async killAll() {
1472
- const results = { killed: [], errors: [] };
1473
- const runningClusters = Array.from(this.clusters.values()).filter(
1474
- (c) => c.state === 'running' || c.state === 'initializing'
1475
- );
1476
-
1477
- for (const cluster of runningClusters) {
1478
- try {
1479
- await this.kill(cluster.id);
1480
- results.killed.push(cluster.id);
1481
- } catch (error) {
1482
- results.errors.push({ id: cluster.id, error: error.message });
1483
- }
1484
- }
1485
-
1486
- return results;
1487
- }
1488
-
1489
- /**
1490
- * Close the orchestrator (prevents further _saveClusters operations)
1491
- * Call before deleting storageDir to prevent ENOENT race conditions during cleanup
1492
- */
1493
- close() {
1494
- this.closed = true;
1495
- }
1496
-
1497
- /**
1498
- * Find the last workflow-triggering message in the ledger
1499
- * Workflow triggers indicate cluster state progression (not AGENT_OUTPUT noise)
1500
- * @param {Array} messages - All messages from ledger
1501
- * @returns {Object|null} - Last workflow trigger message or null
1502
- * @private
1503
- */
1504
- _findLastWorkflowTrigger(messages) {
1505
- for (let i = messages.length - 1; i >= 0; i--) {
1506
- if (WORKFLOW_TRIGGERS.includes(messages[i].topic)) {
1507
- return messages[i];
1508
- }
1509
- }
1510
- return null;
1511
- }
1512
-
1513
- /**
1514
- * Resume a stopped cluster from where it left off
1515
- * Handles both failed clusters (with error context) and cleanly stopped clusters
1516
- * @param {String} clusterId - Cluster ID
1517
- * @param {String} prompt - Optional custom resume prompt
1518
- * @returns {Object} Resumed cluster info
1519
- */
1520
- async resume(clusterId, prompt) {
1521
- const cluster = this.clusters.get(clusterId);
1522
- if (!cluster) {
1523
- throw new Error(`Cluster not found: ${clusterId}`);
1524
- }
1525
-
1526
- if (cluster.state === 'running') {
1527
- throw new Error(
1528
- `Cluster ${clusterId} is still running. Use 'zeroshot stop' first if you want to restart it.`
1529
- );
1530
- }
1531
-
1532
- const failureInfo = this._resolveFailureInfo(cluster, clusterId);
1533
-
1534
- await this._ensureIsolationForResume(clusterId, cluster);
1535
- this._ensureWorktreeForResume(clusterId, cluster);
1536
- await this._restartClusterAgents(cluster);
1537
-
1538
- const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
1539
-
1540
- if (failureInfo) {
1541
- return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
1542
- }
1543
-
1544
- return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
1545
- }
1546
-
1547
- _resolveFailureInfo(cluster, clusterId) {
1548
- if (cluster.failureInfo) {
1549
- return cluster.failureInfo;
1550
- }
1551
-
1552
- const errors = cluster.messageBus.query({
1553
- cluster_id: clusterId,
1554
- topic: 'AGENT_ERROR',
1555
- limit: 10,
1556
- order: 'desc',
1557
- });
1558
-
1559
- if (errors.length === 0) {
1560
- return null;
1561
- }
1562
-
1563
- const firstError = errors[0];
1564
- const failureInfo = {
1565
- agentId: firstError.sender,
1566
- taskId: firstError.content?.data?.taskId || null,
1567
- iteration: firstError.content?.data?.iteration || 0,
1568
- error: firstError.content?.data?.error || firstError.content?.text,
1569
- timestamp: firstError.timestamp,
1570
- };
1571
- this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
1572
- return failureInfo;
1573
- }
1574
-
1575
- _checkContainerExists(containerId) {
1576
- const { spawn } = require('child_process');
1577
- const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
1578
- return new Promise((resolve) => {
1579
- checkContainer.on('close', (code) => resolve(code === 0));
1580
- });
1581
- }
1582
-
1583
- async _ensureIsolationForResume(clusterId, cluster) {
1584
- if (!cluster.isolation?.enabled) {
1585
- return;
1586
- }
1587
-
1588
- const oldContainerId = cluster.isolation.containerId;
1589
- const containerExists = await this._checkContainerExists(oldContainerId);
1590
- if (containerExists) {
1591
- this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
1592
- return;
1593
- }
1594
-
1595
- this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
1596
-
1597
- const workDir = cluster.isolation.workDir;
1598
- if (!workDir) {
1599
- throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
1600
- }
1601
-
1602
- const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
1603
- if (!fs.existsSync(isolatedPath)) {
1604
- throw new Error(
1605
- `Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
1606
- `Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
1607
- );
1608
- }
1609
-
1610
- const providerName = normalizeProviderName(
1611
- cluster.config?.forceProvider ||
1612
- cluster.config?.defaultProvider ||
1613
- loadSettings().defaultProvider ||
1614
- 'claude'
1615
- );
1616
- const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
1617
- workDir,
1618
- image: cluster.isolation.image,
1619
- reuseExistingWorkspace: true,
1620
- provider: providerName,
1621
- });
1622
-
1623
- this._log(`[Orchestrator] New container created: ${newContainerId}`);
1624
- cluster.isolation.containerId = newContainerId;
1625
-
1626
- for (const agent of cluster.agents) {
1627
- if (agent.isolation?.enabled) {
1628
- agent.isolation.containerId = newContainerId;
1629
- agent.isolation.manager = cluster.isolation.manager;
1630
- }
1631
- }
1632
-
1633
- this._log(`[Orchestrator] All agents updated with new container ID`);
1634
- }
1635
-
1636
- _ensureWorktreeForResume(clusterId, cluster) {
1637
- if (!cluster.worktree?.enabled) {
1638
- return;
1639
- }
1640
-
1641
- const worktreePath = cluster.worktree.path;
1642
- if (!fs.existsSync(worktreePath)) {
1643
- throw new Error(
1644
- `Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
1645
- `Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
1646
- );
1647
- }
1648
-
1649
- this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
1650
- this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1651
- }
1652
-
1653
- async _restartClusterAgents(cluster) {
1654
- cluster.state = 'running';
1655
- for (const agent of cluster.agents) {
1656
- if (!agent.running) {
1657
- await agent.start();
1658
- }
1659
- }
1660
- }
1661
-
1662
- _loadRecentMessages(cluster, clusterId, limit) {
1663
- return cluster.messageBus
1664
- .query({
1665
- cluster_id: clusterId,
1666
- limit,
1667
- order: 'desc',
1668
- })
1669
- .reverse();
1670
- }
1671
-
1672
- _buildResumeContext(recentMessages, prompt, options) {
1673
- const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1674
- let context = `${options.header}\n\n`;
1675
-
1676
- if (options.error) {
1677
- context += `Previous error: ${options.error}\n\n`;
1678
- }
1679
-
1680
- context += `## Recent Context\n\n`;
1681
- for (const msg of recentMessages.slice(-10)) {
1682
- if (options.topics.includes(msg.topic)) {
1683
- context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1684
- }
1685
- }
1686
-
1687
- context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1688
- return context;
1689
- }
1690
-
1691
- _triggerMatchesTopic(triggerTopic, topic) {
1692
- if (triggerTopic === topic) return true;
1693
- if (triggerTopic === '*') return true;
1694
- if (triggerTopic.endsWith('*')) {
1695
- const prefix = triggerTopic.slice(0, -1);
1696
- return topic.startsWith(prefix);
1697
- }
1698
- return false;
1699
- }
1700
-
1701
- _selectAgentsToResume(cluster, lastTrigger) {
1702
- const agentsToResume = [];
1703
-
1704
- for (const agent of cluster.agents) {
1705
- if (!agent.config.triggers) continue;
1706
-
1707
- const matchingTrigger = agent.config.triggers.find((trigger) =>
1708
- this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
1709
- );
1710
- if (!matchingTrigger) continue;
1711
-
1712
- if (matchingTrigger.logic?.script) {
1713
- const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
1714
- if (!shouldTrigger) continue;
1715
- }
1716
-
1717
- agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
1718
- }
1719
-
1720
- return agentsToResume;
1721
- }
1722
-
1723
- _resumeAgents(agentsToResume, context) {
1724
- for (const { agent, message } of agentsToResume) {
1725
- this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
1726
- agent.resume(context).catch((err) => {
1727
- console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
1728
- });
1729
- }
1730
- }
1731
-
1732
- _republishIssue(cluster, clusterId, recentMessages) {
1733
- const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
1734
- if (!issueMessage) {
1735
- return false;
1736
- }
1737
-
1738
- cluster.messageBus.publish({
1739
- cluster_id: clusterId,
1740
- topic: 'ISSUE_OPENED',
1741
- sender: 'system',
1742
- receiver: 'broadcast',
1743
- content: issueMessage.content,
1744
- metadata: { _resumed: true, _originalId: issueMessage.id },
1745
- });
1746
-
1747
- return true;
1748
- }
1749
-
1750
- async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
1751
- const { agentId, iteration, error } = failureInfo;
1752
- this._log(
1753
- `[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
1754
- );
1755
- this._log(`[Orchestrator] Previous error: ${error}`);
1756
-
1757
- const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
1758
- if (!failedAgent) {
1759
- throw new Error(`Failed agent '${agentId}' not found in cluster`);
1760
- }
1761
-
1762
- const context = this._buildResumeContext(recentMessages, prompt, {
1763
- header: 'You are resuming from a previous failed attempt.',
1764
- error,
1765
- topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
1766
- });
1767
-
1768
- cluster.failureInfo = null;
1769
- await this._saveClusters();
1770
-
1771
- failedAgent.resume(context).catch((err) => {
1772
- console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
1773
- });
1774
-
1775
- this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
1776
-
1777
- return {
1778
- id: clusterId,
1779
- state: cluster.state,
1780
- resumeType: 'failure',
1781
- resumedAgent: agentId,
1782
- previousError: error,
1783
- };
1784
- }
1785
-
1786
- async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
1787
- this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
1788
-
1789
- const context = this._buildResumeContext(recentMessages, prompt, {
1790
- header: 'Resuming cluster from previous session.',
1791
- topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
1792
- });
1793
-
1794
- const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
1795
- if (lastTrigger) {
1796
- this._log(
1797
- `[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
1798
- );
1799
- } else {
1800
- this._log(`[Orchestrator] No workflow triggers found in ledger`);
1801
- }
1802
-
1803
- const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
1804
- if (agentsToResume.length === 0) {
1805
- if (!lastTrigger) {
1806
- this._log(
1807
- `[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
1808
- );
1809
- this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
1810
-
1811
- if (!this._republishIssue(cluster, clusterId, recentMessages)) {
1812
- throw new Error(
1813
- `Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
1814
- `The cluster may not have started properly. Try: zeroshot run <issue> instead.`
1815
- );
1816
- }
1817
- } else {
1818
- throw new Error(
1819
- `Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
1820
- `Check agent trigger configurations.`
1821
- );
1822
- }
1823
- } else {
1824
- this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
1825
- this._resumeAgents(agentsToResume, context);
1826
- }
1827
-
1828
- await this._saveClusters();
1829
- this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
1830
-
1831
- return {
1832
- id: clusterId,
1833
- state: cluster.state,
1834
- resumeType: 'clean',
1835
- resumedAgents: agentsToResume.map((a) => a.agent.id),
1836
- };
1837
- }
1838
-
1839
- /**
1840
- * Force restart a stale agent with imperative prompt injection
1841
- * @param {string} clusterId - Cluster ID
1842
- * @param {string} agentId - Agent to restart
1843
- * @param {number} staleDuration - How long agent was stale (ms)
1844
- * @private
1845
- */
1846
- async _forceRestartAgent(clusterId, agentId, staleDuration) {
1847
- const cluster = this.clusters.get(clusterId);
1848
- if (!cluster) {
1849
- throw new Error(`Cluster ${clusterId} not found`);
1850
- }
1851
-
1852
- const agent = cluster.agents.find((a) => a.id === agentId);
1853
- if (!agent) {
1854
- throw new Error(`Agent ${agentId} not found in cluster ${clusterId}`);
1855
- }
1856
-
1857
- // Kill current task
1858
- try {
1859
- agent._killTask();
1860
- } catch (err) {
1861
- this._log(`⚠️ Failed to kill agent ${agentId} task:`, err.message);
1862
- }
1863
-
1864
- // Build imperative restart context
1865
- const staleMinutes = Math.round(staleDuration / 60000);
1866
- const imperativePrompt = `
1867
- 🔴 CRITICAL: Your previous session STOPPED PRODUCING OUTPUT for ${staleMinutes} minutes and was detected as STUCK.
1868
-
1869
- ## What Happened
1870
- - Last output timestamp: ${new Date(Date.now() - staleDuration).toISOString()}
1871
- - Detected as stale after ${staleMinutes} minutes of silence
1872
- - Process was forcefully restarted
1873
-
1874
- ## Your Instructions
1875
- You MUST complete your current task. DO NOT STOP until you either:
1876
- 1. Successfully complete the task and publish your completion message, OR
1877
- 2. Explicitly state WHY you cannot complete the task (missing files, impossible requirements, etc.)
1878
-
1879
- If you discovered that files you need to modify don't exist:
1880
- - CREATE them from scratch with the expected implementation
1881
- - DO NOT silently give up
1882
- - DO NOT stop working without explicit explanation
1883
-
1884
- If you are stuck in an impossible situation:
1885
- - EXPLAIN the problem clearly
1886
- - PROPOSE alternative solutions
1887
- - WAIT for guidance - do not exit
1888
-
1889
- ## Resume Your Work
1890
- Continue from where you left off. Review your previous output to understand what you were working on.
1891
- `.trim();
1892
-
1893
- // Get recent context from ledger
1894
- const recentMessages = cluster.messageBus
1895
- .query({
1896
- cluster_id: cluster.id,
1897
- limit: 10,
1898
- order: 'desc',
1899
- })
1900
- .reverse();
1901
-
1902
- const contextText = recentMessages
1903
- .map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
1904
- .join('\n\n');
1905
-
1906
- const fullContext = `${imperativePrompt}\n\n## Recent Context\n${contextText}`;
1907
-
1908
- // Resume agent with imperative prompt
1909
- this._log(
1910
- `🔄 Restarting agent ${agentId} with imperative prompt (${imperativePrompt.length} chars)`
1911
- );
1912
-
1913
- try {
1914
- await agent.resume(fullContext);
1915
- this._log(`✅ Agent ${agentId} successfully restarted`);
1916
- } catch (err) {
1917
- this._log(`❌ Failed to resume agent ${agentId}:`, err.message);
1918
- throw err;
1919
- }
1920
- }
1921
-
1922
- /**
1923
- * Handle operation chain from CLUSTER_OPERATIONS message
1924
- * Executes operations sequentially: add_agents, remove_agents, update_agent, publish
1925
- *
1926
- * Validation strategy:
1927
- * 1. Pre-validate all agent configs before executing any operations
1928
- * 2. Build a mock cluster config with proposed changes
1929
- * 3. Run config-validator on the mock to catch structural issues
1930
- * 4. Only execute operations if validation passes
1931
- *
1932
- * @param {string} clusterId - Cluster ID
1933
- * @param {Array} operations - Array of operation objects
1934
- * @param {string} sender - Who sent the operations (for attribution)
1935
- * @param {Object} context - Isolation context { isolationManager, containerId }
1936
- * @private
1937
- */
1938
- async _handleOperations(clusterId, operations, sender, context = {}) {
1939
- const cluster = this.clusters.get(clusterId);
1940
- if (!cluster) {
1941
- throw new Error(`Cluster ${clusterId} not found`);
1942
- }
1943
-
1944
- this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
1945
-
1946
- // Phase 1: Pre-validate operation structure
1947
- const validationErrors = this._validateOperationChain(operations);
1948
-
1949
- if (validationErrors.length > 0) {
1950
- const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
1951
- this._log(`[Orchestrator] ❌ ${errorMsg}`);
1952
- throw new Error(errorMsg);
1953
- }
1954
-
1955
- // Phase 2: Build mock cluster config with proposed agents
1956
- // Collect all agents that would exist after operations complete
1957
- const existingAgentConfigs = cluster.config.agents || [];
1958
- const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
1959
-
1960
- // Phase 3: Validate proposed cluster config
1961
- const validation = this._validateProposedConfig(
1962
- clusterId,
1963
- cluster,
1964
- proposedAgentConfigs,
1965
- operations
1966
- );
1967
-
1968
- // Log warnings but proceed
1969
- if (validation.warnings.length > 0) {
1970
- this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
1971
- for (const warning of validation.warnings) {
1972
- this._log(` - ${warning}`);
1973
- }
1974
- }
1975
-
1976
- // Phase 4: Execute validated operations
1977
- this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
1978
-
1979
- await this._executeOperations(cluster, operations, sender, context);
1980
-
1981
- this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
1982
-
1983
- // Publish success notification
1984
- cluster.messageBus.publish({
1985
- cluster_id: clusterId,
1986
- topic: 'CLUSTER_OPERATIONS_SUCCESS',
1987
- sender: 'orchestrator',
1988
- content: {
1989
- text: `Executed ${operations.length} operation(s)`,
1990
- data: {
1991
- operationCount: operations.length,
1992
- agentCount: cluster.agents.length,
1993
- },
1994
- },
1995
- });
1996
-
1997
- // Save updated cluster state to disk
1998
- await this._saveClusters();
1999
- }
2000
-
2001
- _validateOperationChain(operations) {
2002
- const validationErrors = [];
2003
-
2004
- for (let i = 0; i < operations.length; i++) {
2005
- const op = operations[i];
2006
- if (!op.action) {
2007
- validationErrors.push(`Operation ${i}: missing 'action' field`);
2008
- continue;
2009
- }
2010
- if (!VALID_OPERATIONS.includes(op.action)) {
2011
- validationErrors.push(
2012
- `Operation ${i}: unknown action '${op.action}'. Valid: ${VALID_OPERATIONS.join(', ')}`
2013
- );
2014
- }
2015
- }
2016
-
2017
- return validationErrors;
2018
- }
2019
-
2020
- _buildProposedAgentConfigs(existingAgentConfigs, operations) {
2021
- const proposedAgentConfigs = [...existingAgentConfigs];
2022
-
2023
- for (const op of operations) {
2024
- if (op.action === 'add_agents' && op.agents) {
2025
- for (const agentConfig of op.agents) {
2026
- const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
2027
- if (existingIdx === -1) {
2028
- proposedAgentConfigs.push(agentConfig);
2029
- }
2030
- }
2031
- } else if (op.action === 'remove_agents' && op.agentIds) {
2032
- for (const agentId of op.agentIds) {
2033
- const idx = proposedAgentConfigs.findIndex((a) => a.id === agentId);
2034
- if (idx !== -1) {
2035
- proposedAgentConfigs.splice(idx, 1);
2036
- }
2037
- }
2038
- } else if (op.action === 'update_agent' && op.agentId && op.updates) {
2039
- const agentConfig = proposedAgentConfigs.find((a) => a.id === op.agentId);
2040
- if (agentConfig) {
2041
- Object.assign(agentConfig, op.updates);
2042
- }
2043
- }
2044
- }
2045
-
2046
- return proposedAgentConfigs;
2047
- }
2048
-
2049
- _validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
2050
- const mockConfig = { agents: proposedAgentConfigs };
2051
- const validation = configValidator.validateConfig(mockConfig);
2052
-
2053
- if (!validation.valid) {
2054
- const errorMsg = `Proposed cluster configuration is invalid:\n - ${validation.errors.join('\n - ')}`;
2055
- this._log(`[Orchestrator] ❌ ${errorMsg}`);
2056
-
2057
- // Publish validation failure for conductor to see and retry
2058
- cluster.messageBus.publish({
2059
- cluster_id: clusterId,
2060
- topic: 'CLUSTER_OPERATIONS_VALIDATION_FAILED',
2061
- sender: 'orchestrator',
2062
- content: {
2063
- text: 'Operation chain would create invalid cluster configuration',
2064
- data: {
2065
- errors: validation.errors,
2066
- warnings: validation.warnings,
2067
- operations: operations,
2068
- },
2069
- },
2070
- });
2071
-
2072
- throw new Error(errorMsg);
2073
- }
2074
-
2075
- return validation;
2076
- }
2077
-
2078
- async _executeOperations(cluster, operations, sender, context) {
2079
- for (let i = 0; i < operations.length; i++) {
2080
- const op = operations[i];
2081
- this._log(` [${i + 1}/${operations.length}] ${op.action}`);
2082
- await this._executeOperation(cluster, op, sender, context);
2083
- }
2084
- }
2085
-
2086
- async _executeOperation(cluster, op, sender, context) {
2087
- switch (op.action) {
2088
- case 'add_agents':
2089
- await this._opAddAgents(cluster, op, context);
2090
- break;
2091
- case 'remove_agents':
2092
- await this._opRemoveAgents(cluster, op);
2093
- break;
2094
- case 'update_agent':
2095
- this._opUpdateAgent(cluster, op);
2096
- break;
2097
- case 'publish':
2098
- this._opPublish(cluster, op, sender);
2099
- break;
2100
- case 'load_config':
2101
- await this._opLoadConfig(cluster, op, context);
2102
- break;
2103
- }
2104
- }
2105
-
2106
- /**
2107
- * Operation: add_agents - Spawn new agents dynamically
2108
- * @private
2109
- */
2110
- async _opAddAgents(cluster, op, context) {
2111
- const agents = op.agents;
2112
- if (!agents || !Array.isArray(agents)) {
2113
- throw new Error('add_agents operation missing agents array');
2114
- }
2115
-
2116
- // CRITICAL: Inject cluster's worktree cwd into dynamically added agents
2117
- // Without this, template agents (planning, implementation, validator) run in
2118
- // original directory instead of worktree, causing --ship mode to pollute main
2119
- const agentCwd = cluster.worktree?.path || cluster.isolation?.workDir || process.cwd();
2120
-
2121
- for (const agentConfig of agents) {
2122
- // Inject cwd if not already set (same as startCluster does for initial agents)
2123
- if (!agentConfig.cwd) {
2124
- agentConfig.cwd = agentCwd;
2125
- this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
2126
- }
2127
-
2128
- // Apply model override if set (for consistency with initial agents)
2129
- if (cluster.modelOverride) {
2130
- applyModelOverride(agentConfig, cluster.modelOverride);
2131
- this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
2132
- }
2133
- // Validate agent config has required fields
2134
- if (!agentConfig.id) {
2135
- throw new Error('Agent config missing required field: id');
2136
- }
2137
-
2138
- // Check for duplicate agent ID
2139
- const existingAgent = cluster.agents.find((a) => a.id === agentConfig.id);
2140
- if (existingAgent) {
2141
- this._log(` ⚠️ Agent ${agentConfig.id} already exists, skipping`);
2142
- continue;
2143
- }
2144
-
2145
- // Add to config agents array (for persistence)
2146
- if (!cluster.config.agents) {
2147
- cluster.config.agents = [];
2148
- }
2149
- cluster.config.agents.push(agentConfig);
2150
-
2151
- // Build agent options
2152
- const agentOptions = {
2153
- testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
2154
- quiet: this.quiet,
2155
- modelOverride: cluster.modelOverride || null,
2156
- };
2157
-
2158
- // TaskRunner DI - propagate to dynamically spawned agents
2159
- if (this.taskRunner) {
2160
- agentOptions.taskRunner = this.taskRunner;
2161
- }
2162
-
2163
- // Pass isolation context if cluster is running in isolation mode
2164
- if (cluster.isolation?.enabled && context.isolationManager) {
2165
- agentOptions.isolation = {
2166
- enabled: true,
2167
- manager: context.isolationManager,
2168
- clusterId: cluster.id,
2169
- };
2170
- }
2171
-
2172
- // Create and start agent
2173
- const agent = new AgentWrapper(agentConfig, cluster.messageBus, cluster, agentOptions);
2174
- cluster.agents.push(agent);
2175
- await agent.start();
2176
-
2177
- this._log(
2178
- ` ✓ Added agent: ${agentConfig.id} (role: ${agentConfig.role || 'unspecified'})`
2179
- );
2180
- }
2181
- }
2182
-
2183
- /**
2184
- * Operation: remove_agents - Stop and remove agents by ID
2185
- * @private
2186
- */
2187
- async _opRemoveAgents(cluster, op) {
2188
- const agentIds = op.agentIds;
2189
- if (!agentIds || !Array.isArray(agentIds)) {
2190
- throw new Error('remove_agents operation missing agentIds array');
2191
- }
2192
-
2193
- for (const agentId of agentIds) {
2194
- const agentIndex = cluster.agents.findIndex((a) => a.id === agentId);
2195
- if (agentIndex === -1) {
2196
- this._log(` ⚠️ Agent ${agentId} not found, skipping removal`);
2197
- continue;
2198
- }
2199
-
2200
- const agent = cluster.agents[agentIndex];
2201
- await agent.stop();
2202
-
2203
- // Remove from cluster.agents
2204
- cluster.agents.splice(agentIndex, 1);
2205
-
2206
- // Remove from config.agents
2207
- if (cluster.config.agents) {
2208
- const configIndex = cluster.config.agents.findIndex((a) => a.id === agentId);
2209
- if (configIndex !== -1) {
2210
- cluster.config.agents.splice(configIndex, 1);
2211
- }
2212
- }
2213
-
2214
- this._log(` ✓ Removed agent: ${agentId}`);
2215
- }
2216
- }
2217
-
2218
- /**
2219
- * Operation: update_agent - Modify existing agent config at runtime
2220
- * Note: Some updates may require agent restart to take effect
2221
- * @private
2222
- */
2223
- _opUpdateAgent(cluster, op) {
2224
- const { agentId, updates } = op;
2225
- if (!agentId) {
2226
- throw new Error('update_agent operation missing agentId');
2227
- }
2228
- if (!updates || typeof updates !== 'object') {
2229
- throw new Error('update_agent operation missing updates object');
2230
- }
2231
-
2232
- const agent = cluster.agents.find((a) => a.id === agentId);
2233
- if (!agent) {
2234
- throw new Error(`update_agent: Agent ${agentId} not found`);
2235
- }
2236
-
2237
- // Apply updates to agent config
2238
- Object.assign(agent.config, updates);
2239
-
2240
- // Also update in cluster.config.agents for persistence
2241
- if (cluster.config.agents) {
2242
- const configAgent = cluster.config.agents.find((a) => a.id === agentId);
2243
- if (configAgent) {
2244
- Object.assign(configAgent, updates);
2245
- }
2246
- }
2247
-
2248
- this._log(` ✓ Updated agent: ${agentId} (fields: ${Object.keys(updates).join(', ')})`);
2249
- }
2250
-
2251
- /**
2252
- * Operation: publish - Publish a message to the bus
2253
- * @private
2254
- */
2255
- _opPublish(cluster, op, sender) {
2256
- const { topic, content, metadata } = op;
2257
- if (!topic) {
2258
- throw new Error('publish operation missing topic');
2259
- }
2260
-
2261
- cluster.messageBus.publish({
2262
- cluster_id: cluster.id,
2263
- topic,
2264
- sender: op.sender || sender,
2265
- receiver: op.receiver || 'broadcast',
2266
- content: content || {},
2267
- metadata: metadata || null,
2268
- });
2269
-
2270
- this._log(` ✓ Published to topic: ${topic}`);
2271
- }
2272
-
2273
- /**
2274
- * Operation: load_config - Load agents from a cluster config
2275
- *
2276
- * Supports two formats:
2277
- * 1. Static config: { config: 'config-name' } - loads from cluster-templates/config-name.json
2278
- * 2. Parameterized: { config: { base: 'template-name', params: {...} } } - resolves base template with params
2279
- *
2280
- * @private
2281
- */
2282
- async _opLoadConfig(cluster, op, context) {
2283
- const { config } = op;
2284
- if (!config) {
2285
- throw new Error('load_config operation missing config');
2286
- }
2287
-
2288
- const templatesDir = path.join(__dirname, '..', 'cluster-templates');
2289
- let loadedConfig;
2290
-
2291
- // Check if config is parameterized ({ base, params }) or static (string)
2292
- if (typeof config === 'object' && config.base) {
2293
- // Parameterized template - resolve with TemplateResolver
2294
- const { base, params } = config;
2295
- this._log(` Loading parameterized template: ${base}`);
2296
- this._log(` Params: ${JSON.stringify(params)}`);
2297
-
2298
- const resolver = new TemplateResolver(templatesDir);
2299
- loadedConfig = resolver.resolve(base, params);
2300
-
2301
- this._log(` ✓ Resolved template: ${base} → ${loadedConfig.agents?.length || 0} agent(s)`);
2302
- } else if (typeof config === 'string') {
2303
- // Static config - load directly from file
2304
- const configPath = path.join(templatesDir, `${config}.json`);
2305
-
2306
- if (!fs.existsSync(configPath)) {
2307
- throw new Error(`Config not found: ${config} (looked in ${configPath})`);
2308
- }
2309
-
2310
- this._log(` Loading static config: ${config}`);
2311
-
2312
- const configContent = fs.readFileSync(configPath, 'utf8');
2313
- loadedConfig = JSON.parse(configContent);
2314
- } else {
2315
- throw new Error(
2316
- `Invalid config format: expected string or {base, params}, got ${typeof config}`
2317
- );
2318
- }
2319
-
2320
- if (!loadedConfig.agents || !Array.isArray(loadedConfig.agents)) {
2321
- throw new Error(`Config has no agents array`);
2322
- }
2323
-
2324
- this._log(` Found ${loadedConfig.agents.length} agent(s)`);
2325
-
2326
- // Add agents from loaded config (reuse existing add_agents logic)
2327
- await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
2328
-
2329
- this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
2330
-
2331
- // Inject completion agent (templates don't include one - orchestrator controls termination)
2332
- await this._injectCompletionAgent(cluster, context);
2333
- }
2334
-
2335
- /**
2336
- * Inject appropriate completion agent based on mode
2337
- * Templates define work, orchestrator controls termination strategy
2338
- * @private
2339
- */
2340
- async _injectCompletionAgent(cluster, context) {
2341
- // Skip if completion agent already exists
2342
- const hasCompletionAgent = cluster.agents.some(
2343
- (a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
2344
- );
2345
- if (hasCompletionAgent) {
2346
- return;
2347
- }
2348
-
2349
- const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
2350
-
2351
- if (isPrMode) {
2352
- // Detect platform from stored cluster metadata OR git context
2353
- let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
2354
-
2355
- // Fallback to git context detection if cluster metadata missing
2356
- if (!platform) {
2357
- const { getPlatformForPR } = require('./issue-providers');
2358
- try {
2359
- platform = getPlatformForPR(cluster.cwd || process.cwd());
2360
- } catch (error) {
2361
- throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
2362
- }
2363
- }
2364
-
2365
- // Generate platform-specific git-pusher agent from template
2366
- const {
2367
- generateGitPusherAgent,
2368
- isPlatformSupported,
2369
- } = require('./agents/git-pusher-template');
2370
-
2371
- if (!isPlatformSupported(platform)) {
2372
- throw new Error(
2373
- `Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
2374
- );
2375
- }
2376
-
2377
- const gitPusherConfig = generateGitPusherAgent(platform);
2378
-
2379
- // Get issue context from ledger
2380
- const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
2381
- const issueNumber = issueMsg?.content?.data?.number || 'unknown';
2382
- const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
2383
-
2384
- // Inject issue context into prompt
2385
- gitPusherConfig.prompt = gitPusherConfig.prompt
2386
- .replace(/\{\{issue_number\}\}/g, issueNumber)
2387
- .replace(/\{\{issue_title\}\}/g, issueTitle);
2388
-
2389
- await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
2390
- this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
2391
- } else {
2392
- // Default completion-detector
2393
- const completionDetector = {
2394
- id: 'completion-detector',
2395
- role: 'orchestrator',
2396
- model: 'haiku',
2397
- timeout: 0,
2398
- triggers: [
2399
- {
2400
- topic: 'VALIDATION_RESULT',
2401
- logic: {
2402
- engine: 'javascript',
2403
- script: `const validators = cluster.getAgentsByRole('validator');
2404
- const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });
2405
- if (!lastPush) return false;
2406
- if (validators.length === 0) return true;
2407
-
2408
- const validatorIds = new Set(validators.map((v) => v.id));
2409
- const results = ledger.query({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });
2410
-
2411
- const latestByValidator = new Map();
2412
- for (const msg of results) {
2413
- if (!validatorIds.has(msg.sender)) continue;
2414
- latestByValidator.set(msg.sender, msg);
2415
- }
2416
-
2417
- if (latestByValidator.size < validators.length) return false;
2418
-
2419
- for (const validator of validators) {
2420
- const msg = latestByValidator.get(validator.id);
2421
- const approved = msg?.content?.data?.approved;
2422
- if (!(approved === true || approved === 'true')) return false;
2423
- }
2424
-
2425
- return true;`,
2426
- },
2427
- action: 'stop_cluster',
2428
- },
2429
- ],
2430
- };
2431
-
2432
- await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
2433
- this._log(` Injected completion-detector agent`);
2434
- }
2435
- }
2436
-
2437
- /**
2438
- * Check if a process with given PID is running
2439
- * @param {Number} pid - Process ID
2440
- * @returns {Boolean} True if process exists
2441
- * @private
2442
- */
2443
- _isProcessRunning(pid) {
2444
- if (!pid) return false;
2445
- try {
2446
- // Signal 0 doesn't kill, just checks if process exists
2447
- process.kill(pid, 0);
2448
- return true;
2449
- } catch (e) {
2450
- // ESRCH = No such process, EPERM = process exists but no permission
2451
- return e.code === 'EPERM';
2452
- }
2453
- }
2454
-
2455
- /**
2456
- * Get cluster status
2457
- * @param {String} clusterId - Cluster ID
2458
- * @returns {Object} Cluster status
2459
- */
2460
- getStatus(clusterId) {
2461
- const cluster = this.clusters.get(clusterId);
2462
- if (!cluster) {
2463
- throw new Error(`Cluster ${clusterId} not found`);
2464
- }
2465
-
2466
- // Detect zombie clusters: state=running but no backing process
2467
- let state = cluster.state;
2468
- let isZombie = false;
2469
- if (state === 'running') {
2470
- if (cluster.pid) {
2471
- // PID recorded - check if process is running
2472
- if (!this._isProcessRunning(cluster.pid)) {
2473
- state = 'zombie';
2474
- isZombie = true;
2475
- this._log(
2476
- `[Orchestrator] Detected zombie cluster ${clusterId} (PID ${cluster.pid} not running)`
2477
- );
2478
- }
2479
- } else {
2480
- // No PID recorded (legacy cluster or bug) - definitely a zombie
2481
- // New code always records PID for running clusters
2482
- state = 'zombie';
2483
- isZombie = true;
2484
- this._log(
2485
- `[Orchestrator] Detected zombie cluster ${clusterId} (no PID recorded - legacy or killed cluster)`
2486
- );
2487
- }
2488
- }
2489
-
2490
- return {
2491
- id: clusterId,
2492
- state: state,
2493
- isZombie: isZombie,
2494
- pid: cluster.pid || null,
2495
- createdAt: cluster.createdAt,
2496
- agents: cluster.agents.map((a) => a.getState()),
2497
- messageCount: cluster.messageBus.count({ cluster_id: clusterId }),
2498
- };
2499
- }
2500
-
2501
- /**
2502
- * List all clusters
2503
- * @returns {Array} List of cluster summaries
2504
- */
2505
- listClusters() {
2506
- return Array.from(this.clusters.values()).map((cluster) => {
2507
- // Detect zombie clusters (state=running but no backing process)
2508
- let state = cluster.state;
2509
- if (state === 'running') {
2510
- if (cluster.pid) {
2511
- if (!this._isProcessRunning(cluster.pid)) {
2512
- state = 'zombie';
2513
- }
2514
- } else {
2515
- // No PID recorded - definitely a zombie
2516
- state = 'zombie';
2517
- }
2518
- }
2519
-
2520
- return {
2521
- id: cluster.id,
2522
- state: state,
2523
- createdAt: cluster.createdAt,
2524
- agentCount: cluster.agents.length,
2525
- messageCount: cluster.messageBus.getAll(cluster.id).length,
2526
- };
2527
- });
2528
- }
2529
-
2530
- /**
2531
- * Get cluster object (for internal use)
2532
- * @param {String} clusterId - Cluster ID
2533
- * @returns {Object} Full cluster object
2534
- */
2535
- getCluster(clusterId) {
2536
- return this.clusters.get(clusterId);
2537
- }
2538
-
2539
- /**
2540
- * Export cluster conversation
2541
- * @param {String} clusterId - Cluster ID
2542
- * @param {String} format - Export format ('json' or 'markdown')
2543
- * @returns {String} Exported data
2544
- */
2545
- export(clusterId, format = 'json') {
2546
- const cluster = this.clusters.get(clusterId);
2547
- if (!cluster) {
2548
- throw new Error(`Cluster ${clusterId} not found`);
2549
- }
2550
-
2551
- const messages = cluster.messageBus.getAll(clusterId);
2552
-
2553
- if (format === 'json') {
2554
- return JSON.stringify(
2555
- {
2556
- cluster_id: clusterId,
2557
- state: cluster.state,
2558
- created_at: cluster.createdAt,
2559
- agents: cluster.agents.map((a) => a.getState()),
2560
- messages,
2561
- },
2562
- null,
2563
- 2
2564
- );
2565
- } else if (format === 'markdown') {
2566
- return this._exportMarkdown(cluster, clusterId, messages);
2567
- } else {
2568
- throw new Error(`Unknown export format: ${format}`);
2569
- }
2570
- }
2571
-
2572
- /**
2573
- * Export cluster as nicely formatted markdown
2574
- * @private
2575
- */
2576
- _exportMarkdown(cluster, clusterId, messages) {
2577
- const { parseProviderChunk } = require('./providers');
2578
-
2579
- // Find task info
2580
- const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
2581
- const taskText = issueOpened?.content?.text || 'Unknown task';
2582
-
2583
- // Calculate duration
2584
- const firstMsg = messages[0];
2585
- const lastMsg = messages[messages.length - 1];
2586
- const durationMs = lastMsg ? lastMsg.timestamp - firstMsg.timestamp : 0;
2587
- const durationMin = Math.round(durationMs / 60000);
2588
-
2589
- // Header
2590
- let md = `# Cluster: ${clusterId}\n\n`;
2591
- md += `| Property | Value |\n|----------|-------|\n`;
2592
- md += `| State | ${cluster.state} |\n`;
2593
- md += `| Created | ${new Date(cluster.createdAt).toLocaleString()} |\n`;
2594
- md += `| Duration | ${durationMin} minutes |\n`;
2595
- md += `| Agents | ${cluster.agents.map((a) => a.id).join(', ')} |\n\n`;
2596
-
2597
- // Task
2598
- md += `## Task\n\n${taskText}\n\n`;
2599
-
2600
- // Group messages by agent for cleaner output
2601
- const agentOutputs = this._collectAgentOutputs(messages);
2602
-
2603
- for (const [agentId, agentMsgs] of agentOutputs) {
2604
- md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
2605
- }
2606
-
2607
- md += this._renderValidationMarkdown(messages);
2608
-
2609
- // Final status
2610
- const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
2611
- if (clusterComplete) {
2612
- md += `## Result\n\n✅ **Cluster completed successfully**\n`;
2613
- }
2614
-
2615
- return md;
2616
- }
2617
-
2618
- _collectAgentOutputs(messages) {
2619
- const agentOutputs = new Map();
2620
-
2621
- for (const msg of messages) {
2622
- if (msg.topic !== 'AGENT_OUTPUT') continue;
2623
- if (!agentOutputs.has(msg.sender)) {
2624
- agentOutputs.set(msg.sender, []);
2625
- }
2626
- agentOutputs.get(msg.sender).push(msg);
2627
- }
2628
-
2629
- return agentOutputs;
2630
- }
2631
-
2632
- _extractAgentOutput(agentMsgs, parseProviderChunk) {
2633
- let text = '';
2634
- const tools = [];
2635
-
2636
- for (const msg of agentMsgs) {
2637
- const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
2638
- if (!content) continue;
2639
-
2640
- const provider = normalizeProviderName(
2641
- msg.content?.data?.provider || msg.sender_provider || 'claude'
2642
- );
2643
- const events = parseProviderChunk(provider, content);
2644
- for (const event of events) {
2645
- switch (event.type) {
2646
- case 'text':
2647
- text += event.text;
2648
- break;
2649
- case 'tool_call':
2650
- tools.push({ name: event.toolName, input: event.input });
2651
- break;
2652
- case 'tool_result':
2653
- if (tools.length > 0) {
2654
- const lastTool = tools[tools.length - 1];
2655
- lastTool.result = event.content;
2656
- lastTool.isError = event.isError;
2657
- }
2658
- break;
2659
- }
2660
- }
2661
- }
2662
-
2663
- return { text, tools };
2664
- }
2665
-
2666
- _renderToolMarkdown(tools) {
2667
- let md = `### Tools Used\n\n`;
2668
-
2669
- for (const tool of tools) {
2670
- const status = tool.isError ? '❌' : '✓';
2671
- md += `- **${tool.name}** ${status}\n`;
2672
- if (tool.input) {
2673
- const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
2674
- if (inputStr.length < 100) {
2675
- md += ` - Input: \`${inputStr}\`\n`;
2676
- }
2677
- }
2678
- }
2679
-
2680
- return `${md}\n`;
2681
- }
2682
-
2683
- _renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
2684
- let md = `## Agent: ${agentId}\n\n`;
2685
- const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
2686
-
2687
- if (text.trim()) {
2688
- md += `### Output\n\n${text.trim()}\n\n`;
2689
- }
2690
-
2691
- if (tools.length > 0) {
2692
- md += this._renderToolMarkdown(tools);
2693
- }
2694
-
2695
- return md;
2696
- }
2697
-
2698
- _renderCriteriaMarkdown(criteriaResults) {
2699
- if (!Array.isArray(criteriaResults)) {
2700
- return '';
2701
- }
2702
-
2703
- let md = '';
2704
- const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
2705
- if (cannotValidateYet.length > 0) {
2706
- md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
2707
- for (const cv of cannotValidateYet) {
2708
- md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
2709
- }
2710
- md += '\n';
2711
- }
2712
-
2713
- const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
2714
- if (cannotValidate.length > 0) {
2715
- md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
2716
- for (const cv of cannotValidate) {
2717
- md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
2718
- }
2719
- md += '\n';
2720
- }
2721
-
2722
- return md;
2723
- }
2724
-
2725
- _renderValidationEntryMarkdown(validation) {
2726
- const data = validation.content?.data || {};
2727
- const approved = data.approved === true || data.approved === 'true';
2728
- const icon = approved ? '✅' : '❌';
2729
- let md = `### ${validation.sender} ${icon}\n\n`;
2730
-
2731
- if (data.summary) {
2732
- md += `${data.summary}\n\n`;
2733
- }
2734
-
2735
- if (!approved && data.issues) {
2736
- const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
2737
- if (Array.isArray(issues) && issues.length > 0) {
2738
- md += `**Issues:**\n`;
2739
- for (const issue of issues) {
2740
- md += `- ${issue}\n`;
2741
- }
2742
- md += '\n';
2743
- }
2744
- }
2745
-
2746
- md += this._renderCriteriaMarkdown(data.criteriaResults);
2747
- return md;
2748
- }
2749
-
2750
- _renderValidationMarkdown(messages) {
2751
- const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
2752
- if (validations.length === 0) {
2753
- return '';
2754
- }
2755
-
2756
- let md = `## Validation Results\n\n`;
2757
- for (const validation of validations) {
2758
- md += this._renderValidationEntryMarkdown(validation);
2759
- }
2760
-
2761
- return md;
2762
- }
2763
-
2764
- /**
2765
- * Validate cluster configuration (delegates to config-validator module)
2766
- * @param {Object} config - Cluster configuration
2767
- * @param {Object} options - Validation options
2768
- * @param {boolean} options.strict - Treat warnings as errors (default: false)
2769
- * @returns {Object} { valid: Boolean, errors: Array, warnings: Array }
2770
- */
2771
- validateConfig(config, options = {}) {
2772
- const result = configValidator.validateConfig(config);
2773
-
2774
- // In strict mode, warnings become errors
2775
- if (options.strict && result.warnings.length > 0) {
2776
- result.errors.push(...result.warnings.map((w) => `[strict] ${w}`));
2777
- result.valid = false;
2778
- }
2779
-
2780
- return result;
2781
- }
2782
-
2783
- /**
2784
- * Load cluster configuration from file
2785
- * @param {String} configPath - Path to config JSON file
2786
- * @param {Object} options - Load options
2787
- * @param {boolean} options.strict - Treat warnings as errors
2788
- * @returns {Object} Parsed configuration
2789
- */
2790
- loadConfig(configPath, options = {}) {
2791
- const fullPath = path.resolve(configPath);
2792
- const content = fs.readFileSync(fullPath, 'utf8');
2793
- const config = JSON.parse(content);
2794
-
2795
- const validation = this.validateConfig(config, options);
2796
-
2797
- // Show warnings (but don't fail unless strict mode)
2798
- if (validation.warnings && validation.warnings.length > 0 && !this.quiet) {
2799
- console.warn('\n⚠️ Configuration warnings:');
2800
- for (const warning of validation.warnings) {
2801
- console.warn(` ${warning}`);
2802
- }
2803
- console.warn('');
2804
- }
2805
-
2806
- if (!validation.valid) {
2807
- const errorMsg = validation.errors.join('\n ');
2808
- throw new Error(`Invalid config:\n ${errorMsg}`);
2809
- }
2810
-
2811
- return config;
2812
- }
2813
- }
2814
-
2815
- module.exports = Orchestrator;