@covibes/zeroshot 5.2.1 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/CHANGELOG.md +174 -189
  2. package/README.md +226 -195
  3. package/cli/commands/providers.js +149 -0
  4. package/cli/index.js +3145 -2366
  5. package/cli/lib/first-run.js +40 -3
  6. package/cli/message-formatters-normal.js +28 -6
  7. package/cluster-templates/base-templates/debug-workflow.json +24 -78
  8. package/cluster-templates/base-templates/full-workflow.json +99 -316
  9. package/cluster-templates/base-templates/single-worker.json +23 -15
  10. package/cluster-templates/base-templates/worker-validator.json +105 -36
  11. package/cluster-templates/conductor-bootstrap.json +9 -7
  12. package/lib/docker-config.js +14 -1
  13. package/lib/git-remote-utils.js +165 -0
  14. package/lib/id-detector.js +10 -7
  15. package/lib/provider-defaults.js +62 -0
  16. package/lib/provider-detection.js +59 -0
  17. package/lib/provider-names.js +57 -0
  18. package/lib/settings/claude-auth.js +78 -0
  19. package/lib/settings.js +298 -15
  20. package/lib/stream-json-parser.js +4 -238
  21. package/package.json +27 -6
  22. package/scripts/setup-merge-queue.sh +170 -0
  23. package/scripts/validate-templates.js +100 -0
  24. package/src/agent/agent-config.js +140 -63
  25. package/src/agent/agent-context-builder.js +336 -165
  26. package/src/agent/agent-hook-executor.js +337 -67
  27. package/src/agent/agent-lifecycle.js +386 -287
  28. package/src/agent/agent-stuck-detector.js +7 -7
  29. package/src/agent/agent-task-executor.js +944 -683
  30. package/src/agent/output-extraction.js +217 -0
  31. package/src/agent/output-reformatter.js +175 -0
  32. package/src/agent/schema-utils.js +146 -0
  33. package/src/agent-wrapper.js +112 -31
  34. package/src/agents/git-pusher-template.js +285 -0
  35. package/src/claude-task-runner.js +145 -44
  36. package/src/config-router.js +13 -13
  37. package/src/config-validator.js +1049 -563
  38. package/src/input-helpers.js +65 -0
  39. package/src/isolation-manager.js +499 -320
  40. package/src/issue-providers/README.md +305 -0
  41. package/src/issue-providers/azure-devops-provider.js +273 -0
  42. package/src/issue-providers/base-provider.js +232 -0
  43. package/src/issue-providers/github-provider.js +179 -0
  44. package/src/issue-providers/gitlab-provider.js +241 -0
  45. package/src/issue-providers/index.js +196 -0
  46. package/src/issue-providers/jira-provider.js +239 -0
  47. package/src/ledger.js +50 -11
  48. package/src/lib/safe-exec.js +88 -0
  49. package/src/orchestrator.js +1348 -757
  50. package/src/preflight.js +306 -149
  51. package/src/process-metrics.js +98 -56
  52. package/src/providers/anthropic/cli-builder.js +73 -0
  53. package/src/providers/anthropic/index.js +204 -0
  54. package/src/providers/anthropic/models.js +23 -0
  55. package/src/providers/anthropic/output-parser.js +177 -0
  56. package/src/providers/base-provider.js +251 -0
  57. package/src/providers/capabilities.js +60 -0
  58. package/src/providers/google/cli-builder.js +55 -0
  59. package/src/providers/google/index.js +116 -0
  60. package/src/providers/google/models.js +24 -0
  61. package/src/providers/google/output-parser.js +101 -0
  62. package/src/providers/index.js +91 -0
  63. package/src/providers/openai/cli-builder.js +133 -0
  64. package/src/providers/openai/index.js +136 -0
  65. package/src/providers/openai/models.js +21 -0
  66. package/src/providers/openai/output-parser.js +143 -0
  67. package/src/providers/opencode/cli-builder.js +42 -0
  68. package/src/providers/opencode/index.js +103 -0
  69. package/src/providers/opencode/models.js +55 -0
  70. package/src/providers/opencode/output-parser.js +122 -0
  71. package/src/schemas/sub-cluster.js +68 -39
  72. package/src/status-footer.js +94 -48
  73. package/src/sub-cluster-wrapper.js +92 -36
  74. package/src/task-runner.js +8 -6
  75. package/src/template-resolver.js +12 -9
  76. package/src/tui/data-poller.js +123 -99
  77. package/src/tui/formatters.js +4 -3
  78. package/src/tui/keybindings.js +259 -318
  79. package/src/tui/layout.js +20 -3
  80. package/src/tui/renderer.js +11 -21
  81. package/task-lib/attachable-watcher.js +150 -111
  82. package/task-lib/claude-recovery.js +156 -0
  83. package/task-lib/commands/episodes.js +105 -0
  84. package/task-lib/commands/list.js +3 -3
  85. package/task-lib/commands/logs.js +231 -189
  86. package/task-lib/commands/resume.js +3 -2
  87. package/task-lib/commands/run.js +12 -3
  88. package/task-lib/commands/schedules.js +111 -61
  89. package/task-lib/config.js +0 -2
  90. package/task-lib/runner.js +128 -50
  91. package/task-lib/scheduler.js +3 -3
  92. package/task-lib/store.js +464 -152
  93. package/task-lib/tui/formatters.js +94 -45
  94. package/task-lib/tui/renderer.js +13 -3
  95. package/task-lib/tui.js +73 -32
  96. package/task-lib/watcher.js +157 -100
  97. package/src/agents/git-pusher-agent.json +0 -20
  98. package/src/github.js +0 -103
@@ -13,15 +13,51 @@ const fs = require('fs');
13
13
  const path = require('path');
14
14
  const os = require('os');
15
15
  const lockfile = require('proper-lockfile');
16
+
17
+ // Stale lock timeout in ms - if lock file is older than this, delete it
18
+ const LOCK_STALE_MS = 5000;
19
+
20
+ /**
21
+ * Remove lock file if it's stale (older than LOCK_STALE_MS)
22
+ * Handles crashes that leave orphaned lock files
23
+ */
24
+ function cleanStaleLock(lockPath) {
25
+ try {
26
+ if (fs.existsSync(lockPath)) {
27
+ const age = Date.now() - fs.statSync(lockPath).mtimeMs;
28
+ if (age > LOCK_STALE_MS) {
29
+ fs.unlinkSync(lockPath);
30
+ }
31
+ }
32
+ } catch {
33
+ // Ignore - another process may have cleaned it
34
+ }
35
+ }
16
36
  const AgentWrapper = require('./agent-wrapper');
17
37
  const SubClusterWrapper = require('./sub-cluster-wrapper');
18
38
  const MessageBus = require('./message-bus');
19
39
  const Ledger = require('./ledger');
20
- const GitHub = require('./github');
40
+ const InputHelpers = require('./input-helpers');
41
+ const { detectProvider } = require('./issue-providers');
21
42
  const IsolationManager = require('./isolation-manager');
22
43
  const { generateName } = require('./name-generator');
23
44
  const configValidator = require('./config-validator');
24
45
  const TemplateResolver = require('./template-resolver');
46
+ const { loadSettings } = require('../lib/settings');
47
+ const { normalizeProviderName } = require('../lib/provider-names');
48
+ const crypto = require('crypto');
49
+
50
+ function applyModelOverride(agentConfig, modelOverride) {
51
+ if (!modelOverride) return;
52
+
53
+ agentConfig.model = modelOverride;
54
+ if (agentConfig.modelRules) {
55
+ delete agentConfig.modelRules;
56
+ }
57
+ if (agentConfig.modelConfig) {
58
+ delete agentConfig.modelConfig;
59
+ }
60
+ }
25
61
 
26
62
  /**
27
63
  * Operation Chain Schema
@@ -66,10 +102,23 @@ class Orchestrator {
66
102
  // Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
67
103
  this.closed = false;
68
104
 
69
- // Load existing clusters from disk (skip if explicitly disabled)
105
+ // Track if clusters are loaded (for lazy loading pattern)
106
+ this._clustersLoaded = options.skipLoad === true;
107
+ }
108
+
109
+ /**
110
+ * Factory method for async initialization
111
+ * Use this instead of `new Orchestrator()` for proper async cluster loading
112
+ * @param {Object} options - Same options as constructor
113
+ * @returns {Promise<Orchestrator>}
114
+ */
115
+ static async create(options = {}) {
116
+ const instance = new Orchestrator({ ...options, skipLoad: true });
70
117
  if (options.skipLoad !== true) {
71
- this._loadClusters();
118
+ await instance._loadClusters();
119
+ instance._clustersLoaded = true;
72
120
  }
121
+ return instance;
73
122
  }
74
123
 
75
124
  /**
@@ -82,12 +131,24 @@ class Orchestrator {
82
131
  }
83
132
  }
84
133
 
134
+ /**
135
+ * Get input source type for metadata
136
+ * @param {Object} input - Input object
137
+ * @returns {string} Source type: 'github', 'file', or 'text'
138
+ * @private
139
+ */
140
+ _getInputSource(input) {
141
+ if (input.issue) return 'github';
142
+ if (input.file) return 'file';
143
+ return 'text';
144
+ }
145
+
85
146
  /**
86
147
  * Load clusters from persistent storage
87
148
  * Uses file locking for consistent reads
88
149
  * @private
89
150
  */
90
- _loadClusters() {
151
+ async _loadClusters() {
91
152
  const clustersFile = path.join(this.storageDir, 'clusters.json');
92
153
  this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
93
154
 
@@ -100,30 +161,20 @@ class Orchestrator {
100
161
  let release;
101
162
 
102
163
  try {
103
- // Acquire lock (sync API doesn't support retries, so we retry manually)
104
- const maxAttempts = 20;
105
- const retryDelayMs = 100;
106
-
107
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
108
- try {
109
- release = lockfile.lockSync(clustersFile, {
110
- lockfilePath,
111
- stale: 30000,
112
- });
113
- break; // Lock acquired
114
- } catch (lockErr) {
115
- if (lockErr.code === 'ELOCKED' && attempt < maxAttempts - 1) {
116
- // Wait and retry
117
- const waitMs = retryDelayMs + Math.random() * retryDelayMs;
118
- const start = Date.now();
119
- while (Date.now() - start < waitMs) {
120
- /* spin wait */
121
- }
122
- continue;
123
- }
124
- throw lockErr;
125
- }
126
- }
164
+ // Clean stale locks from crashed processes
165
+ cleanStaleLock(lockfilePath);
166
+
167
+ // Acquire lock with async API (proper retries without CPU spin-wait)
168
+ release = await lockfile.lock(clustersFile, {
169
+ lockfilePath,
170
+ stale: LOCK_STALE_MS,
171
+ retries: {
172
+ retries: 20,
173
+ minTimeout: 100,
174
+ maxTimeout: 200,
175
+ randomize: true,
176
+ },
177
+ });
127
178
 
128
179
  const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
129
180
  const clusterIds = Object.keys(data);
@@ -138,7 +189,9 @@ class Orchestrator {
138
189
  // Skip clusters whose .db file doesn't exist (orphaned registry entries)
139
190
  const dbPath = path.join(this.storageDir, `${clusterId}.db`);
140
191
  if (!fs.existsSync(dbPath)) {
141
- console.warn(`[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`);
192
+ console.warn(
193
+ `[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
194
+ );
142
195
  clustersToRemove.push(clusterId);
143
196
  continue;
144
197
  }
@@ -152,8 +205,12 @@ class Orchestrator {
152
205
  const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
153
206
  if (messageCount === 0) {
154
207
  console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
155
- console.warn(`[Orchestrator] This likely occurred from SIGINT during initialization.`);
156
- console.warn(`[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`);
208
+ console.warn(
209
+ `[Orchestrator] This likely occurred from SIGINT during initialization.`
210
+ );
211
+ console.warn(
212
+ `[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
213
+ );
157
214
  corruptedClusters.push(clusterId);
158
215
  // Mark cluster as corrupted for visibility in status/list commands
159
216
  cluster.state = 'corrupted';
@@ -168,12 +225,16 @@ class Orchestrator {
168
225
  delete data[clusterId];
169
226
  }
170
227
  fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
171
- this._log(`[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`);
228
+ this._log(
229
+ `[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
230
+ );
172
231
  }
173
232
 
174
233
  // Log summary of corrupted clusters
175
234
  if (corruptedClusters.length > 0) {
176
- console.warn(`\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`);
235
+ console.warn(
236
+ `\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
237
+ );
177
238
  for (const clusterId of corruptedClusters) {
178
239
  console.warn(` - ${clusterId}`);
179
240
  }
@@ -186,7 +247,7 @@ class Orchestrator {
186
247
  console.error(error.stack);
187
248
  } finally {
188
249
  if (release) {
189
- release();
250
+ await release();
190
251
  }
191
252
  }
192
253
  }
@@ -207,8 +268,37 @@ class Orchestrator {
207
268
  const messageBus = new MessageBus(ledger);
208
269
 
209
270
  // Restore isolation manager FIRST if cluster was running in isolation mode
271
+ const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
272
+
273
+ // Reconstruct agent metadata from config (processes are ephemeral)
274
+ // CRITICAL: Pass isolation context to agents if cluster was running in isolation
275
+ const agents = this._rebuildClusterAgents(
276
+ clusterId,
277
+ clusterData,
278
+ messageBus,
279
+ isolation,
280
+ isolationManager
281
+ );
282
+
283
+ const cluster = {
284
+ ...clusterData,
285
+ ledger,
286
+ messageBus,
287
+ agents,
288
+ isolation,
289
+ autoPr: clusterData.autoPr || false,
290
+ };
291
+
292
+ this.clusters.set(clusterId, cluster);
293
+ this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
294
+
295
+ return cluster;
296
+ }
297
+
298
+ _restoreClusterIsolation(clusterId, clusterData) {
210
299
  let isolation = clusterData.isolation || null;
211
300
  let isolationManager = null;
301
+
212
302
  if (isolation?.enabled && isolation.containerId) {
213
303
  isolationManager = new IsolationManager({ image: isolation.image });
214
304
  // Restore the container mapping so cleanup works
@@ -229,74 +319,84 @@ class Orchestrator {
229
319
  );
230
320
  }
231
321
 
232
- // Reconstruct agent metadata from config (processes are ephemeral)
233
- // CRITICAL: Pass isolation context to agents if cluster was running in isolation
234
- const agents = [];
322
+ return { isolation, isolationManager };
323
+ }
235
324
 
236
- // CRITICAL: Determine the correct cwd for agents (worktree > isolation > saved cwd)
237
- // This fixes clusters saved before the cwd injection bug was fixed
325
+ _resolveAgentCwd(clusterData) {
238
326
  const worktreePath = clusterData.worktree?.path;
239
327
  const isolationWorkDir = clusterData.isolation?.workDir;
240
- const agentCwd = worktreePath || isolationWorkDir || null;
241
-
242
- if (clusterData.config?.agents) {
243
- for (const agentConfig of clusterData.config.agents) {
244
- // Fix agents that were saved without cwd (pre-bugfix clusters)
245
- if (!agentConfig.cwd && agentCwd) {
246
- agentConfig.cwd = agentCwd;
247
- this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
248
- }
328
+ return worktreePath || isolationWorkDir || null;
329
+ }
249
330
 
250
- const agentOptions = {
251
- id: clusterId,
252
- quiet: this.quiet,
253
- };
331
+ _buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
332
+ const agentOptions = {
333
+ id: clusterId,
334
+ quiet: this.quiet,
335
+ modelOverride: clusterData.modelOverride || null,
336
+ };
254
337
 
255
- // Inject isolation context if enabled (MUST be done during agent creation)
256
- if (isolation?.enabled && isolationManager) {
257
- agentOptions.isolation = {
258
- enabled: true,
259
- manager: isolationManager,
260
- clusterId,
261
- };
262
- }
338
+ if (isolation?.enabled && isolationManager) {
339
+ agentOptions.isolation = {
340
+ enabled: true,
341
+ manager: isolationManager,
342
+ clusterId,
343
+ };
344
+ }
263
345
 
264
- // Create agent or subcluster wrapper based on type
265
- let agent;
266
- if (agentConfig.type === 'subcluster') {
267
- agent = new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
268
- } else {
269
- agent = new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
270
- }
346
+ return agentOptions;
347
+ }
271
348
 
272
- // Restore agent runtime state if available (for accurate status display)
273
- if (clusterData.agentStates) {
274
- const savedState = clusterData.agentStates.find((s) => s.id === agentConfig.id);
275
- if (savedState) {
276
- agent.state = savedState.state || 'idle';
277
- agent.iteration = savedState.iteration || 0;
278
- agent.currentTask = savedState.currentTask || false;
279
- agent.currentTaskId = savedState.currentTaskId || null;
280
- agent.processPid = savedState.processPid || null;
281
- }
282
- }
349
+ _instantiateAgent(agentConfig, messageBus, clusterId, agentOptions) {
350
+ if (agentConfig.type === 'subcluster') {
351
+ return new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
352
+ }
283
353
 
284
- agents.push(agent);
285
- }
354
+ return new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
355
+ }
356
+
357
+ _restoreAgentState(agent, agentConfig, clusterData) {
358
+ if (!clusterData.agentStates) return;
359
+
360
+ const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
361
+ if (!savedState) return;
362
+
363
+ agent.state = savedState.state || 'idle';
364
+ agent.iteration = savedState.iteration || 0;
365
+ agent.currentTask = savedState.currentTask || false;
366
+ agent.currentTaskId = savedState.currentTaskId || null;
367
+ agent.processPid = savedState.processPid || null;
368
+ }
369
+
370
+ _rebuildClusterAgents(clusterId, clusterData, messageBus, isolation, isolationManager) {
371
+ const agents = [];
372
+ const agentCwd = this._resolveAgentCwd(clusterData);
373
+
374
+ if (!clusterData.config?.agents) {
375
+ return agents;
286
376
  }
287
377
 
288
- const cluster = {
289
- ...clusterData,
290
- ledger,
291
- messageBus,
292
- agents,
293
- isolation,
294
- };
378
+ for (const agentConfig of clusterData.config.agents) {
379
+ if (!agentConfig.cwd && agentCwd) {
380
+ agentConfig.cwd = agentCwd;
381
+ this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
382
+ }
295
383
 
296
- this.clusters.set(clusterId, cluster);
297
- this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
384
+ if (clusterData.modelOverride) {
385
+ applyModelOverride(agentConfig, clusterData.modelOverride);
386
+ }
298
387
 
299
- return cluster;
388
+ const agentOptions = this._buildAgentOptions(
389
+ clusterId,
390
+ clusterData,
391
+ isolation,
392
+ isolationManager
393
+ );
394
+ const agent = this._instantiateAgent(agentConfig, messageBus, clusterId, agentOptions);
395
+ this._restoreAgentState(agent, agentConfig, clusterData);
396
+ agents.push(agent);
397
+ }
398
+
399
+ return agents;
300
400
  }
301
401
 
302
402
  /**
@@ -316,7 +416,7 @@ class Orchestrator {
316
416
  * Uses file locking to prevent race conditions with other processes
317
417
  * @private
318
418
  */
319
- _saveClusters() {
419
+ async _saveClusters() {
320
420
  // Skip saving if orchestrator is closed (prevents race conditions during cleanup)
321
421
  if (this.closed) {
322
422
  return;
@@ -327,30 +427,20 @@ class Orchestrator {
327
427
  let release;
328
428
 
329
429
  try {
330
- // Acquire exclusive lock (sync API doesn't support retries, so we retry manually)
331
- const maxAttempts = 50;
332
- const retryDelayMs = 100;
333
-
334
- for (let attempt = 0; attempt < maxAttempts; attempt++) {
335
- try {
336
- release = lockfile.lockSync(clustersFile, {
337
- lockfilePath,
338
- stale: 30000, // Lock expires after 30s (in case process dies)
339
- });
340
- break; // Lock acquired
341
- } catch (lockErr) {
342
- if (lockErr.code === 'ELOCKED' && attempt < maxAttempts - 1) {
343
- // Wait and retry with jitter
344
- const waitMs = retryDelayMs + Math.random() * retryDelayMs * 2;
345
- const start = Date.now();
346
- while (Date.now() - start < waitMs) {
347
- /* spin wait */
348
- }
349
- continue;
350
- }
351
- throw lockErr;
352
- }
353
- }
430
+ // Clean stale locks from crashed processes
431
+ cleanStaleLock(lockfilePath);
432
+
433
+ // Acquire exclusive lock with async API (proper retries without CPU spin-wait)
434
+ release = await lockfile.lock(clustersFile, {
435
+ lockfilePath,
436
+ stale: LOCK_STALE_MS,
437
+ retries: {
438
+ retries: 50,
439
+ minTimeout: 100,
440
+ maxTimeout: 300,
441
+ randomize: true,
442
+ },
443
+ });
354
444
 
355
445
  // Read existing clusters from file (other processes may have added clusters)
356
446
  let existingClusters = {};
@@ -391,6 +481,10 @@ class Orchestrator {
391
481
  pid: cluster.state === 'running' ? cluster.pid : null,
392
482
  // Persist failure info for resume capability
393
483
  failureInfo: cluster.failureInfo || null,
484
+ // Persist PR mode for completion agent selection
485
+ autoPr: cluster.autoPr || false,
486
+ // Persist model override for consistent agent spawning on resume
487
+ modelOverride: cluster.modelOverride || null,
394
488
  // Persist isolation info (excluding manager instance which can't be serialized)
395
489
  // CRITICAL: workDir is required for resume() to recreate container with same workspace
396
490
  isolation: cluster.isolation
@@ -423,7 +517,7 @@ class Orchestrator {
423
517
  } finally {
424
518
  // Always release lock
425
519
  if (release) {
426
- release();
520
+ await release();
427
521
  }
428
522
  }
429
523
  }
@@ -445,11 +539,14 @@ class Orchestrator {
445
539
  try {
446
540
  if (!fs.existsSync(clustersFile)) return;
447
541
 
542
+ // Clean stale locks from crashed processes
543
+ cleanStaleLock(lockfilePath);
544
+
448
545
  // Try to acquire lock once (polling is best-effort, will retry on next cycle)
449
546
  try {
450
547
  release = lockfile.lockSync(clustersFile, {
451
548
  lockfilePath,
452
- stale: 30000,
549
+ stale: LOCK_STALE_MS,
453
550
  });
454
551
  } catch (lockErr) {
455
552
  // Lock busy - skip this poll cycle, try again next interval
@@ -541,7 +638,11 @@ class Orchestrator {
541
638
  isolation: options.isolation || false,
542
639
  isolationImage: options.isolationImage,
543
640
  worktree: options.worktree || false,
544
- autoPr: process.env.ZEROSHOT_PR === '1',
641
+ autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
642
+ modelOverride: options.modelOverride, // Model override for all agents
643
+ clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
644
+ settings: options.settings, // User settings for issue provider detection
645
+ forceProvider: options.forceProvider, // Force specific issue provider
545
646
  });
546
647
  }
547
648
 
@@ -550,8 +651,15 @@ class Orchestrator {
550
651
  * @private
551
652
  */
552
653
  async _startInternal(config, input = {}, options = {}) {
553
- // Use pre-generated ID from parent process, or generate new one
554
- const clusterId = process.env.ZEROSHOT_CLUSTER_ID || generateName('cluster');
654
+ // Generate a unique cluster ID for this process call.
655
+ // IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
656
+ // - test harnesses may set it globally (breaking multi-start tests)
657
+ // - callers may start multiple clusters in one process
658
+ // Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
659
+ const clusterId = this._generateUniqueClusterId(
660
+ options.clusterId || null,
661
+ config?.dbPath || null
662
+ );
555
663
 
556
664
  // Create ledger and message bus with persistent storage
557
665
  const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
@@ -559,46 +667,11 @@ class Orchestrator {
559
667
  const messageBus = new MessageBus(ledger);
560
668
 
561
669
  // Handle isolation mode (Docker container OR git worktree)
562
- let isolationManager = null;
563
- let containerId = null;
564
- let worktreeInfo = null;
565
-
566
- if (options.isolation) {
567
- // Check Docker availability
568
- if (!IsolationManager.isDockerAvailable()) {
569
- throw new Error('Docker is not available. Install Docker to use --docker mode.');
570
- }
571
-
572
- // Ensure image exists (auto-build if missing)
573
- const image = options.isolationImage || 'zeroshot-cluster-base';
574
- await IsolationManager.ensureImage(image);
575
-
576
- isolationManager = new IsolationManager({ image });
577
- this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
578
-
579
- // Create container with workspace mounted
580
- // CRITICAL: Use options.cwd (git repo root) instead of process.cwd()
581
- const workDir = options.cwd || process.cwd();
582
- containerId = await isolationManager.createContainer(clusterId, {
583
- workDir,
584
- image,
585
- // Mount configuration (CLI overrides)
586
- noMounts: options.noMounts,
587
- mounts: options.mounts,
588
- containerHome: options.containerHome,
589
- });
590
- this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
591
- } else if (options.worktree) {
592
- // Worktree isolation: lightweight git-based isolation (no Docker required)
593
- const workDir = options.cwd || process.cwd();
594
-
595
- isolationManager = new IsolationManager({});
596
- worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
597
-
598
- this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
599
- this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
600
- this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
601
- }
670
+ const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
671
+ options,
672
+ config,
673
+ clusterId
674
+ );
602
675
 
603
676
  // Build cluster object
604
677
  // CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
@@ -621,6 +694,13 @@ class Orchestrator {
621
694
  // Initialization completion tracking (for safe SIGINT handling)
622
695
  initCompletePromise,
623
696
  _resolveInitComplete: resolveInitComplete,
697
+ autoPr: options.autoPr || false,
698
+ // Model override for all agents (applied to dynamically added agents)
699
+ modelOverride: options.modelOverride || null,
700
+ // Issue provider tracking (where issue was fetched from)
701
+ issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
702
+ // Git platform tracking (where PR/MR will be created - independent of issue provider)
703
+ gitPlatform: null, // Set when --pr mode is active
624
704
  // Isolation state (only if enabled)
625
705
  // CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
626
706
  isolation: options.isolation
@@ -648,122 +728,67 @@ class Orchestrator {
648
728
  this.clusters.set(clusterId, cluster);
649
729
 
650
730
  try {
651
- // Fetch input (GitHub issue or text)
731
+ // Fetch input (issue from provider, file, or text)
652
732
  let inputData;
653
733
  if (input.issue) {
654
- inputData = await GitHub.fetchIssue(input.issue);
734
+ // Detect provider and fetch issue
735
+ const ProviderClass = detectProvider(
736
+ input.issue,
737
+ options.settings || {},
738
+ options.forceProvider
739
+ );
740
+ if (!ProviderClass) {
741
+ throw new Error(`No issue provider matched input: ${input.issue}`);
742
+ }
743
+
744
+ const provider = new ProviderClass();
745
+ inputData = await provider.fetchIssue(input.issue, options.settings || {});
746
+
747
+ // Store issue provider for logging/debugging and cross-provider workflows
748
+ cluster.issueProvider = ProviderClass.id;
749
+
655
750
  // Log clickable issue link
656
751
  if (inputData.url) {
657
- this._log(`[Orchestrator] Issue: ${inputData.url}`);
752
+ this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
658
753
  }
754
+ } else if (input.file) {
755
+ inputData = InputHelpers.createFileInput(input.file);
756
+ this._log(`[Orchestrator] File: ${input.file}`);
659
757
  } else if (input.text) {
660
- inputData = GitHub.createTextInput(input.text);
758
+ inputData = InputHelpers.createTextInput(input.text);
661
759
  } else {
662
- throw new Error('Either issue or text input is required');
760
+ throw new Error('Either issue, file, or text input is required');
663
761
  }
664
762
 
665
- // Inject git-pusher agent if --pr is set (replaces completion-detector)
763
+ // Detect git platform for --pr mode (independent of issue provider)
666
764
  if (options.autoPr) {
667
- // Remove completion-detector by ID (git-pusher handles completion + PR)
668
- config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
669
-
670
- // Load and configure git-pusher agent (use fs.readFileSync to avoid require cache)
671
- const gitPusherPath = path.join(__dirname, 'agents', 'git-pusher-agent.json');
672
- const gitPusherConfig = JSON.parse(fs.readFileSync(gitPusherPath, 'utf8'));
673
-
674
- // Inject issue context placeholders
675
- gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
676
- /\{\{issue_number\}\}/g,
677
- inputData.number || 'unknown'
678
- );
679
- gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
680
- /\{\{issue_title\}\}/g,
681
- inputData.title || 'Implementation'
682
- );
765
+ const { detectGitContext } = require('../lib/git-remote-utils');
766
+ const gitContext = detectGitContext(options.cwd);
767
+ cluster.gitPlatform = gitContext?.provider || null;
683
768
 
684
- config.agents.push(gitPusherConfig);
685
- this._log(`[Orchestrator] Injected git-pusher agent (creates PR and auto-merges)`);
769
+ if (cluster.gitPlatform) {
770
+ this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
771
+ }
686
772
  }
687
773
 
774
+ // Inject git-pusher agent if --pr is set (replaces completion-detector)
775
+ this._applyAutoPrConfig(config, inputData, options);
776
+
688
777
  // Inject workers instruction if --workers explicitly provided and > 1
689
- const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
690
- if (workersCount > 1) {
691
- const workerAgent = config.agents.find((a) => a.id === 'worker');
692
- if (workerAgent) {
693
- const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
694
-
695
- if (!workerAgent.prompt) {
696
- workerAgent.prompt = instruction;
697
- } else if (typeof workerAgent.prompt === 'string') {
698
- workerAgent.prompt = instruction + workerAgent.prompt;
699
- } else if (workerAgent.prompt.system) {
700
- workerAgent.prompt.system = instruction + workerAgent.prompt.system;
701
- }
702
- this._log(
703
- `[Orchestrator] Injected parallelization instruction (workers=${workersCount})`
704
- );
705
- }
706
- }
778
+ this._applyWorkerInstruction(config);
707
779
 
708
780
  // Initialize agents with optional mock injection
709
781
  // Check agent type: regular agent or subcluster
710
782
  // CRITICAL: Inject cwd into each agent config for proper working directory
711
783
  // In worktree mode, agents run in the worktree path (not original cwd)
712
- const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
713
- for (const agentConfig of config.agents) {
714
- // Inject cwd if not already set (config may override)
715
- if (!agentConfig.cwd) {
716
- agentConfig.cwd = agentCwd;
717
- }
718
-
719
- const agentOptions = {
720
- testMode: options.testMode || !!this.taskRunner, // Enable testMode if taskRunner provided
721
- quiet: this.quiet,
722
- };
723
-
724
- // Inject mock spawn function if provided (legacy mockExecutor API)
725
- if (options.mockExecutor) {
726
- agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
727
- }
728
-
729
- // TaskRunner DI - new pattern for mocking task execution
730
- // Creates a mockSpawnFn wrapper that delegates to the taskRunner
731
- if (this.taskRunner) {
732
- // CRITICAL: agent is a closure variable capturing the AgentWrapper instance
733
- // We cannot access agent._selectModel() here because agent doesn't exist yet
734
- // Solution: Pass a factory function that will be called when agent is available
735
- agentOptions.taskRunner = this.taskRunner;
736
- }
737
-
738
- // Pass isolation context if enabled
739
- if (cluster.isolation) {
740
- agentOptions.isolation = {
741
- enabled: true,
742
- manager: isolationManager,
743
- clusterId,
744
- };
745
- }
746
-
747
- // Pass worktree context if enabled (lightweight isolation without Docker)
748
- if (cluster.worktree) {
749
- agentOptions.worktree = {
750
- enabled: true,
751
- path: cluster.worktree.path,
752
- branch: cluster.worktree.branch,
753
- repoRoot: cluster.worktree.repoRoot,
754
- };
755
- }
756
-
757
- // Create agent or subcluster wrapper based on type
758
- let agent;
759
- if (agentConfig.type === 'subcluster') {
760
- agent = new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions);
761
- } else {
762
- agent = new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
763
- }
764
-
765
- cluster.agents.push(agent);
766
- }
784
+ this._initializeClusterAgents({
785
+ config,
786
+ cluster,
787
+ messageBus,
788
+ options,
789
+ isolationManager,
790
+ clusterId,
791
+ });
767
792
 
768
793
  // ========================================================================
769
794
  // CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
@@ -777,167 +802,17 @@ class Orchestrator {
777
802
  //
778
803
  // ORDER:
779
804
  // 1. Register subscriptions (lines below)
780
- // 2. Start agents (line ~XXX)
781
- // 3. Publish ISSUE_OPENED (line ~XXX)
805
+ // 2. Start agents
806
+ // 3. Publish ISSUE_OPENED
782
807
  //
783
808
  // DO NOT move subscriptions after agent.start() - this will reintroduce
784
809
  // the race condition fixed in issue #31.
785
810
  // ========================================================================
786
-
787
- // Helper: Subscribe to cluster-scoped topic
788
- const subscribeToClusterTopic = (topic, handler) => {
789
- messageBus.subscribe((message) => {
790
- if (message.topic === topic && message.cluster_id === clusterId) {
791
- handler(message);
792
- }
793
- });
794
- };
795
-
796
- // Watch for CLUSTER_COMPLETE message to auto-stop
797
- subscribeToClusterTopic('CLUSTER_COMPLETE', (message) => {
798
- this._log(`\n${'='.repeat(80)}`);
799
- this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
800
- this._log(`${'='.repeat(80)}`);
801
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
802
- this._log(`Initiated by: ${message.sender}`);
803
- this._log(`${'='.repeat(80)}\n`);
804
-
805
- // Auto-stop cluster
806
- this.stop(clusterId).catch((err) => {
807
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
808
- });
809
- });
810
-
811
- // Watch for CLUSTER_FAILED message to auto-stop
812
- subscribeToClusterTopic('CLUSTER_FAILED', (message) => {
813
- this._log(`\n${'='.repeat(80)}`);
814
- this._log(`❌ CLUSTER FAILED: ${clusterId}`);
815
- this._log(`${'='.repeat(80)}`);
816
- this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
817
- this._log(`Agent: ${message.sender}`);
818
- if (message.content?.text) {
819
- this._log(`Details: ${message.content.text}`);
820
- }
821
- this._log(`${'='.repeat(80)}\n`);
822
-
823
- // Auto-stop cluster
824
- this.stop(clusterId).catch((err) => {
825
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
826
- });
827
- });
828
-
829
- // Watch for AGENT_ERROR - if critical agent fails, stop cluster
830
- subscribeToClusterTopic('AGENT_ERROR', (message) => {
831
- const agentRole = message.content?.data?.role;
832
- const attempts = message.content?.data?.attempts || 1;
833
-
834
- // Save cluster state to persist failureInfo
835
- this._saveClusters();
836
-
837
- // Only stop cluster if non-validator agent exhausted retries
838
- if (agentRole === 'implementation' && attempts >= 3) {
839
- this._log(`\n${'='.repeat(80)}`);
840
- this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
841
- this._log(`${'='.repeat(80)}`);
842
- this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
843
- this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
844
- this._log(`Stopping cluster - worker cannot continue`);
845
- this._log(`${'='.repeat(80)}\n`);
846
-
847
- // Auto-stop cluster
848
- this.stop(clusterId).catch((err) => {
849
- console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
850
- });
851
- }
852
- });
853
-
854
- // Persist agent state changes for accurate status display
855
- messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
856
- const event = message.content?.data?.event;
857
- // Save on key state transitions that affect status display
858
- if (
859
- ['TASK_STARTED', 'TASK_COMPLETED', 'PROCESS_SPAWNED', 'TASK_ID_ASSIGNED', 'STARTED'].includes(
860
- event
861
- )
862
- ) {
863
- this._saveClusters();
864
- }
865
- });
866
-
867
- // Watch for stale agent detection (informational only)
868
- messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
869
- if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
870
-
871
- const agentId = message.content?.data?.agent;
872
- const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
873
- const analysis = message.content?.data?.analysis || 'No analysis available';
874
-
875
- this._log(
876
- `⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
877
- );
878
- this._log(` Analysis: ${analysis}`);
879
- this._log(` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`);
880
- });
881
-
882
- // Watch for CLUSTER_OPERATIONS - dynamic agent spawn/removal/update
883
- subscribeToClusterTopic('CLUSTER_OPERATIONS', (message) => {
884
- let operations = message.content?.data?.operations;
885
-
886
- // Parse operations if they came as a JSON string
887
- if (typeof operations === 'string') {
888
- try {
889
- operations = JSON.parse(operations);
890
- } catch (e) {
891
- this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
892
- return;
893
- }
894
- }
895
-
896
- if (!operations || !Array.isArray(operations)) {
897
- this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
898
- return;
899
- }
900
-
901
- this._log(`\n${'='.repeat(80)}`);
902
- this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
903
- this._log(`${'='.repeat(80)}`);
904
- if (message.content?.data?.reasoning) {
905
- this._log(`Reasoning: ${message.content.data.reasoning}`);
906
- }
907
- this._log(`Operations: ${operations.length}`);
908
- this._log(`${'='.repeat(80)}\n`);
909
-
910
- // Execute operation chain
911
- this._handleOperations(clusterId, operations, message.sender, {
912
- isolationManager,
913
- containerId,
914
- }).catch((err) => {
915
- console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
916
- // Publish failure message
917
- messageBus.publish({
918
- cluster_id: clusterId,
919
- topic: 'CLUSTER_OPERATIONS_FAILED',
920
- sender: 'orchestrator',
921
- content: {
922
- text: `Operation chain failed: ${err.message}`,
923
- data: {
924
- error: err.message,
925
- operations: operations,
926
- },
927
- },
928
- });
929
-
930
- // CRITICAL: Stop cluster on operation failure
931
- this._log(`\n${'='.repeat(80)}`);
932
- this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
933
- this._log(`${'='.repeat(80)}`);
934
- this._log(`Error: ${err.message}`);
935
- this._log(`${'='.repeat(80)}\n`);
936
-
937
- this.stop(clusterId).catch((stopErr) => {
938
- console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
939
- });
940
- });
811
+ this._registerClusterSubscriptions({
812
+ messageBus,
813
+ clusterId,
814
+ isolationManager,
815
+ containerId,
941
816
  });
942
817
 
943
818
  // Start all agents
@@ -961,7 +836,7 @@ class Orchestrator {
961
836
  },
962
837
  },
963
838
  metadata: {
964
- source: input.issue ? 'github' : 'text',
839
+ source: this._getInputSource(input),
965
840
  },
966
841
  });
967
842
 
@@ -989,7 +864,7 @@ class Orchestrator {
989
864
  // ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
990
865
 
991
866
  // Save cluster to disk
992
- this._saveClusters();
867
+ await this._saveClusters();
993
868
 
994
869
  return {
995
870
  id: clusterId,
@@ -1009,10 +884,485 @@ class Orchestrator {
1009
884
  }
1010
885
  }
1011
886
 
1012
- /**
1013
- * Stop a cluster
1014
- * @param {String} clusterId - Cluster ID
1015
- */
887
+ _initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
888
+ const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
889
+
890
+ for (const agentConfig of config.agents) {
891
+ if (!agentConfig.cwd) {
892
+ agentConfig.cwd = agentCwd;
893
+ }
894
+
895
+ if (options.modelOverride) {
896
+ applyModelOverride(agentConfig, options.modelOverride);
897
+ this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
898
+ }
899
+
900
+ const agentOptions = {
901
+ testMode: options.testMode || !!this.taskRunner,
902
+ quiet: this.quiet,
903
+ modelOverride: options.modelOverride || null,
904
+ };
905
+
906
+ if (options.mockExecutor) {
907
+ agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
908
+ }
909
+
910
+ if (this.taskRunner) {
911
+ agentOptions.taskRunner = this.taskRunner;
912
+ }
913
+
914
+ if (cluster.isolation) {
915
+ agentOptions.isolation = {
916
+ enabled: true,
917
+ manager: isolationManager,
918
+ clusterId,
919
+ };
920
+ }
921
+
922
+ if (cluster.worktree) {
923
+ agentOptions.worktree = {
924
+ enabled: true,
925
+ path: cluster.worktree.path,
926
+ branch: cluster.worktree.branch,
927
+ repoRoot: cluster.worktree.repoRoot,
928
+ };
929
+ }
930
+
931
+ const agent =
932
+ agentConfig.type === 'subcluster'
933
+ ? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
934
+ : new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
935
+
936
+ cluster.agents.push(agent);
937
+ }
938
+ }
939
+
940
+ _subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
941
+ messageBus.subscribe((message) => {
942
+ if (message.topic === topic && message.cluster_id === clusterId) {
943
+ handler(message);
944
+ }
945
+ });
946
+ }
947
+
948
+ _registerClusterCompletionHandlers(messageBus, clusterId) {
949
+ this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
950
+ this._log(`\n${'='.repeat(80)}`);
951
+ this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
952
+ this._log(`${'='.repeat(80)}`);
953
+ this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
954
+ this._log(`Initiated by: ${message.sender}`);
955
+ this._log(`${'='.repeat(80)}\n`);
956
+
957
+ this.stop(clusterId).catch((err) => {
958
+ console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
959
+ });
960
+ });
961
+
962
+ this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
963
+ this._log(`\n${'='.repeat(80)}`);
964
+ this._log(`❌ CLUSTER FAILED: ${clusterId}`);
965
+ this._log(`${'='.repeat(80)}`);
966
+ this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
967
+ this._log(`Agent: ${message.sender}`);
968
+ if (message.content?.text) {
969
+ this._log(`Details: ${message.content.text}`);
970
+ }
971
+ this._log(`${'='.repeat(80)}\n`);
972
+
973
+ this.stop(clusterId).catch((err) => {
974
+ console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
975
+ });
976
+ });
977
+ }
978
+
979
+ _registerAgentErrorHandler(messageBus, clusterId) {
980
+ this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
981
+ const agentRole = message.content?.data?.role;
982
+ const attempts = message.content?.data?.attempts || 1;
983
+
984
+ await this._saveClusters();
985
+
986
+ if (agentRole === 'implementation' && attempts >= 3) {
987
+ this._log(`\n${'='.repeat(80)}`);
988
+ this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
989
+ this._log(`${'='.repeat(80)}`);
990
+ this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
991
+ this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
992
+ this._log(`Stopping cluster - worker cannot continue`);
993
+ this._log(`${'='.repeat(80)}\n`);
994
+
995
+ this.stop(clusterId).catch((err) => {
996
+ console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
997
+ });
998
+ }
999
+ });
1000
+ }
1001
+
1002
+ _registerAgentLifecycleHandlers(messageBus, clusterId) {
1003
+ messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
1004
+ const event = message.content?.data?.event;
1005
+ if (
1006
+ [
1007
+ 'TASK_STARTED',
1008
+ 'TASK_COMPLETED',
1009
+ 'PROCESS_SPAWNED',
1010
+ 'TASK_ID_ASSIGNED',
1011
+ 'STARTED',
1012
+ ].includes(event)
1013
+ ) {
1014
+ await this._saveClusters();
1015
+ }
1016
+ });
1017
+
1018
+ messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
1019
+ if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
1020
+
1021
+ const agentId = message.content?.data?.agent;
1022
+ const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
1023
+ const analysis = message.content?.data?.analysis || 'No analysis available';
1024
+
1025
+ this._log(
1026
+ `⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
1027
+ );
1028
+ this._log(` Analysis: ${analysis}`);
1029
+ this._log(
1030
+ ` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
1031
+ );
1032
+ });
1033
+ }
1034
+
1035
+ _registerConductorWatchdog(messageBus, clusterId) {
1036
+ const timeoutMs = 30000;
1037
+ let watchdogTimer = null;
1038
+ let completedAt = null;
1039
+
1040
+ this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
1041
+ const event = message.content?.data?.event;
1042
+ const role = message.content?.data?.role;
1043
+
1044
+ if (event === 'TASK_COMPLETED' && role === 'conductor') {
1045
+ completedAt = Date.now();
1046
+ this._log(
1047
+ `⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
1048
+ );
1049
+
1050
+ watchdogTimer = setTimeout(() => {
1051
+ const clusterOps = messageBus.query({
1052
+ cluster_id: clusterId,
1053
+ topic: 'CLUSTER_OPERATIONS',
1054
+ limit: 1,
1055
+ });
1056
+ if (clusterOps.length === 0) {
1057
+ console.error(`\n${'='.repeat(80)}`);
1058
+ console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
1059
+ console.error(`${'='.repeat(80)}`);
1060
+ console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
1061
+ console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
1062
+ console.error(
1063
+ `Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
1064
+ );
1065
+ console.error(`${'='.repeat(80)}\n`);
1066
+
1067
+ messageBus.publish({
1068
+ cluster_id: clusterId,
1069
+ topic: 'CLUSTER_FAILED',
1070
+ sender: 'orchestrator',
1071
+ content: {
1072
+ text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
1073
+ data: {
1074
+ reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
1075
+ conductorCompletedAt: completedAt,
1076
+ timeoutMs: timeoutMs,
1077
+ },
1078
+ },
1079
+ });
1080
+ }
1081
+ }, timeoutMs);
1082
+ }
1083
+ });
1084
+
1085
+ return {
1086
+ clear: () => {
1087
+ if (!watchdogTimer) {
1088
+ return;
1089
+ }
1090
+ clearTimeout(watchdogTimer);
1091
+ watchdogTimer = null;
1092
+ const elapsed = completedAt ? Date.now() - completedAt : 0;
1093
+ this._log(
1094
+ `✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
1095
+ );
1096
+ },
1097
+ };
1098
+ }
1099
+
1100
+ _registerClusterOperationsHandler(
1101
+ messageBus,
1102
+ clusterId,
1103
+ isolationManager,
1104
+ containerId,
1105
+ watchdog
1106
+ ) {
1107
+ this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
1108
+ watchdog?.clear?.();
1109
+
1110
+ let operations = message.content?.data?.operations;
1111
+ if (typeof operations === 'string') {
1112
+ try {
1113
+ operations = JSON.parse(operations);
1114
+ } catch (e) {
1115
+ this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
1116
+ return;
1117
+ }
1118
+ }
1119
+
1120
+ if (!operations || !Array.isArray(operations)) {
1121
+ this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
1122
+ return;
1123
+ }
1124
+
1125
+ this._log(`\n${'='.repeat(80)}`);
1126
+ this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
1127
+ this._log(`${'='.repeat(80)}`);
1128
+ if (message.content?.data?.reasoning) {
1129
+ this._log(`Reasoning: ${message.content.data.reasoning}`);
1130
+ }
1131
+ this._log(`Operations: ${operations.length}`);
1132
+ this._log(`${'='.repeat(80)}\n`);
1133
+
1134
+ this._handleOperations(clusterId, operations, message.sender, {
1135
+ isolationManager,
1136
+ containerId,
1137
+ }).catch((err) => {
1138
+ console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
1139
+ messageBus.publish({
1140
+ cluster_id: clusterId,
1141
+ topic: 'CLUSTER_OPERATIONS_FAILED',
1142
+ sender: 'orchestrator',
1143
+ content: {
1144
+ text: `Operation chain failed: ${err.message}`,
1145
+ data: {
1146
+ error: err.message,
1147
+ operations: operations,
1148
+ },
1149
+ },
1150
+ });
1151
+
1152
+ this._log(`\n${'='.repeat(80)}`);
1153
+ this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
1154
+ this._log(`${'='.repeat(80)}`);
1155
+ this._log(`Error: ${err.message}`);
1156
+ this._log(`${'='.repeat(80)}\n`);
1157
+
1158
+ this.stop(clusterId).catch((stopErr) => {
1159
+ console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
1160
+ });
1161
+ });
1162
+ });
1163
+ }
1164
+
1165
+ _registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
1166
+ this._registerClusterCompletionHandlers(messageBus, clusterId);
1167
+ this._registerAgentErrorHandler(messageBus, clusterId);
1168
+ this._registerAgentLifecycleHandlers(messageBus, clusterId);
1169
+
1170
+ const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
1171
+ this._registerClusterOperationsHandler(
1172
+ messageBus,
1173
+ clusterId,
1174
+ isolationManager,
1175
+ containerId,
1176
+ watchdog
1177
+ );
1178
+ }
1179
+
1180
+ async _initializeIsolation(options, config, clusterId) {
1181
+ let isolationManager = null;
1182
+ let containerId = null;
1183
+ let worktreeInfo = null;
1184
+
1185
+ if (options.isolation) {
1186
+ if (!IsolationManager.isDockerAvailable()) {
1187
+ throw new Error('Docker is not available. Install Docker to use --docker mode.');
1188
+ }
1189
+
1190
+ const image = options.isolationImage || 'zeroshot-cluster-base';
1191
+ await IsolationManager.ensureImage(image);
1192
+
1193
+ isolationManager = new IsolationManager({ image });
1194
+ this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
1195
+
1196
+ const workDir = options.cwd || process.cwd();
1197
+ const providerName = normalizeProviderName(
1198
+ config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
1199
+ );
1200
+ containerId = await isolationManager.createContainer(clusterId, {
1201
+ workDir,
1202
+ image,
1203
+ noMounts: options.noMounts,
1204
+ mounts: options.mounts,
1205
+ containerHome: options.containerHome,
1206
+ provider: providerName,
1207
+ });
1208
+ this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
1209
+ } else if (options.worktree) {
1210
+ const workDir = options.cwd || process.cwd();
1211
+
1212
+ isolationManager = new IsolationManager({});
1213
+ worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
1214
+
1215
+ this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
1216
+ this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
1217
+ this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
1218
+ }
1219
+
1220
+ return { isolationManager, containerId, worktreeInfo };
1221
+ }
1222
+
1223
+ _applyAutoPrConfig(config, inputData, options) {
1224
+ if (!options.autoPr) {
1225
+ return;
1226
+ }
1227
+
1228
+ config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
1229
+
1230
+ // Detect git platform (independent of issue provider)
1231
+ const { getPlatformForPR } = require('./issue-providers');
1232
+ const { detectGitContext } = require('../lib/git-remote-utils');
1233
+
1234
+ let platform;
1235
+ try {
1236
+ // ALWAYS use git context, regardless of issue provider
1237
+ // This allows Jira issues in GitHub repos, etc.
1238
+ platform = getPlatformForPR(options.cwd);
1239
+ } catch (error) {
1240
+ throw new Error(`--pr mode failed: ${error.message}`);
1241
+ }
1242
+
1243
+ // Check for repo mismatch (issue URL repo vs current git remote)
1244
+ // This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
1245
+ // from a different repo directory
1246
+ let skipCloseIssue = false;
1247
+ const gitContext = detectGitContext(options.cwd);
1248
+ if (inputData.url && gitContext) {
1249
+ const issueHost = this._extractHostFromUrl(inputData.url);
1250
+ const gitHost = gitContext.host;
1251
+
1252
+ // Compare hosts - if different platforms, warn and don't close issue
1253
+ if (issueHost && gitHost && issueHost !== gitHost) {
1254
+ this._log(
1255
+ `[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
1256
+ );
1257
+ this._log(
1258
+ `[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
1259
+ );
1260
+ this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
1261
+ skipCloseIssue = true;
1262
+ }
1263
+ }
1264
+
1265
+ // Generate platform-specific git-pusher agent from template
1266
+ const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
1267
+
1268
+ if (!isPlatformSupported(platform)) {
1269
+ throw new Error(
1270
+ `Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
1271
+ );
1272
+ }
1273
+
1274
+ const gitPusherConfig = generateGitPusherAgent(platform);
1275
+
1276
+ // Template replacement for issue context
1277
+ const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
1278
+ gitPusherConfig.prompt = gitPusherConfig.prompt
1279
+ .replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
1280
+ .replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
1281
+ .replace(/Closes #\{\{issue_number\}\}/g, issueRef);
1282
+
1283
+ config.agents.push(gitPusherConfig);
1284
+ this._log(
1285
+ `[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
1286
+ );
1287
+ }
1288
+
1289
+ /**
1290
+ * Extract hostname from a URL
1291
+ * @private
1292
+ */
1293
+ _extractHostFromUrl(url) {
1294
+ try {
1295
+ const match = url.match(/https?:\/\/([^/]+)/);
1296
+ return match ? match[1] : null;
1297
+ } catch {
1298
+ return null;
1299
+ }
1300
+ }
1301
+
1302
+ _applyWorkerInstruction(config) {
1303
+ const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
1304
+ if (workersCount <= 1) {
1305
+ return;
1306
+ }
1307
+
1308
+ const workerAgent = config.agents.find((a) => a.id === 'worker');
1309
+ if (!workerAgent) {
1310
+ return;
1311
+ }
1312
+
1313
+ const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
1314
+
1315
+ if (!workerAgent.prompt) {
1316
+ workerAgent.prompt = instruction;
1317
+ } else if (typeof workerAgent.prompt === 'string') {
1318
+ workerAgent.prompt = instruction + workerAgent.prompt;
1319
+ } else if (workerAgent.prompt.system) {
1320
+ workerAgent.prompt.system = instruction + workerAgent.prompt.system;
1321
+ }
1322
+
1323
+ this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
1324
+ }
1325
+
1326
+ /**
1327
+ * Generate a unique cluster ID, safe for concurrent starts in-process.
1328
+ * If an explicit ID is provided, uses it as a base and suffixes on collision.
1329
+ * @private
1330
+ */
1331
+ _generateUniqueClusterId(explicitId, explicitDbPath) {
1332
+ const baseId = explicitId || generateName('cluster');
1333
+ const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
1334
+
1335
+ // Fast path: base is unused.
1336
+ if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
1337
+ return baseId;
1338
+ }
1339
+
1340
+ // Collision: suffix with random bytes to avoid race conditions under concurrency.
1341
+ for (let attempt = 0; attempt < 50; attempt++) {
1342
+ const suffix = crypto.randomBytes(3).toString('hex');
1343
+ const candidateId = `${baseId}-${suffix}`;
1344
+ const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1345
+ if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1346
+ return candidateId;
1347
+ }
1348
+ }
1349
+
1350
+ // Last resort: new generated name (should never happen).
1351
+ for (let attempt = 0; attempt < 50; attempt++) {
1352
+ const candidateId = generateName('cluster');
1353
+ const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
1354
+ if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
1355
+ return candidateId;
1356
+ }
1357
+ }
1358
+
1359
+ throw new Error('Failed to generate unique cluster ID after many attempts');
1360
+ }
1361
+
1362
+ /**
1363
+ * Stop a cluster
1364
+ * @param {String} clusterId - Cluster ID
1365
+ */
1016
1366
  async stop(clusterId) {
1017
1367
  const cluster = this.clusters.get(clusterId);
1018
1368
  if (!cluster) {
@@ -1040,7 +1390,9 @@ class Orchestrator {
1040
1390
  // Clean up isolation container if enabled
1041
1391
  // CRITICAL: Preserve workspace for resume capability - only delete on kill()
1042
1392
  if (cluster.isolation?.manager) {
1043
- this._log(`[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`);
1393
+ this._log(
1394
+ `[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
1395
+ );
1044
1396
  await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
1045
1397
  this._log(`[Orchestrator] Container stopped, workspace preserved`);
1046
1398
  }
@@ -1058,7 +1410,7 @@ class Orchestrator {
1058
1410
  this._log(`Cluster ${clusterId} stopped`);
1059
1411
 
1060
1412
  // Save updated state
1061
- this._saveClusters();
1413
+ await this._saveClusters();
1062
1414
  }
1063
1415
 
1064
1416
  /**
@@ -1080,7 +1432,9 @@ class Orchestrator {
1080
1432
 
1081
1433
  // Force remove isolation container AND workspace (full cleanup, no resume)
1082
1434
  if (cluster.isolation?.manager) {
1083
- this._log(`[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`);
1435
+ this._log(
1436
+ `[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
1437
+ );
1084
1438
  await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
1085
1439
  this._log(`[Orchestrator] Container and workspace removed`);
1086
1440
  }
@@ -1104,7 +1458,7 @@ class Orchestrator {
1104
1458
  this._log(`Cluster ${clusterId} killed`);
1105
1459
 
1106
1460
  // Save updated state (will be marked as 'killed' in file)
1107
- this._saveClusters();
1461
+ await this._saveClusters();
1108
1462
 
1109
1463
  // Now remove from memory after persisting
1110
1464
  this.clusters.delete(clusterId);
@@ -1175,271 +1529,303 @@ class Orchestrator {
1175
1529
  );
1176
1530
  }
1177
1531
 
1178
- // Get failure info - either from saved state or from ledger
1179
- let failureInfo = cluster.failureInfo;
1532
+ const failureInfo = this._resolveFailureInfo(cluster, clusterId);
1180
1533
 
1181
- if (!failureInfo) {
1182
- // Query ledger for AGENT_ERROR messages to find failed agent
1183
- const errors = cluster.messageBus.query({
1184
- cluster_id: clusterId,
1185
- topic: 'AGENT_ERROR',
1186
- limit: 10,
1187
- });
1534
+ await this._ensureIsolationForResume(clusterId, cluster);
1535
+ this._ensureWorktreeForResume(clusterId, cluster);
1536
+ await this._restartClusterAgents(cluster);
1188
1537
 
1189
- if (errors.length > 0) {
1190
- // Use the first error found
1191
- const firstError = errors[0];
1192
- failureInfo = {
1193
- agentId: firstError.sender,
1194
- taskId: firstError.content?.data?.taskId || null,
1195
- iteration: firstError.content?.data?.iteration || 0,
1196
- error: firstError.content?.data?.error || firstError.content?.text,
1197
- timestamp: firstError.timestamp,
1198
- };
1199
- this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
1200
- }
1538
+ const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
1539
+
1540
+ if (failureInfo) {
1541
+ return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
1201
1542
  }
1202
1543
 
1203
- // CRITICAL: Recreate isolation container if needed
1204
- if (cluster.isolation?.enabled) {
1205
- const { spawn } = require('child_process');
1206
- const oldContainerId = cluster.isolation.containerId;
1544
+ return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
1545
+ }
1207
1546
 
1208
- // Check if container exists
1209
- const checkContainer = spawn('docker', ['inspect', oldContainerId], {
1210
- stdio: 'ignore',
1211
- });
1212
- const containerExists = await new Promise((resolve) => {
1213
- checkContainer.on('close', (code) => resolve(code === 0));
1214
- });
1547
+ _resolveFailureInfo(cluster, clusterId) {
1548
+ if (cluster.failureInfo) {
1549
+ return cluster.failureInfo;
1550
+ }
1215
1551
 
1216
- if (!containerExists) {
1217
- this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
1552
+ const errors = cluster.messageBus.query({
1553
+ cluster_id: clusterId,
1554
+ topic: 'AGENT_ERROR',
1555
+ limit: 10,
1556
+ order: 'desc',
1557
+ });
1218
1558
 
1219
- // Create new container using saved workDir (CRITICAL for isolation mode resume)
1220
- // The isolated workspace at /tmp/zeroshot-isolated/{clusterId} was preserved by stop()
1221
- const workDir = cluster.isolation.workDir;
1222
- if (!workDir) {
1223
- throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
1224
- }
1559
+ if (errors.length === 0) {
1560
+ return null;
1561
+ }
1225
1562
 
1226
- // Check if isolated workspace still exists (it should, if stop() was used)
1227
- const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
1228
- if (!fs.existsSync(isolatedPath)) {
1229
- throw new Error(
1230
- `Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
1231
- `Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
1232
- );
1233
- }
1563
+ const firstError = errors[0];
1564
+ const failureInfo = {
1565
+ agentId: firstError.sender,
1566
+ taskId: firstError.content?.data?.taskId || null,
1567
+ iteration: firstError.content?.data?.iteration || 0,
1568
+ error: firstError.content?.data?.error || firstError.content?.text,
1569
+ timestamp: firstError.timestamp,
1570
+ };
1571
+ this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
1572
+ return failureInfo;
1573
+ }
1234
1574
 
1235
- const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
1236
- workDir, // Use saved workDir, NOT process.cwd()
1237
- image: cluster.isolation.image,
1238
- reuseExistingWorkspace: true, // CRITICAL: Don't wipe existing work
1239
- });
1575
+ _checkContainerExists(containerId) {
1576
+ const { spawn } = require('child_process');
1577
+ const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
1578
+ return new Promise((resolve) => {
1579
+ checkContainer.on('close', (code) => resolve(code === 0));
1580
+ });
1581
+ }
1240
1582
 
1241
- this._log(`[Orchestrator] New container created: ${newContainerId}`);
1583
+ async _ensureIsolationForResume(clusterId, cluster) {
1584
+ if (!cluster.isolation?.enabled) {
1585
+ return;
1586
+ }
1242
1587
 
1243
- // Update cluster isolation state
1244
- cluster.isolation.containerId = newContainerId;
1588
+ const oldContainerId = cluster.isolation.containerId;
1589
+ const containerExists = await this._checkContainerExists(oldContainerId);
1590
+ if (containerExists) {
1591
+ this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
1592
+ return;
1593
+ }
1245
1594
 
1246
- // CRITICAL: Update all agents' isolation context with new container ID
1247
- for (const agent of cluster.agents) {
1248
- if (agent.isolation?.enabled) {
1249
- agent.isolation.containerId = newContainerId;
1250
- agent.isolation.manager = cluster.isolation.manager;
1251
- }
1252
- }
1595
+ this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
1253
1596
 
1254
- this._log(`[Orchestrator] All agents updated with new container ID`);
1255
- } else {
1256
- this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
1257
- }
1597
+ const workDir = cluster.isolation.workDir;
1598
+ if (!workDir) {
1599
+ throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
1258
1600
  }
1259
1601
 
1260
- // Verify worktree still exists for worktree isolation mode
1261
- if (cluster.worktree?.enabled) {
1262
- const worktreePath = cluster.worktree.path;
1263
- if (!fs.existsSync(worktreePath)) {
1264
- throw new Error(
1265
- `Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
1266
- `Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
1267
- );
1602
+ const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
1603
+ if (!fs.existsSync(isolatedPath)) {
1604
+ throw new Error(
1605
+ `Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
1606
+ `Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
1607
+ );
1608
+ }
1609
+
1610
+ const providerName = normalizeProviderName(
1611
+ cluster.config?.forceProvider ||
1612
+ cluster.config?.defaultProvider ||
1613
+ loadSettings().defaultProvider ||
1614
+ 'claude'
1615
+ );
1616
+ const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
1617
+ workDir,
1618
+ image: cluster.isolation.image,
1619
+ reuseExistingWorkspace: true,
1620
+ provider: providerName,
1621
+ });
1622
+
1623
+ this._log(`[Orchestrator] New container created: ${newContainerId}`);
1624
+ cluster.isolation.containerId = newContainerId;
1625
+
1626
+ for (const agent of cluster.agents) {
1627
+ if (agent.isolation?.enabled) {
1628
+ agent.isolation.containerId = newContainerId;
1629
+ agent.isolation.manager = cluster.isolation.manager;
1268
1630
  }
1631
+ }
1269
1632
 
1270
- this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
1271
- this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1633
+ this._log(`[Orchestrator] All agents updated with new container ID`);
1634
+ }
1635
+
1636
+ _ensureWorktreeForResume(clusterId, cluster) {
1637
+ if (!cluster.worktree?.enabled) {
1638
+ return;
1272
1639
  }
1273
1640
 
1274
- // Restart all agents
1641
+ const worktreePath = cluster.worktree.path;
1642
+ if (!fs.existsSync(worktreePath)) {
1643
+ throw new Error(
1644
+ `Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
1645
+ `Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
1646
+ );
1647
+ }
1648
+
1649
+ this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
1650
+ this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
1651
+ }
1652
+
1653
+ async _restartClusterAgents(cluster) {
1275
1654
  cluster.state = 'running';
1276
1655
  for (const agent of cluster.agents) {
1277
1656
  if (!agent.running) {
1278
1657
  await agent.start();
1279
1658
  }
1280
1659
  }
1660
+ }
1661
+
1662
+ _loadRecentMessages(cluster, clusterId, limit) {
1663
+ return cluster.messageBus
1664
+ .query({
1665
+ cluster_id: clusterId,
1666
+ limit,
1667
+ order: 'desc',
1668
+ })
1669
+ .reverse();
1670
+ }
1671
+
1672
+ _buildResumeContext(recentMessages, prompt, options) {
1673
+ const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1674
+ let context = `${options.header}\n\n`;
1675
+
1676
+ if (options.error) {
1677
+ context += `Previous error: ${options.error}\n\n`;
1678
+ }
1679
+
1680
+ context += `## Recent Context\n\n`;
1681
+ for (const msg of recentMessages.slice(-10)) {
1682
+ if (options.topics.includes(msg.topic)) {
1683
+ context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1684
+ }
1685
+ }
1686
+
1687
+ context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1688
+ return context;
1689
+ }
1690
+
1691
+ _triggerMatchesTopic(triggerTopic, topic) {
1692
+ if (triggerTopic === topic) return true;
1693
+ if (triggerTopic === '*') return true;
1694
+ if (triggerTopic.endsWith('*')) {
1695
+ const prefix = triggerTopic.slice(0, -1);
1696
+ return topic.startsWith(prefix);
1697
+ }
1698
+ return false;
1699
+ }
1700
+
1701
+ _selectAgentsToResume(cluster, lastTrigger) {
1702
+ const agentsToResume = [];
1703
+
1704
+ for (const agent of cluster.agents) {
1705
+ if (!agent.config.triggers) continue;
1706
+
1707
+ const matchingTrigger = agent.config.triggers.find((trigger) =>
1708
+ this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
1709
+ );
1710
+ if (!matchingTrigger) continue;
1711
+
1712
+ if (matchingTrigger.logic?.script) {
1713
+ const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
1714
+ if (!shouldTrigger) continue;
1715
+ }
1716
+
1717
+ agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
1718
+ }
1719
+
1720
+ return agentsToResume;
1721
+ }
1722
+
1723
+ _resumeAgents(agentsToResume, context) {
1724
+ for (const { agent, message } of agentsToResume) {
1725
+ this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
1726
+ agent.resume(context).catch((err) => {
1727
+ console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
1728
+ });
1729
+ }
1730
+ }
1731
+
1732
+ _republishIssue(cluster, clusterId, recentMessages) {
1733
+ const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
1734
+ if (!issueMessage) {
1735
+ return false;
1736
+ }
1281
1737
 
1282
- // Query recent messages from ledger to provide context
1283
- const recentMessages = cluster.messageBus.query({
1738
+ cluster.messageBus.publish({
1284
1739
  cluster_id: clusterId,
1285
- limit: 50,
1740
+ topic: 'ISSUE_OPENED',
1741
+ sender: 'system',
1742
+ receiver: 'broadcast',
1743
+ content: issueMessage.content,
1744
+ metadata: { _resumed: true, _originalId: issueMessage.id },
1286
1745
  });
1287
1746
 
1288
- // CASE 1: Failed cluster - Resume the failed agent with error context
1289
- if (failureInfo) {
1290
- const { agentId, iteration, error } = failureInfo;
1291
- this._log(
1292
- `[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
1293
- );
1294
- this._log(`[Orchestrator] Previous error: ${error}`);
1295
-
1296
- // Find the failed agent
1297
- const failedAgent = cluster.agents.find((a) => a.id === agentId);
1298
- if (!failedAgent) {
1299
- throw new Error(`Failed agent '${agentId}' not found in cluster`);
1300
- }
1301
-
1302
- // Build failure resume context
1303
- const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1304
- let context = `You are resuming from a previous failed attempt.\n\n`;
1305
- context += `Previous error: ${error}\n\n`;
1306
- context += `## Recent Context\n\n`;
1747
+ return true;
1748
+ }
1307
1749
 
1308
- for (const msg of recentMessages.slice(-10)) {
1309
- if (msg.topic === 'AGENT_OUTPUT' || msg.topic === 'VALIDATION_RESULT') {
1310
- context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1311
- }
1312
- }
1750
+ async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
1751
+ const { agentId, iteration, error } = failureInfo;
1752
+ this._log(
1753
+ `[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
1754
+ );
1755
+ this._log(`[Orchestrator] Previous error: ${error}`);
1313
1756
 
1314
- context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1757
+ const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
1758
+ if (!failedAgent) {
1759
+ throw new Error(`Failed agent '${agentId}' not found in cluster`);
1760
+ }
1315
1761
 
1316
- // Clear failure info since we're resuming
1317
- cluster.failureInfo = null;
1762
+ const context = this._buildResumeContext(recentMessages, prompt, {
1763
+ header: 'You are resuming from a previous failed attempt.',
1764
+ error,
1765
+ topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
1766
+ });
1318
1767
 
1319
- // Save updated state
1320
- this._saveClusters();
1768
+ cluster.failureInfo = null;
1769
+ await this._saveClusters();
1321
1770
 
1322
- // Resume the failed agent
1323
- failedAgent.resume(context).catch((err) => {
1324
- console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
1325
- });
1771
+ failedAgent.resume(context).catch((err) => {
1772
+ console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
1773
+ });
1326
1774
 
1327
- this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
1775
+ this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
1328
1776
 
1329
- return {
1330
- id: clusterId,
1331
- state: cluster.state,
1332
- resumeType: 'failure',
1333
- resumedAgent: agentId,
1334
- previousError: error,
1335
- };
1336
- }
1777
+ return {
1778
+ id: clusterId,
1779
+ state: cluster.state,
1780
+ resumeType: 'failure',
1781
+ resumedAgent: agentId,
1782
+ previousError: error,
1783
+ };
1784
+ }
1337
1785
 
1338
- // CASE 2: Cleanly stopped cluster - Resume by re-triggering based on ledger state
1786
+ async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
1339
1787
  this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
1340
1788
 
1341
- // Build generic resume context
1342
- const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
1343
- let context = `Resuming cluster from previous session.\n\n`;
1344
- context += `## Recent Context\n\n`;
1345
-
1346
- for (const msg of recentMessages.slice(-10)) {
1347
- if (
1348
- msg.topic === 'AGENT_OUTPUT' ||
1349
- msg.topic === 'VALIDATION_RESULT' ||
1350
- msg.topic === 'ISSUE_OPENED'
1351
- ) {
1352
- context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
1353
- }
1354
- }
1355
-
1356
- context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
1789
+ const context = this._buildResumeContext(recentMessages, prompt, {
1790
+ header: 'Resuming cluster from previous session.',
1791
+ topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
1792
+ });
1357
1793
 
1358
- // Find the LAST workflow trigger - not arbitrary last 5 messages
1359
- // This is the message that indicates current workflow state
1360
1794
  const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
1361
- const agentsToResume = [];
1362
-
1363
1795
  if (lastTrigger) {
1364
1796
  this._log(
1365
1797
  `[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
1366
1798
  );
1367
-
1368
- for (const agent of cluster.agents) {
1369
- if (!agent.config.triggers) continue;
1370
-
1371
- const matchingTrigger = agent.config.triggers.find((trigger) => {
1372
- // Exact match
1373
- if (trigger.topic === lastTrigger.topic) return true;
1374
- // Wildcard match
1375
- if (trigger.topic === '*') return true;
1376
- // Prefix match (e.g., "VALIDATION_*")
1377
- if (trigger.topic.endsWith('*')) {
1378
- const prefix = trigger.topic.slice(0, -1);
1379
- return lastTrigger.topic.startsWith(prefix);
1380
- }
1381
- return false;
1382
- });
1383
-
1384
- if (matchingTrigger) {
1385
- // Evaluate logic script if present
1386
- if (matchingTrigger.logic?.script) {
1387
- const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
1388
- if (!shouldTrigger) continue;
1389
- }
1390
- agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
1391
- }
1392
- }
1393
1799
  } else {
1394
1800
  this._log(`[Orchestrator] No workflow triggers found in ledger`);
1395
1801
  }
1396
1802
 
1803
+ const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
1397
1804
  if (agentsToResume.length === 0) {
1398
1805
  if (!lastTrigger) {
1399
- // No workflow activity - cluster never really started
1400
1806
  this._log(
1401
1807
  `[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
1402
1808
  );
1403
1809
  this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
1404
1810
 
1405
- // Re-publish the original issue if we have it
1406
- const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
1407
- if (issueMessage) {
1408
- cluster.messageBus.publish({
1409
- cluster_id: clusterId,
1410
- topic: 'ISSUE_OPENED',
1411
- sender: 'system',
1412
- receiver: 'broadcast',
1413
- content: issueMessage.content,
1414
- metadata: { _resumed: true, _originalId: issueMessage.id },
1415
- });
1416
- } else {
1811
+ if (!this._republishIssue(cluster, clusterId, recentMessages)) {
1417
1812
  throw new Error(
1418
1813
  `Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
1419
1814
  `The cluster may not have started properly. Try: zeroshot run <issue> instead.`
1420
1815
  );
1421
1816
  }
1422
1817
  } else {
1423
- // Had a trigger but no agents matched - something is wrong with agent configs
1424
1818
  throw new Error(
1425
1819
  `Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
1426
1820
  `Check agent trigger configurations.`
1427
1821
  );
1428
1822
  }
1429
1823
  } else {
1430
- // Resume agents that should run based on ledger state
1431
1824
  this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
1432
- for (const { agent, message } of agentsToResume) {
1433
- this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
1434
- agent.resume(context).catch((err) => {
1435
- console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
1436
- });
1437
- }
1825
+ this._resumeAgents(agentsToResume, context);
1438
1826
  }
1439
1827
 
1440
- // Save updated state
1441
- this._saveClusters();
1442
-
1828
+ await this._saveClusters();
1443
1829
  this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
1444
1830
 
1445
1831
  return {
@@ -1505,10 +1891,13 @@ Continue from where you left off. Review your previous output to understand what
1505
1891
  `.trim();
1506
1892
 
1507
1893
  // Get recent context from ledger
1508
- const recentMessages = cluster.messageBus.query({
1509
- cluster_id: cluster.id,
1510
- limit: 10,
1511
- });
1894
+ const recentMessages = cluster.messageBus
1895
+ .query({
1896
+ cluster_id: cluster.id,
1897
+ limit: 10,
1898
+ order: 'desc',
1899
+ })
1900
+ .reverse();
1512
1901
 
1513
1902
  const contextText = recentMessages
1514
1903
  .map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
@@ -1555,7 +1944,63 @@ Continue from where you left off. Review your previous output to understand what
1555
1944
  this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
1556
1945
 
1557
1946
  // Phase 1: Pre-validate operation structure
1947
+ const validationErrors = this._validateOperationChain(operations);
1948
+
1949
+ if (validationErrors.length > 0) {
1950
+ const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
1951
+ this._log(`[Orchestrator] ❌ ${errorMsg}`);
1952
+ throw new Error(errorMsg);
1953
+ }
1954
+
1955
+ // Phase 2: Build mock cluster config with proposed agents
1956
+ // Collect all agents that would exist after operations complete
1957
+ const existingAgentConfigs = cluster.config.agents || [];
1958
+ const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
1959
+
1960
+ // Phase 3: Validate proposed cluster config
1961
+ const validation = this._validateProposedConfig(
1962
+ clusterId,
1963
+ cluster,
1964
+ proposedAgentConfigs,
1965
+ operations
1966
+ );
1967
+
1968
+ // Log warnings but proceed
1969
+ if (validation.warnings.length > 0) {
1970
+ this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
1971
+ for (const warning of validation.warnings) {
1972
+ this._log(` - ${warning}`);
1973
+ }
1974
+ }
1975
+
1976
+ // Phase 4: Execute validated operations
1977
+ this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
1978
+
1979
+ await this._executeOperations(cluster, operations, sender, context);
1980
+
1981
+ this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
1982
+
1983
+ // Publish success notification
1984
+ cluster.messageBus.publish({
1985
+ cluster_id: clusterId,
1986
+ topic: 'CLUSTER_OPERATIONS_SUCCESS',
1987
+ sender: 'orchestrator',
1988
+ content: {
1989
+ text: `Executed ${operations.length} operation(s)`,
1990
+ data: {
1991
+ operationCount: operations.length,
1992
+ agentCount: cluster.agents.length,
1993
+ },
1994
+ },
1995
+ });
1996
+
1997
+ // Save updated cluster state to disk
1998
+ await this._saveClusters();
1999
+ }
2000
+
2001
+ _validateOperationChain(operations) {
1558
2002
  const validationErrors = [];
2003
+
1559
2004
  for (let i = 0; i < operations.length; i++) {
1560
2005
  const op = operations[i];
1561
2006
  if (!op.action) {
@@ -1569,21 +2014,15 @@ Continue from where you left off. Review your previous output to understand what
1569
2014
  }
1570
2015
  }
1571
2016
 
1572
- if (validationErrors.length > 0) {
1573
- const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
1574
- this._log(`[Orchestrator] ❌ ${errorMsg}`);
1575
- throw new Error(errorMsg);
1576
- }
2017
+ return validationErrors;
2018
+ }
1577
2019
 
1578
- // Phase 2: Build mock cluster config with proposed agents
1579
- // Collect all agents that would exist after operations complete
1580
- const existingAgentConfigs = cluster.config.agents || [];
2020
+ _buildProposedAgentConfigs(existingAgentConfigs, operations) {
1581
2021
  const proposedAgentConfigs = [...existingAgentConfigs];
1582
2022
 
1583
2023
  for (const op of operations) {
1584
2024
  if (op.action === 'add_agents' && op.agents) {
1585
2025
  for (const agentConfig of op.agents) {
1586
- // Check for duplicate before adding
1587
2026
  const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
1588
2027
  if (existingIdx === -1) {
1589
2028
  proposedAgentConfigs.push(agentConfig);
@@ -1604,7 +2043,10 @@ Continue from where you left off. Review your previous output to understand what
1604
2043
  }
1605
2044
  }
1606
2045
 
1607
- // Phase 3: Validate proposed cluster config
2046
+ return proposedAgentConfigs;
2047
+ }
2048
+
2049
+ _validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
1608
2050
  const mockConfig = { agents: proposedAgentConfigs };
1609
2051
  const validation = configValidator.validateConfig(mockConfig);
1610
2052
 
@@ -1630,62 +2072,35 @@ Continue from where you left off. Review your previous output to understand what
1630
2072
  throw new Error(errorMsg);
1631
2073
  }
1632
2074
 
1633
- // Log warnings but proceed
1634
- if (validation.warnings.length > 0) {
1635
- this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
1636
- for (const warning of validation.warnings) {
1637
- this._log(` - ${warning}`);
1638
- }
1639
- }
1640
-
1641
- // Phase 4: Execute validated operations
1642
- this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
2075
+ return validation;
2076
+ }
1643
2077
 
2078
+ async _executeOperations(cluster, operations, sender, context) {
1644
2079
  for (let i = 0; i < operations.length; i++) {
1645
2080
  const op = operations[i];
1646
2081
  this._log(` [${i + 1}/${operations.length}] ${op.action}`);
1647
-
1648
- switch (op.action) {
1649
- case 'add_agents':
1650
- await this._opAddAgents(cluster, op, context);
1651
- break;
1652
-
1653
- case 'remove_agents':
1654
- await this._opRemoveAgents(cluster, op);
1655
- break;
1656
-
1657
- case 'update_agent':
1658
- this._opUpdateAgent(cluster, op);
1659
- break;
1660
-
1661
- case 'publish':
1662
- this._opPublish(cluster, op, sender);
1663
- break;
1664
-
1665
- case 'load_config':
1666
- await this._opLoadConfig(cluster, op, context);
1667
- break;
1668
- }
2082
+ await this._executeOperation(cluster, op, sender, context);
1669
2083
  }
2084
+ }
1670
2085
 
1671
- this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
1672
-
1673
- // Publish success notification
1674
- cluster.messageBus.publish({
1675
- cluster_id: clusterId,
1676
- topic: 'CLUSTER_OPERATIONS_SUCCESS',
1677
- sender: 'orchestrator',
1678
- content: {
1679
- text: `Executed ${operations.length} operation(s)`,
1680
- data: {
1681
- operationCount: operations.length,
1682
- agentCount: cluster.agents.length,
1683
- },
1684
- },
1685
- });
1686
-
1687
- // Save updated cluster state to disk
1688
- this._saveClusters();
2086
+ async _executeOperation(cluster, op, sender, context) {
2087
+ switch (op.action) {
2088
+ case 'add_agents':
2089
+ await this._opAddAgents(cluster, op, context);
2090
+ break;
2091
+ case 'remove_agents':
2092
+ await this._opRemoveAgents(cluster, op);
2093
+ break;
2094
+ case 'update_agent':
2095
+ this._opUpdateAgent(cluster, op);
2096
+ break;
2097
+ case 'publish':
2098
+ this._opPublish(cluster, op, sender);
2099
+ break;
2100
+ case 'load_config':
2101
+ await this._opLoadConfig(cluster, op, context);
2102
+ break;
2103
+ }
1689
2104
  }
1690
2105
 
1691
2106
  /**
@@ -1709,6 +2124,12 @@ Continue from where you left off. Review your previous output to understand what
1709
2124
  agentConfig.cwd = agentCwd;
1710
2125
  this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
1711
2126
  }
2127
+
2128
+ // Apply model override if set (for consistency with initial agents)
2129
+ if (cluster.modelOverride) {
2130
+ applyModelOverride(agentConfig, cluster.modelOverride);
2131
+ this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
2132
+ }
1712
2133
  // Validate agent config has required fields
1713
2134
  if (!agentConfig.id) {
1714
2135
  throw new Error('Agent config missing required field: id');
@@ -1731,6 +2152,7 @@ Continue from where you left off. Review your previous output to understand what
1731
2152
  const agentOptions = {
1732
2153
  testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
1733
2154
  quiet: this.quiet,
2155
+ modelOverride: cluster.modelOverride || null,
1734
2156
  };
1735
2157
 
1736
2158
  // TaskRunner DI - propagate to dynamically spawned agents
@@ -1905,6 +2327,111 @@ Continue from where you left off. Review your previous output to understand what
1905
2327
  await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
1906
2328
 
1907
2329
  this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
2330
+
2331
+ // Inject completion agent (templates don't include one - orchestrator controls termination)
2332
+ await this._injectCompletionAgent(cluster, context);
2333
+ }
2334
+
2335
+ /**
2336
+ * Inject appropriate completion agent based on mode
2337
+ * Templates define work, orchestrator controls termination strategy
2338
+ * @private
2339
+ */
2340
+ async _injectCompletionAgent(cluster, context) {
2341
+ // Skip if completion agent already exists
2342
+ const hasCompletionAgent = cluster.agents.some(
2343
+ (a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
2344
+ );
2345
+ if (hasCompletionAgent) {
2346
+ return;
2347
+ }
2348
+
2349
+ const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
2350
+
2351
+ if (isPrMode) {
2352
+ // Detect platform from stored cluster metadata OR git context
2353
+ let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
2354
+
2355
+ // Fallback to git context detection if cluster metadata missing
2356
+ if (!platform) {
2357
+ const { getPlatformForPR } = require('./issue-providers');
2358
+ try {
2359
+ platform = getPlatformForPR(cluster.cwd || process.cwd());
2360
+ } catch (error) {
2361
+ throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
2362
+ }
2363
+ }
2364
+
2365
+ // Generate platform-specific git-pusher agent from template
2366
+ const {
2367
+ generateGitPusherAgent,
2368
+ isPlatformSupported,
2369
+ } = require('./agents/git-pusher-template');
2370
+
2371
+ if (!isPlatformSupported(platform)) {
2372
+ throw new Error(
2373
+ `Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
2374
+ );
2375
+ }
2376
+
2377
+ const gitPusherConfig = generateGitPusherAgent(platform);
2378
+
2379
+ // Get issue context from ledger
2380
+ const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
2381
+ const issueNumber = issueMsg?.content?.data?.number || 'unknown';
2382
+ const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
2383
+
2384
+ // Inject issue context into prompt
2385
+ gitPusherConfig.prompt = gitPusherConfig.prompt
2386
+ .replace(/\{\{issue_number\}\}/g, issueNumber)
2387
+ .replace(/\{\{issue_title\}\}/g, issueTitle);
2388
+
2389
+ await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
2390
+ this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
2391
+ } else {
2392
+ // Default completion-detector
2393
+ const completionDetector = {
2394
+ id: 'completion-detector',
2395
+ role: 'orchestrator',
2396
+ model: 'haiku',
2397
+ timeout: 0,
2398
+ triggers: [
2399
+ {
2400
+ topic: 'VALIDATION_RESULT',
2401
+ logic: {
2402
+ engine: 'javascript',
2403
+ script: `const validators = cluster.getAgentsByRole('validator');
2404
+ const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });
2405
+ if (!lastPush) return false;
2406
+ if (validators.length === 0) return true;
2407
+
2408
+ const validatorIds = new Set(validators.map((v) => v.id));
2409
+ const results = ledger.query({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });
2410
+
2411
+ const latestByValidator = new Map();
2412
+ for (const msg of results) {
2413
+ if (!validatorIds.has(msg.sender)) continue;
2414
+ latestByValidator.set(msg.sender, msg);
2415
+ }
2416
+
2417
+ if (latestByValidator.size < validators.length) return false;
2418
+
2419
+ for (const validator of validators) {
2420
+ const msg = latestByValidator.get(validator.id);
2421
+ const approved = msg?.content?.data?.approved;
2422
+ if (!(approved === true || approved === 'true')) return false;
2423
+ }
2424
+
2425
+ return true;`,
2426
+ },
2427
+ action: 'stop_cluster',
2428
+ },
2429
+ ],
2430
+ };
2431
+
2432
+ await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
2433
+ this._log(` Injected completion-detector agent`);
2434
+ }
1908
2435
  }
1909
2436
 
1910
2437
  /**
@@ -2047,7 +2574,7 @@ Continue from where you left off. Review your previous output to understand what
2047
2574
  * @private
2048
2575
  */
2049
2576
  _exportMarkdown(cluster, clusterId, messages) {
2050
- const { parseChunk } = require('../lib/stream-json-parser');
2577
+ const { parseProviderChunk } = require('./providers');
2051
2578
 
2052
2579
  // Find task info
2053
2580
  const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
@@ -2071,100 +2598,164 @@ Continue from where you left off. Review your previous output to understand what
2071
2598
  md += `## Task\n\n${taskText}\n\n`;
2072
2599
 
2073
2600
  // Group messages by agent for cleaner output
2601
+ const agentOutputs = this._collectAgentOutputs(messages);
2602
+
2603
+ for (const [agentId, agentMsgs] of agentOutputs) {
2604
+ md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
2605
+ }
2606
+
2607
+ md += this._renderValidationMarkdown(messages);
2608
+
2609
+ // Final status
2610
+ const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
2611
+ if (clusterComplete) {
2612
+ md += `## Result\n\n✅ **Cluster completed successfully**\n`;
2613
+ }
2614
+
2615
+ return md;
2616
+ }
2617
+
2618
+ _collectAgentOutputs(messages) {
2074
2619
  const agentOutputs = new Map();
2075
2620
 
2076
2621
  for (const msg of messages) {
2077
- if (msg.topic === 'AGENT_OUTPUT') {
2078
- if (!agentOutputs.has(msg.sender)) {
2079
- agentOutputs.set(msg.sender, []);
2622
+ if (msg.topic !== 'AGENT_OUTPUT') continue;
2623
+ if (!agentOutputs.has(msg.sender)) {
2624
+ agentOutputs.set(msg.sender, []);
2625
+ }
2626
+ agentOutputs.get(msg.sender).push(msg);
2627
+ }
2628
+
2629
+ return agentOutputs;
2630
+ }
2631
+
2632
+ _extractAgentOutput(agentMsgs, parseProviderChunk) {
2633
+ let text = '';
2634
+ const tools = [];
2635
+
2636
+ for (const msg of agentMsgs) {
2637
+ const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
2638
+ if (!content) continue;
2639
+
2640
+ const provider = normalizeProviderName(
2641
+ msg.content?.data?.provider || msg.sender_provider || 'claude'
2642
+ );
2643
+ const events = parseProviderChunk(provider, content);
2644
+ for (const event of events) {
2645
+ switch (event.type) {
2646
+ case 'text':
2647
+ text += event.text;
2648
+ break;
2649
+ case 'tool_call':
2650
+ tools.push({ name: event.toolName, input: event.input });
2651
+ break;
2652
+ case 'tool_result':
2653
+ if (tools.length > 0) {
2654
+ const lastTool = tools[tools.length - 1];
2655
+ lastTool.result = event.content;
2656
+ lastTool.isError = event.isError;
2657
+ }
2658
+ break;
2080
2659
  }
2081
- agentOutputs.get(msg.sender).push(msg);
2082
2660
  }
2083
2661
  }
2084
2662
 
2085
- // Agent sections
2086
- for (const [agentId, agentMsgs] of agentOutputs) {
2087
- md += `## Agent: ${agentId}\n\n`;
2088
-
2089
- let text = '';
2090
- let tools = [];
2091
-
2092
- for (const msg of agentMsgs) {
2093
- const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
2094
- if (!content) continue;
2095
-
2096
- const events = parseChunk(content);
2097
- for (const event of events) {
2098
- switch (event.type) {
2099
- case 'text':
2100
- text += event.text;
2101
- break;
2102
- case 'tool_call':
2103
- tools.push({ name: event.toolName, input: event.input });
2104
- break;
2105
- case 'tool_result':
2106
- if (tools.length > 0) {
2107
- const lastTool = tools[tools.length - 1];
2108
- lastTool.result = event.content;
2109
- lastTool.isError = event.isError;
2110
- }
2111
- break;
2112
- }
2663
+ return { text, tools };
2664
+ }
2665
+
2666
+ _renderToolMarkdown(tools) {
2667
+ let md = `### Tools Used\n\n`;
2668
+
2669
+ for (const tool of tools) {
2670
+ const status = tool.isError ? '❌' : '✓';
2671
+ md += `- **${tool.name}** ${status}\n`;
2672
+ if (tool.input) {
2673
+ const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
2674
+ if (inputStr.length < 100) {
2675
+ md += ` - Input: \`${inputStr}\`\n`;
2113
2676
  }
2114
2677
  }
2678
+ }
2679
+
2680
+ return `${md}\n`;
2681
+ }
2682
+
2683
+ _renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
2684
+ let md = `## Agent: ${agentId}\n\n`;
2685
+ const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
2686
+
2687
+ if (text.trim()) {
2688
+ md += `### Output\n\n${text.trim()}\n\n`;
2689
+ }
2690
+
2691
+ if (tools.length > 0) {
2692
+ md += this._renderToolMarkdown(tools);
2693
+ }
2694
+
2695
+ return md;
2696
+ }
2697
+
2698
+ _renderCriteriaMarkdown(criteriaResults) {
2699
+ if (!Array.isArray(criteriaResults)) {
2700
+ return '';
2701
+ }
2702
+
2703
+ let md = '';
2704
+ const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
2705
+ if (cannotValidateYet.length > 0) {
2706
+ md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
2707
+ for (const cv of cannotValidateYet) {
2708
+ md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
2709
+ }
2710
+ md += '\n';
2711
+ }
2115
2712
 
2116
- // Output text
2117
- if (text.trim()) {
2118
- md += `### Output\n\n${text.trim()}\n\n`;
2713
+ const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
2714
+ if (cannotValidate.length > 0) {
2715
+ md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
2716
+ for (const cv of cannotValidate) {
2717
+ md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
2119
2718
  }
2719
+ md += '\n';
2720
+ }
2120
2721
 
2121
- // Tools used
2122
- if (tools.length > 0) {
2123
- md += `### Tools Used\n\n`;
2124
- for (const tool of tools) {
2125
- const status = tool.isError ? '❌' : '✓';
2126
- md += `- **${tool.name}** ${status}\n`;
2127
- if (tool.input) {
2128
- const inputStr =
2129
- typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
2130
- if (inputStr.length < 100) {
2131
- md += ` - Input: \`${inputStr}\`\n`;
2132
- }
2133
- }
2722
+ return md;
2723
+ }
2724
+
2725
+ _renderValidationEntryMarkdown(validation) {
2726
+ const data = validation.content?.data || {};
2727
+ const approved = data.approved === true || data.approved === 'true';
2728
+ const icon = approved ? '✅' : '❌';
2729
+ let md = `### ${validation.sender} ${icon}\n\n`;
2730
+
2731
+ if (data.summary) {
2732
+ md += `${data.summary}\n\n`;
2733
+ }
2734
+
2735
+ if (!approved && data.issues) {
2736
+ const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
2737
+ if (Array.isArray(issues) && issues.length > 0) {
2738
+ md += `**Issues:**\n`;
2739
+ for (const issue of issues) {
2740
+ md += `- ${issue}\n`;
2134
2741
  }
2135
2742
  md += '\n';
2136
2743
  }
2137
2744
  }
2138
2745
 
2139
- // Validation results
2746
+ md += this._renderCriteriaMarkdown(data.criteriaResults);
2747
+ return md;
2748
+ }
2749
+
2750
+ _renderValidationMarkdown(messages) {
2140
2751
  const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
2141
- if (validations.length > 0) {
2142
- md += `## Validation Results\n\n`;
2143
- for (const v of validations) {
2144
- const data = v.content?.data || {};
2145
- const approved = data.approved === true || data.approved === 'true';
2146
- const icon = approved ? '✅' : '❌';
2147
- md += `### ${v.sender} ${icon}\n\n`;
2148
- if (data.summary) {
2149
- md += `${data.summary}\n\n`;
2150
- }
2151
- if (!approved && data.issues) {
2152
- const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
2153
- if (Array.isArray(issues) && issues.length > 0) {
2154
- md += `**Issues:**\n`;
2155
- for (const issue of issues) {
2156
- md += `- ${issue}\n`;
2157
- }
2158
- md += '\n';
2159
- }
2160
- }
2161
- }
2752
+ if (validations.length === 0) {
2753
+ return '';
2162
2754
  }
2163
2755
 
2164
- // Final status
2165
- const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
2166
- if (clusterComplete) {
2167
- md += `## Result\n\n✅ **Cluster completed successfully**\n`;
2756
+ let md = `## Validation Results\n\n`;
2757
+ for (const validation of validations) {
2758
+ md += this._renderValidationEntryMarkdown(validation);
2168
2759
  }
2169
2760
 
2170
2761
  return md;