@covibes/zeroshot 5.3.0 → 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -14
- package/cli/commands/providers.js +8 -9
- package/cli/index.js +3032 -2409
- package/cli/message-formatters-normal.js +28 -6
- package/cluster-templates/base-templates/debug-workflow.json +1 -1
- package/cluster-templates/base-templates/full-workflow.json +72 -188
- package/cluster-templates/base-templates/worker-validator.json +59 -3
- package/cluster-templates/conductor-bootstrap.json +4 -4
- package/lib/docker-config.js +8 -0
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +10 -7
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-names.js +2 -1
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +161 -63
- package/package.json +7 -2
- package/scripts/setup-merge-queue.sh +170 -0
- package/src/agent/agent-config.js +135 -82
- package/src/agent/agent-context-builder.js +297 -188
- package/src/agent/agent-hook-executor.js +310 -113
- package/src/agent/agent-lifecycle.js +385 -325
- package/src/agent/agent-stuck-detector.js +7 -7
- package/src/agent/agent-task-executor.js +824 -565
- package/src/agent/output-extraction.js +41 -24
- package/src/agent/output-reformatter.js +1 -1
- package/src/agent/schema-utils.js +108 -73
- package/src/agent-wrapper.js +10 -1
- package/src/agents/git-pusher-template.js +285 -0
- package/src/claude-task-runner.js +85 -34
- package/src/config-validator.js +922 -657
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +289 -199
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +273 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +179 -0
- package/src/issue-providers/gitlab-provider.js +241 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +239 -0
- package/src/ledger.js +22 -5
- package/src/lib/safe-exec.js +88 -0
- package/src/orchestrator.js +1107 -811
- package/src/preflight.js +313 -159
- package/src/process-metrics.js +98 -56
- package/src/providers/anthropic/cli-builder.js +53 -25
- package/src/providers/anthropic/index.js +72 -2
- package/src/providers/anthropic/output-parser.js +32 -14
- package/src/providers/base-provider.js +70 -0
- package/src/providers/capabilities.js +9 -0
- package/src/providers/google/output-parser.js +47 -38
- package/src/providers/index.js +19 -3
- package/src/providers/openai/cli-builder.js +14 -3
- package/src/providers/openai/index.js +1 -0
- package/src/providers/openai/output-parser.js +44 -30
- package/src/providers/opencode/cli-builder.js +42 -0
- package/src/providers/opencode/index.js +103 -0
- package/src/providers/opencode/models.js +55 -0
- package/src/providers/opencode/output-parser.js +122 -0
- package/src/schemas/sub-cluster.js +68 -39
- package/src/status-footer.js +94 -48
- package/src/sub-cluster-wrapper.js +76 -35
- package/src/template-resolver.js +12 -9
- package/src/tui/data-poller.js +123 -99
- package/src/tui/formatters.js +4 -3
- package/src/tui/keybindings.js +259 -318
- package/src/tui/renderer.js +11 -21
- package/task-lib/attachable-watcher.js +118 -81
- package/task-lib/claude-recovery.js +61 -24
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/list.js +2 -2
- package/task-lib/commands/logs.js +231 -189
- package/task-lib/commands/schedules.js +111 -61
- package/task-lib/config.js +0 -2
- package/task-lib/runner.js +84 -27
- package/task-lib/scheduler.js +1 -1
- package/task-lib/store.js +467 -168
- package/task-lib/tui/formatters.js +94 -45
- package/task-lib/tui/renderer.js +13 -3
- package/task-lib/tui.js +73 -32
- package/task-lib/watcher.js +128 -90
- package/src/agents/git-pusher-agent.json +0 -20
- package/src/github.js +0 -139
package/src/orchestrator.js
CHANGED
|
@@ -37,7 +37,8 @@ const AgentWrapper = require('./agent-wrapper');
|
|
|
37
37
|
const SubClusterWrapper = require('./sub-cluster-wrapper');
|
|
38
38
|
const MessageBus = require('./message-bus');
|
|
39
39
|
const Ledger = require('./ledger');
|
|
40
|
-
const
|
|
40
|
+
const InputHelpers = require('./input-helpers');
|
|
41
|
+
const { detectProvider } = require('./issue-providers');
|
|
41
42
|
const IsolationManager = require('./isolation-manager');
|
|
42
43
|
const { generateName } = require('./name-generator');
|
|
43
44
|
const configValidator = require('./config-validator');
|
|
@@ -130,6 +131,18 @@ class Orchestrator {
|
|
|
130
131
|
}
|
|
131
132
|
}
|
|
132
133
|
|
|
134
|
+
/**
|
|
135
|
+
* Get input source type for metadata
|
|
136
|
+
* @param {Object} input - Input object
|
|
137
|
+
* @returns {string} Source type: 'github', 'file', or 'text'
|
|
138
|
+
* @private
|
|
139
|
+
*/
|
|
140
|
+
_getInputSource(input) {
|
|
141
|
+
if (input.issue) return 'github';
|
|
142
|
+
if (input.file) return 'file';
|
|
143
|
+
return 'text';
|
|
144
|
+
}
|
|
145
|
+
|
|
133
146
|
/**
|
|
134
147
|
* Load clusters from persistent storage
|
|
135
148
|
* Uses file locking for consistent reads
|
|
@@ -255,8 +268,37 @@ class Orchestrator {
|
|
|
255
268
|
const messageBus = new MessageBus(ledger);
|
|
256
269
|
|
|
257
270
|
// Restore isolation manager FIRST if cluster was running in isolation mode
|
|
271
|
+
const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
|
|
272
|
+
|
|
273
|
+
// Reconstruct agent metadata from config (processes are ephemeral)
|
|
274
|
+
// CRITICAL: Pass isolation context to agents if cluster was running in isolation
|
|
275
|
+
const agents = this._rebuildClusterAgents(
|
|
276
|
+
clusterId,
|
|
277
|
+
clusterData,
|
|
278
|
+
messageBus,
|
|
279
|
+
isolation,
|
|
280
|
+
isolationManager
|
|
281
|
+
);
|
|
282
|
+
|
|
283
|
+
const cluster = {
|
|
284
|
+
...clusterData,
|
|
285
|
+
ledger,
|
|
286
|
+
messageBus,
|
|
287
|
+
agents,
|
|
288
|
+
isolation,
|
|
289
|
+
autoPr: clusterData.autoPr || false,
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
this.clusters.set(clusterId, cluster);
|
|
293
|
+
this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
|
|
294
|
+
|
|
295
|
+
return cluster;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
_restoreClusterIsolation(clusterId, clusterData) {
|
|
258
299
|
let isolation = clusterData.isolation || null;
|
|
259
300
|
let isolationManager = null;
|
|
301
|
+
|
|
260
302
|
if (isolation?.enabled && isolation.containerId) {
|
|
261
303
|
isolationManager = new IsolationManager({ image: isolation.image });
|
|
262
304
|
// Restore the container mapping so cleanup works
|
|
@@ -277,80 +319,84 @@ class Orchestrator {
|
|
|
277
319
|
);
|
|
278
320
|
}
|
|
279
321
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const agents = [];
|
|
322
|
+
return { isolation, isolationManager };
|
|
323
|
+
}
|
|
283
324
|
|
|
284
|
-
|
|
285
|
-
// This fixes clusters saved before the cwd injection bug was fixed
|
|
325
|
+
_resolveAgentCwd(clusterData) {
|
|
286
326
|
const worktreePath = clusterData.worktree?.path;
|
|
287
327
|
const isolationWorkDir = clusterData.isolation?.workDir;
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
if (clusterData.config?.agents) {
|
|
291
|
-
for (const agentConfig of clusterData.config.agents) {
|
|
292
|
-
// Fix agents that were saved without cwd (pre-bugfix clusters)
|
|
293
|
-
if (!agentConfig.cwd && agentCwd) {
|
|
294
|
-
agentConfig.cwd = agentCwd;
|
|
295
|
-
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
296
|
-
}
|
|
328
|
+
return worktreePath || isolationWorkDir || null;
|
|
329
|
+
}
|
|
297
330
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
331
|
+
_buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
|
|
332
|
+
const agentOptions = {
|
|
333
|
+
id: clusterId,
|
|
334
|
+
quiet: this.quiet,
|
|
335
|
+
modelOverride: clusterData.modelOverride || null,
|
|
336
|
+
};
|
|
301
337
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
338
|
+
if (isolation?.enabled && isolationManager) {
|
|
339
|
+
agentOptions.isolation = {
|
|
340
|
+
enabled: true,
|
|
341
|
+
manager: isolationManager,
|
|
342
|
+
clusterId,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
307
345
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
agentOptions.isolation = {
|
|
311
|
-
enabled: true,
|
|
312
|
-
manager: isolationManager,
|
|
313
|
-
clusterId,
|
|
314
|
-
};
|
|
315
|
-
}
|
|
346
|
+
return agentOptions;
|
|
347
|
+
}
|
|
316
348
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
} else {
|
|
322
|
-
agent = new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
323
|
-
}
|
|
349
|
+
_instantiateAgent(agentConfig, messageBus, clusterId, agentOptions) {
|
|
350
|
+
if (agentConfig.type === 'subcluster') {
|
|
351
|
+
return new SubClusterWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
352
|
+
}
|
|
324
353
|
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
const savedState = clusterData.agentStates.find((s) => s.id === agentConfig.id);
|
|
328
|
-
if (savedState) {
|
|
329
|
-
agent.state = savedState.state || 'idle';
|
|
330
|
-
agent.iteration = savedState.iteration || 0;
|
|
331
|
-
agent.currentTask = savedState.currentTask || false;
|
|
332
|
-
agent.currentTaskId = savedState.currentTaskId || null;
|
|
333
|
-
agent.processPid = savedState.processPid || null;
|
|
334
|
-
}
|
|
335
|
-
}
|
|
354
|
+
return new AgentWrapper(agentConfig, messageBus, { id: clusterId }, agentOptions);
|
|
355
|
+
}
|
|
336
356
|
|
|
337
|
-
|
|
338
|
-
|
|
357
|
+
_restoreAgentState(agent, agentConfig, clusterData) {
|
|
358
|
+
if (!clusterData.agentStates) return;
|
|
359
|
+
|
|
360
|
+
const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
|
|
361
|
+
if (!savedState) return;
|
|
362
|
+
|
|
363
|
+
agent.state = savedState.state || 'idle';
|
|
364
|
+
agent.iteration = savedState.iteration || 0;
|
|
365
|
+
agent.currentTask = savedState.currentTask || false;
|
|
366
|
+
agent.currentTaskId = savedState.currentTaskId || null;
|
|
367
|
+
agent.processPid = savedState.processPid || null;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
_rebuildClusterAgents(clusterId, clusterData, messageBus, isolation, isolationManager) {
|
|
371
|
+
const agents = [];
|
|
372
|
+
const agentCwd = this._resolveAgentCwd(clusterData);
|
|
373
|
+
|
|
374
|
+
if (!clusterData.config?.agents) {
|
|
375
|
+
return agents;
|
|
339
376
|
}
|
|
340
377
|
|
|
341
|
-
const
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
isolation,
|
|
347
|
-
autoPr: clusterData.autoPr || false,
|
|
348
|
-
};
|
|
378
|
+
for (const agentConfig of clusterData.config.agents) {
|
|
379
|
+
if (!agentConfig.cwd && agentCwd) {
|
|
380
|
+
agentConfig.cwd = agentCwd;
|
|
381
|
+
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
382
|
+
}
|
|
349
383
|
|
|
350
|
-
|
|
351
|
-
|
|
384
|
+
if (clusterData.modelOverride) {
|
|
385
|
+
applyModelOverride(agentConfig, clusterData.modelOverride);
|
|
386
|
+
}
|
|
352
387
|
|
|
353
|
-
|
|
388
|
+
const agentOptions = this._buildAgentOptions(
|
|
389
|
+
clusterId,
|
|
390
|
+
clusterData,
|
|
391
|
+
isolation,
|
|
392
|
+
isolationManager
|
|
393
|
+
);
|
|
394
|
+
const agent = this._instantiateAgent(agentConfig, messageBus, clusterId, agentOptions);
|
|
395
|
+
this._restoreAgentState(agent, agentConfig, clusterData);
|
|
396
|
+
agents.push(agent);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return agents;
|
|
354
400
|
}
|
|
355
401
|
|
|
356
402
|
/**
|
|
@@ -595,6 +641,8 @@ class Orchestrator {
|
|
|
595
641
|
autoPr: options.autoPr || process.env.ZEROSHOT_PR === '1',
|
|
596
642
|
modelOverride: options.modelOverride, // Model override for all agents
|
|
597
643
|
clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
|
|
644
|
+
settings: options.settings, // User settings for issue provider detection
|
|
645
|
+
forceProvider: options.forceProvider, // Force specific issue provider
|
|
598
646
|
});
|
|
599
647
|
}
|
|
600
648
|
|
|
@@ -619,50 +667,11 @@ class Orchestrator {
|
|
|
619
667
|
const messageBus = new MessageBus(ledger);
|
|
620
668
|
|
|
621
669
|
// Handle isolation mode (Docker container OR git worktree)
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
// Check Docker availability
|
|
628
|
-
if (!IsolationManager.isDockerAvailable()) {
|
|
629
|
-
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
// Ensure image exists (auto-build if missing)
|
|
633
|
-
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
634
|
-
await IsolationManager.ensureImage(image);
|
|
635
|
-
|
|
636
|
-
isolationManager = new IsolationManager({ image });
|
|
637
|
-
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
638
|
-
|
|
639
|
-
// Create container with workspace mounted
|
|
640
|
-
// CRITICAL: Use options.cwd (git repo root) instead of process.cwd()
|
|
641
|
-
const workDir = options.cwd || process.cwd();
|
|
642
|
-
const providerName = normalizeProviderName(
|
|
643
|
-
config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
|
|
644
|
-
);
|
|
645
|
-
containerId = await isolationManager.createContainer(clusterId, {
|
|
646
|
-
workDir,
|
|
647
|
-
image,
|
|
648
|
-
// Mount configuration (CLI overrides)
|
|
649
|
-
noMounts: options.noMounts,
|
|
650
|
-
mounts: options.mounts,
|
|
651
|
-
containerHome: options.containerHome,
|
|
652
|
-
provider: providerName,
|
|
653
|
-
});
|
|
654
|
-
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
655
|
-
} else if (options.worktree) {
|
|
656
|
-
// Worktree isolation: lightweight git-based isolation (no Docker required)
|
|
657
|
-
const workDir = options.cwd || process.cwd();
|
|
658
|
-
|
|
659
|
-
isolationManager = new IsolationManager({});
|
|
660
|
-
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
|
|
661
|
-
|
|
662
|
-
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
663
|
-
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
664
|
-
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
665
|
-
}
|
|
670
|
+
const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
|
|
671
|
+
options,
|
|
672
|
+
config,
|
|
673
|
+
clusterId
|
|
674
|
+
);
|
|
666
675
|
|
|
667
676
|
// Build cluster object
|
|
668
677
|
// CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
|
|
@@ -688,6 +697,10 @@ class Orchestrator {
|
|
|
688
697
|
autoPr: options.autoPr || false,
|
|
689
698
|
// Model override for all agents (applied to dynamically added agents)
|
|
690
699
|
modelOverride: options.modelOverride || null,
|
|
700
|
+
// Issue provider tracking (where issue was fetched from)
|
|
701
|
+
issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
|
|
702
|
+
// Git platform tracking (where PR/MR will be created - independent of issue provider)
|
|
703
|
+
gitPlatform: null, // Set when --pr mode is active
|
|
691
704
|
// Isolation state (only if enabled)
|
|
692
705
|
// CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
|
|
693
706
|
isolation: options.isolation
|
|
@@ -715,134 +728,67 @@ class Orchestrator {
|
|
|
715
728
|
this.clusters.set(clusterId, cluster);
|
|
716
729
|
|
|
717
730
|
try {
|
|
718
|
-
// Fetch input (
|
|
731
|
+
// Fetch input (issue from provider, file, or text)
|
|
719
732
|
let inputData;
|
|
720
733
|
if (input.issue) {
|
|
721
|
-
|
|
734
|
+
// Detect provider and fetch issue
|
|
735
|
+
const ProviderClass = detectProvider(
|
|
736
|
+
input.issue,
|
|
737
|
+
options.settings || {},
|
|
738
|
+
options.forceProvider
|
|
739
|
+
);
|
|
740
|
+
if (!ProviderClass) {
|
|
741
|
+
throw new Error(`No issue provider matched input: ${input.issue}`);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
const provider = new ProviderClass();
|
|
745
|
+
inputData = await provider.fetchIssue(input.issue, options.settings || {});
|
|
746
|
+
|
|
747
|
+
// Store issue provider for logging/debugging and cross-provider workflows
|
|
748
|
+
cluster.issueProvider = ProviderClass.id;
|
|
749
|
+
|
|
722
750
|
// Log clickable issue link
|
|
723
751
|
if (inputData.url) {
|
|
724
|
-
this._log(`[Orchestrator] Issue: ${inputData.url}`);
|
|
752
|
+
this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
|
|
725
753
|
}
|
|
726
754
|
} else if (input.file) {
|
|
727
|
-
inputData =
|
|
755
|
+
inputData = InputHelpers.createFileInput(input.file);
|
|
728
756
|
this._log(`[Orchestrator] File: ${input.file}`);
|
|
729
757
|
} else if (input.text) {
|
|
730
|
-
inputData =
|
|
758
|
+
inputData = InputHelpers.createTextInput(input.text);
|
|
731
759
|
} else {
|
|
732
760
|
throw new Error('Either issue, file, or text input is required');
|
|
733
761
|
}
|
|
734
762
|
|
|
735
|
-
//
|
|
763
|
+
// Detect git platform for --pr mode (independent of issue provider)
|
|
736
764
|
if (options.autoPr) {
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
// Load and configure git-pusher agent (use fs.readFileSync to avoid require cache)
|
|
741
|
-
const gitPusherPath = path.join(__dirname, 'agents', 'git-pusher-agent.json');
|
|
742
|
-
const gitPusherConfig = JSON.parse(fs.readFileSync(gitPusherPath, 'utf8'));
|
|
743
|
-
|
|
744
|
-
// Inject issue context placeholders
|
|
745
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
|
|
746
|
-
/\{\{issue_number\}\}/g,
|
|
747
|
-
inputData.number || 'unknown'
|
|
748
|
-
);
|
|
749
|
-
gitPusherConfig.prompt = gitPusherConfig.prompt.replace(
|
|
750
|
-
/\{\{issue_title\}\}/g,
|
|
751
|
-
inputData.title || 'Implementation'
|
|
752
|
-
);
|
|
765
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
766
|
+
const gitContext = detectGitContext(options.cwd);
|
|
767
|
+
cluster.gitPlatform = gitContext?.provider || null;
|
|
753
768
|
|
|
754
|
-
|
|
755
|
-
|
|
769
|
+
if (cluster.gitPlatform) {
|
|
770
|
+
this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
|
|
771
|
+
}
|
|
756
772
|
}
|
|
757
773
|
|
|
774
|
+
// Inject git-pusher agent if --pr is set (replaces completion-detector)
|
|
775
|
+
this._applyAutoPrConfig(config, inputData, options);
|
|
776
|
+
|
|
758
777
|
// Inject workers instruction if --workers explicitly provided and > 1
|
|
759
|
-
|
|
760
|
-
? parseInt(process.env.ZEROSHOT_WORKERS)
|
|
761
|
-
: 0;
|
|
762
|
-
if (workersCount > 1) {
|
|
763
|
-
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
764
|
-
if (workerAgent) {
|
|
765
|
-
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
766
|
-
|
|
767
|
-
if (!workerAgent.prompt) {
|
|
768
|
-
workerAgent.prompt = instruction;
|
|
769
|
-
} else if (typeof workerAgent.prompt === 'string') {
|
|
770
|
-
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
771
|
-
} else if (workerAgent.prompt.system) {
|
|
772
|
-
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
773
|
-
}
|
|
774
|
-
this._log(
|
|
775
|
-
`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`
|
|
776
|
-
);
|
|
777
|
-
}
|
|
778
|
-
}
|
|
778
|
+
this._applyWorkerInstruction(config);
|
|
779
779
|
|
|
780
780
|
// Initialize agents with optional mock injection
|
|
781
781
|
// Check agent type: regular agent or subcluster
|
|
782
782
|
// CRITICAL: Inject cwd into each agent config for proper working directory
|
|
783
783
|
// In worktree mode, agents run in the worktree path (not original cwd)
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
if (options.modelOverride) {
|
|
793
|
-
applyModelOverride(agentConfig, options.modelOverride);
|
|
794
|
-
this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
const agentOptions = {
|
|
798
|
-
testMode: options.testMode || !!this.taskRunner, // Enable testMode if taskRunner provided
|
|
799
|
-
quiet: this.quiet,
|
|
800
|
-
modelOverride: options.modelOverride || null,
|
|
801
|
-
};
|
|
802
|
-
|
|
803
|
-
// Inject mock spawn function if provided (legacy mockExecutor API)
|
|
804
|
-
if (options.mockExecutor) {
|
|
805
|
-
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
// TaskRunner DI - new pattern for mocking task execution
|
|
809
|
-
// Creates a mockSpawnFn wrapper that delegates to the taskRunner
|
|
810
|
-
if (this.taskRunner) {
|
|
811
|
-
// CRITICAL: agent is a closure variable capturing the AgentWrapper instance
|
|
812
|
-
// We cannot access agent._selectModel() here because agent doesn't exist yet
|
|
813
|
-
// Solution: Pass a factory function that will be called when agent is available
|
|
814
|
-
agentOptions.taskRunner = this.taskRunner;
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
// Pass isolation context if enabled
|
|
818
|
-
if (cluster.isolation) {
|
|
819
|
-
agentOptions.isolation = {
|
|
820
|
-
enabled: true,
|
|
821
|
-
manager: isolationManager,
|
|
822
|
-
clusterId,
|
|
823
|
-
};
|
|
824
|
-
}
|
|
825
|
-
|
|
826
|
-
// Pass worktree context if enabled (lightweight isolation without Docker)
|
|
827
|
-
if (cluster.worktree) {
|
|
828
|
-
agentOptions.worktree = {
|
|
829
|
-
enabled: true,
|
|
830
|
-
path: cluster.worktree.path,
|
|
831
|
-
branch: cluster.worktree.branch,
|
|
832
|
-
repoRoot: cluster.worktree.repoRoot,
|
|
833
|
-
};
|
|
834
|
-
}
|
|
835
|
-
|
|
836
|
-
// Create agent or subcluster wrapper based on type
|
|
837
|
-
let agent;
|
|
838
|
-
if (agentConfig.type === 'subcluster') {
|
|
839
|
-
agent = new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
840
|
-
} else {
|
|
841
|
-
agent = new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
cluster.agents.push(agent);
|
|
845
|
-
}
|
|
784
|
+
this._initializeClusterAgents({
|
|
785
|
+
config,
|
|
786
|
+
cluster,
|
|
787
|
+
messageBus,
|
|
788
|
+
options,
|
|
789
|
+
isolationManager,
|
|
790
|
+
clusterId,
|
|
791
|
+
});
|
|
846
792
|
|
|
847
793
|
// ========================================================================
|
|
848
794
|
// CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
|
|
@@ -862,229 +808,11 @@ class Orchestrator {
|
|
|
862
808
|
// DO NOT move subscriptions after agent.start() - this will reintroduce
|
|
863
809
|
// the race condition fixed in issue #31.
|
|
864
810
|
// ========================================================================
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
handler(message);
|
|
871
|
-
}
|
|
872
|
-
});
|
|
873
|
-
};
|
|
874
|
-
|
|
875
|
-
// Watch for CLUSTER_COMPLETE message to auto-stop
|
|
876
|
-
subscribeToClusterTopic('CLUSTER_COMPLETE', (message) => {
|
|
877
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
878
|
-
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
879
|
-
this._log(`${'='.repeat(80)}`);
|
|
880
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
881
|
-
this._log(`Initiated by: ${message.sender}`);
|
|
882
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
883
|
-
|
|
884
|
-
// Auto-stop cluster
|
|
885
|
-
this.stop(clusterId).catch((err) => {
|
|
886
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
887
|
-
});
|
|
888
|
-
});
|
|
889
|
-
|
|
890
|
-
// Watch for CLUSTER_FAILED message to auto-stop
|
|
891
|
-
subscribeToClusterTopic('CLUSTER_FAILED', (message) => {
|
|
892
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
893
|
-
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
894
|
-
this._log(`${'='.repeat(80)}`);
|
|
895
|
-
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
896
|
-
this._log(`Agent: ${message.sender}`);
|
|
897
|
-
if (message.content?.text) {
|
|
898
|
-
this._log(`Details: ${message.content.text}`);
|
|
899
|
-
}
|
|
900
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
901
|
-
|
|
902
|
-
// Auto-stop cluster
|
|
903
|
-
this.stop(clusterId).catch((err) => {
|
|
904
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
905
|
-
});
|
|
906
|
-
});
|
|
907
|
-
|
|
908
|
-
// Watch for AGENT_ERROR - if critical agent fails, stop cluster
|
|
909
|
-
subscribeToClusterTopic('AGENT_ERROR', async (message) => {
|
|
910
|
-
const agentRole = message.content?.data?.role;
|
|
911
|
-
const attempts = message.content?.data?.attempts || 1;
|
|
912
|
-
|
|
913
|
-
// Save cluster state to persist failureInfo
|
|
914
|
-
await await this._saveClusters();
|
|
915
|
-
|
|
916
|
-
// Only stop cluster if non-validator agent exhausted retries
|
|
917
|
-
if (agentRole === 'implementation' && attempts >= 3) {
|
|
918
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
919
|
-
this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
|
|
920
|
-
this._log(`${'='.repeat(80)}`);
|
|
921
|
-
this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
|
|
922
|
-
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
923
|
-
this._log(`Stopping cluster - worker cannot continue`);
|
|
924
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
925
|
-
|
|
926
|
-
// Auto-stop cluster
|
|
927
|
-
this.stop(clusterId).catch((err) => {
|
|
928
|
-
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
929
|
-
});
|
|
930
|
-
}
|
|
931
|
-
});
|
|
932
|
-
|
|
933
|
-
// Persist agent state changes for accurate status display
|
|
934
|
-
messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
|
|
935
|
-
const event = message.content?.data?.event;
|
|
936
|
-
// Save on key state transitions that affect status display
|
|
937
|
-
if (
|
|
938
|
-
[
|
|
939
|
-
'TASK_STARTED',
|
|
940
|
-
'TASK_COMPLETED',
|
|
941
|
-
'PROCESS_SPAWNED',
|
|
942
|
-
'TASK_ID_ASSIGNED',
|
|
943
|
-
'STARTED',
|
|
944
|
-
].includes(event)
|
|
945
|
-
) {
|
|
946
|
-
await await this._saveClusters();
|
|
947
|
-
}
|
|
948
|
-
});
|
|
949
|
-
|
|
950
|
-
// Watch for stale agent detection (informational only)
|
|
951
|
-
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
952
|
-
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
953
|
-
|
|
954
|
-
const agentId = message.content?.data?.agent;
|
|
955
|
-
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
956
|
-
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
957
|
-
|
|
958
|
-
this._log(
|
|
959
|
-
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
960
|
-
);
|
|
961
|
-
this._log(` Analysis: ${analysis}`);
|
|
962
|
-
this._log(
|
|
963
|
-
` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
|
|
964
|
-
);
|
|
965
|
-
});
|
|
966
|
-
|
|
967
|
-
// CONDUCTOR WATCHDOG: If conductor completes but CLUSTER_OPERATIONS never arrives, FAIL FAST
|
|
968
|
-
// This catches the silent failure where conductor outputs result but hook fails to publish
|
|
969
|
-
const CONDUCTOR_WATCHDOG_TIMEOUT_MS = 30000; // 30 seconds
|
|
970
|
-
let conductorWatchdogTimer = null;
|
|
971
|
-
let conductorCompletedAt = null;
|
|
972
|
-
|
|
973
|
-
// Start watchdog when conductor completes
|
|
974
|
-
subscribeToClusterTopic('AGENT_LIFECYCLE', (message) => {
|
|
975
|
-
const event = message.content?.data?.event;
|
|
976
|
-
const role = message.content?.data?.role;
|
|
977
|
-
|
|
978
|
-
// Conductor completed - start watchdog
|
|
979
|
-
if (event === 'TASK_COMPLETED' && role === 'conductor') {
|
|
980
|
-
conductorCompletedAt = Date.now();
|
|
981
|
-
this._log(
|
|
982
|
-
`⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${CONDUCTOR_WATCHDOG_TIMEOUT_MS / 1000}s`
|
|
983
|
-
);
|
|
984
|
-
|
|
985
|
-
conductorWatchdogTimer = setTimeout(() => {
|
|
986
|
-
// Check if CLUSTER_OPERATIONS was received
|
|
987
|
-
const clusterOps = messageBus.query({ topic: 'CLUSTER_OPERATIONS', limit: 1 });
|
|
988
|
-
if (clusterOps.length === 0) {
|
|
989
|
-
console.error(`\n${'='.repeat(80)}`);
|
|
990
|
-
console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
|
|
991
|
-
console.error(`${'='.repeat(80)}`);
|
|
992
|
-
console.error(
|
|
993
|
-
`Conductor completed ${CONDUCTOR_WATCHDOG_TIMEOUT_MS / 1000}s ago but no CLUSTER_OPERATIONS`
|
|
994
|
-
);
|
|
995
|
-
console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
|
|
996
|
-
console.error(
|
|
997
|
-
`Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
|
|
998
|
-
);
|
|
999
|
-
console.error(`${'='.repeat(80)}\n`);
|
|
1000
|
-
|
|
1001
|
-
// Publish CLUSTER_FAILED to stop the cluster
|
|
1002
|
-
messageBus.publish({
|
|
1003
|
-
cluster_id: clusterId,
|
|
1004
|
-
topic: 'CLUSTER_FAILED',
|
|
1005
|
-
sender: 'orchestrator',
|
|
1006
|
-
content: {
|
|
1007
|
-
text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
|
|
1008
|
-
data: {
|
|
1009
|
-
reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
|
|
1010
|
-
conductorCompletedAt,
|
|
1011
|
-
timeoutMs: CONDUCTOR_WATCHDOG_TIMEOUT_MS,
|
|
1012
|
-
},
|
|
1013
|
-
},
|
|
1014
|
-
});
|
|
1015
|
-
}
|
|
1016
|
-
}, CONDUCTOR_WATCHDOG_TIMEOUT_MS);
|
|
1017
|
-
}
|
|
1018
|
-
});
|
|
1019
|
-
|
|
1020
|
-
// Watch for CLUSTER_OPERATIONS - dynamic agent spawn/removal/update
|
|
1021
|
-
subscribeToClusterTopic('CLUSTER_OPERATIONS', (message) => {
|
|
1022
|
-
// Clear conductor watchdog - CLUSTER_OPERATIONS received successfully
|
|
1023
|
-
if (conductorWatchdogTimer) {
|
|
1024
|
-
clearTimeout(conductorWatchdogTimer);
|
|
1025
|
-
conductorWatchdogTimer = null;
|
|
1026
|
-
const elapsed = conductorCompletedAt ? Date.now() - conductorCompletedAt : 0;
|
|
1027
|
-
this._log(
|
|
1028
|
-
`✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
|
|
1029
|
-
);
|
|
1030
|
-
}
|
|
1031
|
-
let operations = message.content?.data?.operations;
|
|
1032
|
-
|
|
1033
|
-
// Parse operations if they came as a JSON string
|
|
1034
|
-
if (typeof operations === 'string') {
|
|
1035
|
-
try {
|
|
1036
|
-
operations = JSON.parse(operations);
|
|
1037
|
-
} catch (e) {
|
|
1038
|
-
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
1039
|
-
return;
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
if (!operations || !Array.isArray(operations)) {
|
|
1044
|
-
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
1045
|
-
return;
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
1049
|
-
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
1050
|
-
this._log(`${'='.repeat(80)}`);
|
|
1051
|
-
if (message.content?.data?.reasoning) {
|
|
1052
|
-
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
1053
|
-
}
|
|
1054
|
-
this._log(`Operations: ${operations.length}`);
|
|
1055
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
1056
|
-
|
|
1057
|
-
// Execute operation chain
|
|
1058
|
-
this._handleOperations(clusterId, operations, message.sender, {
|
|
1059
|
-
isolationManager,
|
|
1060
|
-
containerId,
|
|
1061
|
-
}).catch((err) => {
|
|
1062
|
-
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
1063
|
-
// Publish failure message
|
|
1064
|
-
messageBus.publish({
|
|
1065
|
-
cluster_id: clusterId,
|
|
1066
|
-
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
1067
|
-
sender: 'orchestrator',
|
|
1068
|
-
content: {
|
|
1069
|
-
text: `Operation chain failed: ${err.message}`,
|
|
1070
|
-
data: {
|
|
1071
|
-
error: err.message,
|
|
1072
|
-
operations: operations,
|
|
1073
|
-
},
|
|
1074
|
-
},
|
|
1075
|
-
});
|
|
1076
|
-
|
|
1077
|
-
// CRITICAL: Stop cluster on operation failure
|
|
1078
|
-
this._log(`\n${'='.repeat(80)}`);
|
|
1079
|
-
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
1080
|
-
this._log(`${'='.repeat(80)}`);
|
|
1081
|
-
this._log(`Error: ${err.message}`);
|
|
1082
|
-
this._log(`${'='.repeat(80)}\n`);
|
|
1083
|
-
|
|
1084
|
-
this.stop(clusterId).catch((stopErr) => {
|
|
1085
|
-
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
1086
|
-
});
|
|
1087
|
-
});
|
|
811
|
+
this._registerClusterSubscriptions({
|
|
812
|
+
messageBus,
|
|
813
|
+
clusterId,
|
|
814
|
+
isolationManager,
|
|
815
|
+
containerId,
|
|
1088
816
|
});
|
|
1089
817
|
|
|
1090
818
|
// Start all agents
|
|
@@ -1108,7 +836,7 @@ class Orchestrator {
|
|
|
1108
836
|
},
|
|
1109
837
|
},
|
|
1110
838
|
metadata: {
|
|
1111
|
-
source:
|
|
839
|
+
source: this._getInputSource(input),
|
|
1112
840
|
},
|
|
1113
841
|
});
|
|
1114
842
|
|
|
@@ -1156,36 +884,475 @@ class Orchestrator {
|
|
|
1156
884
|
}
|
|
1157
885
|
}
|
|
1158
886
|
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
* If an explicit ID is provided, uses it as a base and suffixes on collision.
|
|
1162
|
-
* @private
|
|
1163
|
-
*/
|
|
1164
|
-
_generateUniqueClusterId(explicitId, explicitDbPath) {
|
|
1165
|
-
const baseId = explicitId || generateName('cluster');
|
|
1166
|
-
const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
|
|
1167
|
-
|
|
1168
|
-
// Fast path: base is unused.
|
|
1169
|
-
if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
|
|
1170
|
-
return baseId;
|
|
1171
|
-
}
|
|
887
|
+
_initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
|
|
888
|
+
const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
|
|
1172
889
|
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
const candidateId = `${baseId}-${suffix}`;
|
|
1177
|
-
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1178
|
-
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1179
|
-
return candidateId;
|
|
890
|
+
for (const agentConfig of config.agents) {
|
|
891
|
+
if (!agentConfig.cwd) {
|
|
892
|
+
agentConfig.cwd = agentCwd;
|
|
1180
893
|
}
|
|
1181
|
-
}
|
|
1182
894
|
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
895
|
+
if (options.modelOverride) {
|
|
896
|
+
applyModelOverride(agentConfig, options.modelOverride);
|
|
897
|
+
this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
const agentOptions = {
|
|
901
|
+
testMode: options.testMode || !!this.taskRunner,
|
|
902
|
+
quiet: this.quiet,
|
|
903
|
+
modelOverride: options.modelOverride || null,
|
|
904
|
+
};
|
|
905
|
+
|
|
906
|
+
if (options.mockExecutor) {
|
|
907
|
+
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
if (this.taskRunner) {
|
|
911
|
+
agentOptions.taskRunner = this.taskRunner;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
if (cluster.isolation) {
|
|
915
|
+
agentOptions.isolation = {
|
|
916
|
+
enabled: true,
|
|
917
|
+
manager: isolationManager,
|
|
918
|
+
clusterId,
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
if (cluster.worktree) {
|
|
923
|
+
agentOptions.worktree = {
|
|
924
|
+
enabled: true,
|
|
925
|
+
path: cluster.worktree.path,
|
|
926
|
+
branch: cluster.worktree.branch,
|
|
927
|
+
repoRoot: cluster.worktree.repoRoot,
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
const agent =
|
|
932
|
+
agentConfig.type === 'subcluster'
|
|
933
|
+
? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
|
|
934
|
+
: new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
935
|
+
|
|
936
|
+
cluster.agents.push(agent);
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
_subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
|
|
941
|
+
messageBus.subscribe((message) => {
|
|
942
|
+
if (message.topic === topic && message.cluster_id === clusterId) {
|
|
943
|
+
handler(message);
|
|
944
|
+
}
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
_registerClusterCompletionHandlers(messageBus, clusterId) {
|
|
949
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
|
|
950
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
951
|
+
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
952
|
+
this._log(`${'='.repeat(80)}`);
|
|
953
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
954
|
+
this._log(`Initiated by: ${message.sender}`);
|
|
955
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
956
|
+
|
|
957
|
+
this.stop(clusterId).catch((err) => {
|
|
958
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
959
|
+
});
|
|
960
|
+
});
|
|
961
|
+
|
|
962
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
|
|
963
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
964
|
+
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
965
|
+
this._log(`${'='.repeat(80)}`);
|
|
966
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
967
|
+
this._log(`Agent: ${message.sender}`);
|
|
968
|
+
if (message.content?.text) {
|
|
969
|
+
this._log(`Details: ${message.content.text}`);
|
|
970
|
+
}
|
|
971
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
972
|
+
|
|
973
|
+
this.stop(clusterId).catch((err) => {
|
|
974
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
975
|
+
});
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
_registerAgentErrorHandler(messageBus, clusterId) {
|
|
980
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
|
|
981
|
+
const agentRole = message.content?.data?.role;
|
|
982
|
+
const attempts = message.content?.data?.attempts || 1;
|
|
983
|
+
|
|
984
|
+
await this._saveClusters();
|
|
985
|
+
|
|
986
|
+
if (agentRole === 'implementation' && attempts >= 3) {
|
|
987
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
988
|
+
this._log(`❌ WORKER AGENT FAILED: ${clusterId}`);
|
|
989
|
+
this._log(`${'='.repeat(80)}`);
|
|
990
|
+
this._log(`Worker agent ${message.sender} failed after ${attempts} attempts`);
|
|
991
|
+
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
992
|
+
this._log(`Stopping cluster - worker cannot continue`);
|
|
993
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
994
|
+
|
|
995
|
+
this.stop(clusterId).catch((err) => {
|
|
996
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
_registerAgentLifecycleHandlers(messageBus, clusterId) {
|
|
1003
|
+
messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
|
|
1004
|
+
const event = message.content?.data?.event;
|
|
1005
|
+
if (
|
|
1006
|
+
[
|
|
1007
|
+
'TASK_STARTED',
|
|
1008
|
+
'TASK_COMPLETED',
|
|
1009
|
+
'PROCESS_SPAWNED',
|
|
1010
|
+
'TASK_ID_ASSIGNED',
|
|
1011
|
+
'STARTED',
|
|
1012
|
+
].includes(event)
|
|
1013
|
+
) {
|
|
1014
|
+
await this._saveClusters();
|
|
1015
|
+
}
|
|
1016
|
+
});
|
|
1017
|
+
|
|
1018
|
+
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
1019
|
+
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
1020
|
+
|
|
1021
|
+
const agentId = message.content?.data?.agent;
|
|
1022
|
+
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
1023
|
+
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
1024
|
+
|
|
1025
|
+
this._log(
|
|
1026
|
+
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
1027
|
+
);
|
|
1028
|
+
this._log(` Analysis: ${analysis}`);
|
|
1029
|
+
this._log(
|
|
1030
|
+
` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
|
|
1031
|
+
);
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
_registerConductorWatchdog(messageBus, clusterId) {
|
|
1036
|
+
const timeoutMs = 30000;
|
|
1037
|
+
let watchdogTimer = null;
|
|
1038
|
+
let completedAt = null;
|
|
1039
|
+
|
|
1040
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
|
|
1041
|
+
const event = message.content?.data?.event;
|
|
1042
|
+
const role = message.content?.data?.role;
|
|
1043
|
+
|
|
1044
|
+
if (event === 'TASK_COMPLETED' && role === 'conductor') {
|
|
1045
|
+
completedAt = Date.now();
|
|
1046
|
+
this._log(
|
|
1047
|
+
`⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
|
|
1048
|
+
);
|
|
1049
|
+
|
|
1050
|
+
watchdogTimer = setTimeout(() => {
|
|
1051
|
+
const clusterOps = messageBus.query({
|
|
1052
|
+
cluster_id: clusterId,
|
|
1053
|
+
topic: 'CLUSTER_OPERATIONS',
|
|
1054
|
+
limit: 1,
|
|
1055
|
+
});
|
|
1056
|
+
if (clusterOps.length === 0) {
|
|
1057
|
+
console.error(`\n${'='.repeat(80)}`);
|
|
1058
|
+
console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
|
|
1059
|
+
console.error(`${'='.repeat(80)}`);
|
|
1060
|
+
console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
|
|
1061
|
+
console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
|
|
1062
|
+
console.error(
|
|
1063
|
+
`Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
|
|
1064
|
+
);
|
|
1065
|
+
console.error(`${'='.repeat(80)}\n`);
|
|
1066
|
+
|
|
1067
|
+
messageBus.publish({
|
|
1068
|
+
cluster_id: clusterId,
|
|
1069
|
+
topic: 'CLUSTER_FAILED',
|
|
1070
|
+
sender: 'orchestrator',
|
|
1071
|
+
content: {
|
|
1072
|
+
text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
|
|
1073
|
+
data: {
|
|
1074
|
+
reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
|
|
1075
|
+
conductorCompletedAt: completedAt,
|
|
1076
|
+
timeoutMs: timeoutMs,
|
|
1077
|
+
},
|
|
1078
|
+
},
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
}, timeoutMs);
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
return {
|
|
1086
|
+
clear: () => {
|
|
1087
|
+
if (!watchdogTimer) {
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
1090
|
+
clearTimeout(watchdogTimer);
|
|
1091
|
+
watchdogTimer = null;
|
|
1092
|
+
const elapsed = completedAt ? Date.now() - completedAt : 0;
|
|
1093
|
+
this._log(
|
|
1094
|
+
`✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
|
|
1095
|
+
);
|
|
1096
|
+
},
|
|
1097
|
+
};
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
_registerClusterOperationsHandler(
|
|
1101
|
+
messageBus,
|
|
1102
|
+
clusterId,
|
|
1103
|
+
isolationManager,
|
|
1104
|
+
containerId,
|
|
1105
|
+
watchdog
|
|
1106
|
+
) {
|
|
1107
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
|
|
1108
|
+
watchdog?.clear?.();
|
|
1109
|
+
|
|
1110
|
+
let operations = message.content?.data?.operations;
|
|
1111
|
+
if (typeof operations === 'string') {
|
|
1112
|
+
try {
|
|
1113
|
+
operations = JSON.parse(operations);
|
|
1114
|
+
} catch (e) {
|
|
1115
|
+
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
if (!operations || !Array.isArray(operations)) {
|
|
1121
|
+
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1126
|
+
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
1127
|
+
this._log(`${'='.repeat(80)}`);
|
|
1128
|
+
if (message.content?.data?.reasoning) {
|
|
1129
|
+
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
1130
|
+
}
|
|
1131
|
+
this._log(`Operations: ${operations.length}`);
|
|
1132
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1133
|
+
|
|
1134
|
+
this._handleOperations(clusterId, operations, message.sender, {
|
|
1135
|
+
isolationManager,
|
|
1136
|
+
containerId,
|
|
1137
|
+
}).catch((err) => {
|
|
1138
|
+
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
1139
|
+
messageBus.publish({
|
|
1140
|
+
cluster_id: clusterId,
|
|
1141
|
+
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
1142
|
+
sender: 'orchestrator',
|
|
1143
|
+
content: {
|
|
1144
|
+
text: `Operation chain failed: ${err.message}`,
|
|
1145
|
+
data: {
|
|
1146
|
+
error: err.message,
|
|
1147
|
+
operations: operations,
|
|
1148
|
+
},
|
|
1149
|
+
},
|
|
1150
|
+
});
|
|
1151
|
+
|
|
1152
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1153
|
+
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
1154
|
+
this._log(`${'='.repeat(80)}`);
|
|
1155
|
+
this._log(`Error: ${err.message}`);
|
|
1156
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1157
|
+
|
|
1158
|
+
this.stop(clusterId).catch((stopErr) => {
|
|
1159
|
+
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
1160
|
+
});
|
|
1161
|
+
});
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
_registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
|
|
1166
|
+
this._registerClusterCompletionHandlers(messageBus, clusterId);
|
|
1167
|
+
this._registerAgentErrorHandler(messageBus, clusterId);
|
|
1168
|
+
this._registerAgentLifecycleHandlers(messageBus, clusterId);
|
|
1169
|
+
|
|
1170
|
+
const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
|
|
1171
|
+
this._registerClusterOperationsHandler(
|
|
1172
|
+
messageBus,
|
|
1173
|
+
clusterId,
|
|
1174
|
+
isolationManager,
|
|
1175
|
+
containerId,
|
|
1176
|
+
watchdog
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
async _initializeIsolation(options, config, clusterId) {
|
|
1181
|
+
let isolationManager = null;
|
|
1182
|
+
let containerId = null;
|
|
1183
|
+
let worktreeInfo = null;
|
|
1184
|
+
|
|
1185
|
+
if (options.isolation) {
|
|
1186
|
+
if (!IsolationManager.isDockerAvailable()) {
|
|
1187
|
+
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
1191
|
+
await IsolationManager.ensureImage(image);
|
|
1192
|
+
|
|
1193
|
+
isolationManager = new IsolationManager({ image });
|
|
1194
|
+
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
1195
|
+
|
|
1196
|
+
const workDir = options.cwd || process.cwd();
|
|
1197
|
+
const providerName = normalizeProviderName(
|
|
1198
|
+
config.forceProvider || config.defaultProvider || loadSettings().defaultProvider || 'claude'
|
|
1199
|
+
);
|
|
1200
|
+
containerId = await isolationManager.createContainer(clusterId, {
|
|
1201
|
+
workDir,
|
|
1202
|
+
image,
|
|
1203
|
+
noMounts: options.noMounts,
|
|
1204
|
+
mounts: options.mounts,
|
|
1205
|
+
containerHome: options.containerHome,
|
|
1206
|
+
provider: providerName,
|
|
1207
|
+
});
|
|
1208
|
+
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
1209
|
+
} else if (options.worktree) {
|
|
1210
|
+
const workDir = options.cwd || process.cwd();
|
|
1211
|
+
|
|
1212
|
+
isolationManager = new IsolationManager({});
|
|
1213
|
+
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir);
|
|
1214
|
+
|
|
1215
|
+
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
1216
|
+
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
1217
|
+
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
return { isolationManager, containerId, worktreeInfo };
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
_applyAutoPrConfig(config, inputData, options) {
|
|
1224
|
+
if (!options.autoPr) {
|
|
1225
|
+
return;
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
|
|
1229
|
+
|
|
1230
|
+
// Detect git platform (independent of issue provider)
|
|
1231
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
1232
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
1233
|
+
|
|
1234
|
+
let platform;
|
|
1235
|
+
try {
|
|
1236
|
+
// ALWAYS use git context, regardless of issue provider
|
|
1237
|
+
// This allows Jira issues in GitHub repos, etc.
|
|
1238
|
+
platform = getPlatformForPR(options.cwd);
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
throw new Error(`--pr mode failed: ${error.message}`);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// Check for repo mismatch (issue URL repo vs current git remote)
|
|
1244
|
+
// This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
|
|
1245
|
+
// from a different repo directory
|
|
1246
|
+
let skipCloseIssue = false;
|
|
1247
|
+
const gitContext = detectGitContext(options.cwd);
|
|
1248
|
+
if (inputData.url && gitContext) {
|
|
1249
|
+
const issueHost = this._extractHostFromUrl(inputData.url);
|
|
1250
|
+
const gitHost = gitContext.host;
|
|
1251
|
+
|
|
1252
|
+
// Compare hosts - if different platforms, warn and don't close issue
|
|
1253
|
+
if (issueHost && gitHost && issueHost !== gitHost) {
|
|
1254
|
+
this._log(
|
|
1255
|
+
`[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
|
|
1256
|
+
);
|
|
1257
|
+
this._log(
|
|
1258
|
+
`[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
|
|
1259
|
+
);
|
|
1260
|
+
this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
|
|
1261
|
+
skipCloseIssue = true;
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
// Generate platform-specific git-pusher agent from template
|
|
1266
|
+
const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
|
|
1267
|
+
|
|
1268
|
+
if (!isPlatformSupported(platform)) {
|
|
1269
|
+
throw new Error(
|
|
1270
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
1271
|
+
);
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
1275
|
+
|
|
1276
|
+
// Template replacement for issue context
|
|
1277
|
+
const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
|
|
1278
|
+
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
1279
|
+
.replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
|
|
1280
|
+
.replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
|
|
1281
|
+
.replace(/Closes #\{\{issue_number\}\}/g, issueRef);
|
|
1282
|
+
|
|
1283
|
+
config.agents.push(gitPusherConfig);
|
|
1284
|
+
this._log(
|
|
1285
|
+
`[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
/**
|
|
1290
|
+
* Extract hostname from a URL
|
|
1291
|
+
* @private
|
|
1292
|
+
*/
|
|
1293
|
+
_extractHostFromUrl(url) {
|
|
1294
|
+
try {
|
|
1295
|
+
const match = url.match(/https?:\/\/([^/]+)/);
|
|
1296
|
+
return match ? match[1] : null;
|
|
1297
|
+
} catch {
|
|
1298
|
+
return null;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
_applyWorkerInstruction(config) {
|
|
1303
|
+
const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
|
|
1304
|
+
if (workersCount <= 1) {
|
|
1305
|
+
return;
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
1309
|
+
if (!workerAgent) {
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
1314
|
+
|
|
1315
|
+
if (!workerAgent.prompt) {
|
|
1316
|
+
workerAgent.prompt = instruction;
|
|
1317
|
+
} else if (typeof workerAgent.prompt === 'string') {
|
|
1318
|
+
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
1319
|
+
} else if (workerAgent.prompt.system) {
|
|
1320
|
+
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
/**
|
|
1327
|
+
* Generate a unique cluster ID, safe for concurrent starts in-process.
|
|
1328
|
+
* If an explicit ID is provided, uses it as a base and suffixes on collision.
|
|
1329
|
+
* @private
|
|
1330
|
+
*/
|
|
1331
|
+
_generateUniqueClusterId(explicitId, explicitDbPath) {
|
|
1332
|
+
const baseId = explicitId || generateName('cluster');
|
|
1333
|
+
const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
|
|
1334
|
+
|
|
1335
|
+
// Fast path: base is unused.
|
|
1336
|
+
if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
|
|
1337
|
+
return baseId;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
// Collision: suffix with random bytes to avoid race conditions under concurrency.
|
|
1341
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1342
|
+
const suffix = crypto.randomBytes(3).toString('hex');
|
|
1343
|
+
const candidateId = `${baseId}-${suffix}`;
|
|
1344
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1345
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1346
|
+
return candidateId;
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
// Last resort: new generated name (should never happen).
|
|
1351
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1352
|
+
const candidateId = generateName('cluster');
|
|
1353
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1354
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1355
|
+
return candidateId;
|
|
1189
1356
|
}
|
|
1190
1357
|
}
|
|
1191
1358
|
|
|
@@ -1362,284 +1529,303 @@ class Orchestrator {
|
|
|
1362
1529
|
);
|
|
1363
1530
|
}
|
|
1364
1531
|
|
|
1365
|
-
|
|
1366
|
-
let failureInfo = cluster.failureInfo;
|
|
1532
|
+
const failureInfo = this._resolveFailureInfo(cluster, clusterId);
|
|
1367
1533
|
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
cluster_id: clusterId,
|
|
1372
|
-
topic: 'AGENT_ERROR',
|
|
1373
|
-
limit: 10,
|
|
1374
|
-
order: 'desc',
|
|
1375
|
-
});
|
|
1534
|
+
await this._ensureIsolationForResume(clusterId, cluster);
|
|
1535
|
+
this._ensureWorktreeForResume(clusterId, cluster);
|
|
1536
|
+
await this._restartClusterAgents(cluster);
|
|
1376
1537
|
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
agentId: firstError.sender,
|
|
1382
|
-
taskId: firstError.content?.data?.taskId || null,
|
|
1383
|
-
iteration: firstError.content?.data?.iteration || 0,
|
|
1384
|
-
error: firstError.content?.data?.error || firstError.content?.text,
|
|
1385
|
-
timestamp: firstError.timestamp,
|
|
1386
|
-
};
|
|
1387
|
-
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
1388
|
-
}
|
|
1538
|
+
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
1539
|
+
|
|
1540
|
+
if (failureInfo) {
|
|
1541
|
+
return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
|
|
1389
1542
|
}
|
|
1390
1543
|
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
const { spawn } = require('child_process');
|
|
1394
|
-
const oldContainerId = cluster.isolation.containerId;
|
|
1544
|
+
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
|
|
1545
|
+
}
|
|
1395
1546
|
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
const containerExists = await new Promise((resolve) => {
|
|
1401
|
-
checkContainer.on('close', (code) => resolve(code === 0));
|
|
1402
|
-
});
|
|
1547
|
+
_resolveFailureInfo(cluster, clusterId) {
|
|
1548
|
+
if (cluster.failureInfo) {
|
|
1549
|
+
return cluster.failureInfo;
|
|
1550
|
+
}
|
|
1403
1551
|
|
|
1404
|
-
|
|
1405
|
-
|
|
1552
|
+
const errors = cluster.messageBus.query({
|
|
1553
|
+
cluster_id: clusterId,
|
|
1554
|
+
topic: 'AGENT_ERROR',
|
|
1555
|
+
limit: 10,
|
|
1556
|
+
order: 'desc',
|
|
1557
|
+
});
|
|
1406
1558
|
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
if (!workDir) {
|
|
1411
|
-
throw new Error(
|
|
1412
|
-
`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`
|
|
1413
|
-
);
|
|
1414
|
-
}
|
|
1559
|
+
if (errors.length === 0) {
|
|
1560
|
+
return null;
|
|
1561
|
+
}
|
|
1415
1562
|
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1563
|
+
const firstError = errors[0];
|
|
1564
|
+
const failureInfo = {
|
|
1565
|
+
agentId: firstError.sender,
|
|
1566
|
+
taskId: firstError.content?.data?.taskId || null,
|
|
1567
|
+
iteration: firstError.content?.data?.iteration || 0,
|
|
1568
|
+
error: firstError.content?.data?.error || firstError.content?.text,
|
|
1569
|
+
timestamp: firstError.timestamp,
|
|
1570
|
+
};
|
|
1571
|
+
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
1572
|
+
return failureInfo;
|
|
1573
|
+
}
|
|
1424
1574
|
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
workDir, // Use saved workDir, NOT process.cwd()
|
|
1433
|
-
image: cluster.isolation.image,
|
|
1434
|
-
reuseExistingWorkspace: true, // CRITICAL: Don't wipe existing work
|
|
1435
|
-
provider: providerName,
|
|
1436
|
-
});
|
|
1575
|
+
_checkContainerExists(containerId) {
|
|
1576
|
+
const { spawn } = require('child_process');
|
|
1577
|
+
const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
|
|
1578
|
+
return new Promise((resolve) => {
|
|
1579
|
+
checkContainer.on('close', (code) => resolve(code === 0));
|
|
1580
|
+
});
|
|
1581
|
+
}
|
|
1437
1582
|
|
|
1438
|
-
|
|
1583
|
+
async _ensureIsolationForResume(clusterId, cluster) {
|
|
1584
|
+
if (!cluster.isolation?.enabled) {
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1439
1587
|
|
|
1440
|
-
|
|
1441
|
-
|
|
1588
|
+
const oldContainerId = cluster.isolation.containerId;
|
|
1589
|
+
const containerExists = await this._checkContainerExists(oldContainerId);
|
|
1590
|
+
if (containerExists) {
|
|
1591
|
+
this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1442
1594
|
|
|
1443
|
-
|
|
1444
|
-
for (const agent of cluster.agents) {
|
|
1445
|
-
if (agent.isolation?.enabled) {
|
|
1446
|
-
agent.isolation.containerId = newContainerId;
|
|
1447
|
-
agent.isolation.manager = cluster.isolation.manager;
|
|
1448
|
-
}
|
|
1449
|
-
}
|
|
1595
|
+
this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
|
|
1450
1596
|
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1597
|
+
const workDir = cluster.isolation.workDir;
|
|
1598
|
+
if (!workDir) {
|
|
1599
|
+
throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
|
|
1603
|
+
if (!fs.existsSync(isolatedPath)) {
|
|
1604
|
+
throw new Error(
|
|
1605
|
+
`Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
|
|
1606
|
+
`Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
|
|
1607
|
+
);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
const providerName = normalizeProviderName(
|
|
1611
|
+
cluster.config?.forceProvider ||
|
|
1612
|
+
cluster.config?.defaultProvider ||
|
|
1613
|
+
loadSettings().defaultProvider ||
|
|
1614
|
+
'claude'
|
|
1615
|
+
);
|
|
1616
|
+
const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
|
|
1617
|
+
workDir,
|
|
1618
|
+
image: cluster.isolation.image,
|
|
1619
|
+
reuseExistingWorkspace: true,
|
|
1620
|
+
provider: providerName,
|
|
1621
|
+
});
|
|
1622
|
+
|
|
1623
|
+
this._log(`[Orchestrator] New container created: ${newContainerId}`);
|
|
1624
|
+
cluster.isolation.containerId = newContainerId;
|
|
1625
|
+
|
|
1626
|
+
for (const agent of cluster.agents) {
|
|
1627
|
+
if (agent.isolation?.enabled) {
|
|
1628
|
+
agent.isolation.containerId = newContainerId;
|
|
1629
|
+
agent.isolation.manager = cluster.isolation.manager;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
this._log(`[Orchestrator] All agents updated with new container ID`);
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
_ensureWorktreeForResume(clusterId, cluster) {
|
|
1637
|
+
if (!cluster.worktree?.enabled) {
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
const worktreePath = cluster.worktree.path;
|
|
1642
|
+
if (!fs.existsSync(worktreePath)) {
|
|
1643
|
+
throw new Error(
|
|
1644
|
+
`Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
|
|
1645
|
+
`Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
|
|
1650
|
+
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
async _restartClusterAgents(cluster) {
|
|
1654
|
+
cluster.state = 'running';
|
|
1655
|
+
for (const agent of cluster.agents) {
|
|
1656
|
+
if (!agent.running) {
|
|
1657
|
+
await agent.start();
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
_loadRecentMessages(cluster, clusterId, limit) {
|
|
1663
|
+
return cluster.messageBus
|
|
1664
|
+
.query({
|
|
1665
|
+
cluster_id: clusterId,
|
|
1666
|
+
limit,
|
|
1667
|
+
order: 'desc',
|
|
1668
|
+
})
|
|
1669
|
+
.reverse();
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
_buildResumeContext(recentMessages, prompt, options) {
|
|
1673
|
+
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
1674
|
+
let context = `${options.header}\n\n`;
|
|
1675
|
+
|
|
1676
|
+
if (options.error) {
|
|
1677
|
+
context += `Previous error: ${options.error}\n\n`;
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
context += `## Recent Context\n\n`;
|
|
1681
|
+
for (const msg of recentMessages.slice(-10)) {
|
|
1682
|
+
if (options.topics.includes(msg.topic)) {
|
|
1683
|
+
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1454
1684
|
}
|
|
1455
1685
|
}
|
|
1456
1686
|
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
if (!fs.existsSync(worktreePath)) {
|
|
1461
|
-
throw new Error(
|
|
1462
|
-
`Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
|
|
1463
|
-
`Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
|
|
1464
|
-
);
|
|
1465
|
-
}
|
|
1687
|
+
context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
|
|
1688
|
+
return context;
|
|
1689
|
+
}
|
|
1466
1690
|
|
|
1467
|
-
|
|
1468
|
-
|
|
1691
|
+
_triggerMatchesTopic(triggerTopic, topic) {
|
|
1692
|
+
if (triggerTopic === topic) return true;
|
|
1693
|
+
if (triggerTopic === '*') return true;
|
|
1694
|
+
if (triggerTopic.endsWith('*')) {
|
|
1695
|
+
const prefix = triggerTopic.slice(0, -1);
|
|
1696
|
+
return topic.startsWith(prefix);
|
|
1469
1697
|
}
|
|
1698
|
+
return false;
|
|
1699
|
+
}
|
|
1470
1700
|
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
for (const agent of cluster.agents) {
|
|
1474
|
-
if (!agent.running) {
|
|
1475
|
-
await agent.start();
|
|
1476
|
-
}
|
|
1477
|
-
}
|
|
1701
|
+
_selectAgentsToResume(cluster, lastTrigger) {
|
|
1702
|
+
const agentsToResume = [];
|
|
1478
1703
|
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
.query({
|
|
1482
|
-
cluster_id: clusterId,
|
|
1483
|
-
limit: 50,
|
|
1484
|
-
order: 'desc',
|
|
1485
|
-
})
|
|
1486
|
-
.reverse();
|
|
1704
|
+
for (const agent of cluster.agents) {
|
|
1705
|
+
if (!agent.config.triggers) continue;
|
|
1487
1706
|
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
const { agentId, iteration, error } = failureInfo;
|
|
1491
|
-
this._log(
|
|
1492
|
-
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
1707
|
+
const matchingTrigger = agent.config.triggers.find((trigger) =>
|
|
1708
|
+
this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
|
|
1493
1709
|
);
|
|
1494
|
-
|
|
1710
|
+
if (!matchingTrigger) continue;
|
|
1495
1711
|
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
1712
|
+
if (matchingTrigger.logic?.script) {
|
|
1713
|
+
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
1714
|
+
if (!shouldTrigger) continue;
|
|
1500
1715
|
}
|
|
1501
1716
|
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
let context = `You are resuming from a previous failed attempt.\n\n`;
|
|
1505
|
-
context += `Previous error: ${error}\n\n`;
|
|
1506
|
-
context += `## Recent Context\n\n`;
|
|
1717
|
+
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
1718
|
+
}
|
|
1507
1719
|
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1511
|
-
}
|
|
1512
|
-
}
|
|
1720
|
+
return agentsToResume;
|
|
1721
|
+
}
|
|
1513
1722
|
|
|
1514
|
-
|
|
1723
|
+
_resumeAgents(agentsToResume, context) {
|
|
1724
|
+
for (const { agent, message } of agentsToResume) {
|
|
1725
|
+
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
1726
|
+
agent.resume(context).catch((err) => {
|
|
1727
|
+
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
1728
|
+
});
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1515
1731
|
|
|
1516
|
-
|
|
1517
|
-
|
|
1732
|
+
_republishIssue(cluster, clusterId, recentMessages) {
|
|
1733
|
+
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
1734
|
+
if (!issueMessage) {
|
|
1735
|
+
return false;
|
|
1736
|
+
}
|
|
1518
1737
|
|
|
1519
|
-
|
|
1520
|
-
|
|
1738
|
+
cluster.messageBus.publish({
|
|
1739
|
+
cluster_id: clusterId,
|
|
1740
|
+
topic: 'ISSUE_OPENED',
|
|
1741
|
+
sender: 'system',
|
|
1742
|
+
receiver: 'broadcast',
|
|
1743
|
+
content: issueMessage.content,
|
|
1744
|
+
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
1745
|
+
});
|
|
1521
1746
|
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
|
|
1525
|
-
});
|
|
1747
|
+
return true;
|
|
1748
|
+
}
|
|
1526
1749
|
|
|
1527
|
-
|
|
1750
|
+
async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
|
|
1751
|
+
const { agentId, iteration, error } = failureInfo;
|
|
1752
|
+
this._log(
|
|
1753
|
+
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
1754
|
+
);
|
|
1755
|
+
this._log(`[Orchestrator] Previous error: ${error}`);
|
|
1528
1756
|
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
resumeType: 'failure',
|
|
1533
|
-
resumedAgent: agentId,
|
|
1534
|
-
previousError: error,
|
|
1535
|
-
};
|
|
1757
|
+
const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
|
|
1758
|
+
if (!failedAgent) {
|
|
1759
|
+
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
1536
1760
|
}
|
|
1537
1761
|
|
|
1538
|
-
|
|
1539
|
-
|
|
1762
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1763
|
+
header: 'You are resuming from a previous failed attempt.',
|
|
1764
|
+
error,
|
|
1765
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
|
|
1766
|
+
});
|
|
1540
1767
|
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
let context = `Resuming cluster from previous session.\n\n`;
|
|
1544
|
-
context += `## Recent Context\n\n`;
|
|
1768
|
+
cluster.failureInfo = null;
|
|
1769
|
+
await this._saveClusters();
|
|
1545
1770
|
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
msg.topic === 'VALIDATION_RESULT' ||
|
|
1550
|
-
msg.topic === 'ISSUE_OPENED'
|
|
1551
|
-
) {
|
|
1552
|
-
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
1553
|
-
}
|
|
1554
|
-
}
|
|
1771
|
+
failedAgent.resume(context).catch((err) => {
|
|
1772
|
+
console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
|
|
1773
|
+
});
|
|
1555
1774
|
|
|
1556
|
-
|
|
1775
|
+
this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
|
|
1557
1776
|
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1777
|
+
return {
|
|
1778
|
+
id: clusterId,
|
|
1779
|
+
state: cluster.state,
|
|
1780
|
+
resumeType: 'failure',
|
|
1781
|
+
resumedAgent: agentId,
|
|
1782
|
+
previousError: error,
|
|
1783
|
+
};
|
|
1784
|
+
}
|
|
1562
1785
|
|
|
1786
|
+
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
|
|
1787
|
+
this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
|
|
1788
|
+
|
|
1789
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
1790
|
+
header: 'Resuming cluster from previous session.',
|
|
1791
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
|
|
1792
|
+
});
|
|
1793
|
+
|
|
1794
|
+
const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
|
|
1563
1795
|
if (lastTrigger) {
|
|
1564
1796
|
this._log(
|
|
1565
1797
|
`[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
|
|
1566
1798
|
);
|
|
1567
|
-
|
|
1568
|
-
for (const agent of cluster.agents) {
|
|
1569
|
-
if (!agent.config.triggers) continue;
|
|
1570
|
-
|
|
1571
|
-
const matchingTrigger = agent.config.triggers.find((trigger) => {
|
|
1572
|
-
// Exact match
|
|
1573
|
-
if (trigger.topic === lastTrigger.topic) return true;
|
|
1574
|
-
// Wildcard match
|
|
1575
|
-
if (trigger.topic === '*') return true;
|
|
1576
|
-
// Prefix match (e.g., "VALIDATION_*")
|
|
1577
|
-
if (trigger.topic.endsWith('*')) {
|
|
1578
|
-
const prefix = trigger.topic.slice(0, -1);
|
|
1579
|
-
return lastTrigger.topic.startsWith(prefix);
|
|
1580
|
-
}
|
|
1581
|
-
return false;
|
|
1582
|
-
});
|
|
1583
|
-
|
|
1584
|
-
if (matchingTrigger) {
|
|
1585
|
-
// Evaluate logic script if present
|
|
1586
|
-
if (matchingTrigger.logic?.script) {
|
|
1587
|
-
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
1588
|
-
if (!shouldTrigger) continue;
|
|
1589
|
-
}
|
|
1590
|
-
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
1799
|
} else {
|
|
1594
1800
|
this._log(`[Orchestrator] No workflow triggers found in ledger`);
|
|
1595
1801
|
}
|
|
1596
1802
|
|
|
1803
|
+
const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
|
|
1597
1804
|
if (agentsToResume.length === 0) {
|
|
1598
1805
|
if (!lastTrigger) {
|
|
1599
|
-
// No workflow activity - cluster never really started
|
|
1600
1806
|
this._log(
|
|
1601
1807
|
`[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
|
|
1602
1808
|
);
|
|
1603
1809
|
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
1604
1810
|
|
|
1605
|
-
|
|
1606
|
-
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
1607
|
-
if (issueMessage) {
|
|
1608
|
-
cluster.messageBus.publish({
|
|
1609
|
-
cluster_id: clusterId,
|
|
1610
|
-
topic: 'ISSUE_OPENED',
|
|
1611
|
-
sender: 'system',
|
|
1612
|
-
receiver: 'broadcast',
|
|
1613
|
-
content: issueMessage.content,
|
|
1614
|
-
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
1615
|
-
});
|
|
1616
|
-
} else {
|
|
1811
|
+
if (!this._republishIssue(cluster, clusterId, recentMessages)) {
|
|
1617
1812
|
throw new Error(
|
|
1618
1813
|
`Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
|
|
1619
1814
|
`The cluster may not have started properly. Try: zeroshot run <issue> instead.`
|
|
1620
1815
|
);
|
|
1621
1816
|
}
|
|
1622
1817
|
} else {
|
|
1623
|
-
// Had a trigger but no agents matched - something is wrong with agent configs
|
|
1624
1818
|
throw new Error(
|
|
1625
1819
|
`Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
|
|
1626
1820
|
`Check agent trigger configurations.`
|
|
1627
1821
|
);
|
|
1628
1822
|
}
|
|
1629
1823
|
} else {
|
|
1630
|
-
// Resume agents that should run based on ledger state
|
|
1631
1824
|
this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
|
|
1632
|
-
|
|
1633
|
-
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
1634
|
-
agent.resume(context).catch((err) => {
|
|
1635
|
-
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
1636
|
-
});
|
|
1637
|
-
}
|
|
1825
|
+
this._resumeAgents(agentsToResume, context);
|
|
1638
1826
|
}
|
|
1639
1827
|
|
|
1640
|
-
// Save updated state
|
|
1641
1828
|
await this._saveClusters();
|
|
1642
|
-
|
|
1643
1829
|
this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
|
|
1644
1830
|
|
|
1645
1831
|
return {
|
|
@@ -1758,7 +1944,63 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1758
1944
|
this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
|
|
1759
1945
|
|
|
1760
1946
|
// Phase 1: Pre-validate operation structure
|
|
1947
|
+
const validationErrors = this._validateOperationChain(operations);
|
|
1948
|
+
|
|
1949
|
+
if (validationErrors.length > 0) {
|
|
1950
|
+
const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
|
|
1951
|
+
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
1952
|
+
throw new Error(errorMsg);
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
// Phase 2: Build mock cluster config with proposed agents
|
|
1956
|
+
// Collect all agents that would exist after operations complete
|
|
1957
|
+
const existingAgentConfigs = cluster.config.agents || [];
|
|
1958
|
+
const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
|
|
1959
|
+
|
|
1960
|
+
// Phase 3: Validate proposed cluster config
|
|
1961
|
+
const validation = this._validateProposedConfig(
|
|
1962
|
+
clusterId,
|
|
1963
|
+
cluster,
|
|
1964
|
+
proposedAgentConfigs,
|
|
1965
|
+
operations
|
|
1966
|
+
);
|
|
1967
|
+
|
|
1968
|
+
// Log warnings but proceed
|
|
1969
|
+
if (validation.warnings.length > 0) {
|
|
1970
|
+
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
1971
|
+
for (const warning of validation.warnings) {
|
|
1972
|
+
this._log(` - ${warning}`);
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
// Phase 4: Execute validated operations
|
|
1977
|
+
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
1978
|
+
|
|
1979
|
+
await this._executeOperations(cluster, operations, sender, context);
|
|
1980
|
+
|
|
1981
|
+
this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
|
|
1982
|
+
|
|
1983
|
+
// Publish success notification
|
|
1984
|
+
cluster.messageBus.publish({
|
|
1985
|
+
cluster_id: clusterId,
|
|
1986
|
+
topic: 'CLUSTER_OPERATIONS_SUCCESS',
|
|
1987
|
+
sender: 'orchestrator',
|
|
1988
|
+
content: {
|
|
1989
|
+
text: `Executed ${operations.length} operation(s)`,
|
|
1990
|
+
data: {
|
|
1991
|
+
operationCount: operations.length,
|
|
1992
|
+
agentCount: cluster.agents.length,
|
|
1993
|
+
},
|
|
1994
|
+
},
|
|
1995
|
+
});
|
|
1996
|
+
|
|
1997
|
+
// Save updated cluster state to disk
|
|
1998
|
+
await this._saveClusters();
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
_validateOperationChain(operations) {
|
|
1761
2002
|
const validationErrors = [];
|
|
2003
|
+
|
|
1762
2004
|
for (let i = 0; i < operations.length; i++) {
|
|
1763
2005
|
const op = operations[i];
|
|
1764
2006
|
if (!op.action) {
|
|
@@ -1772,21 +2014,15 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1772
2014
|
}
|
|
1773
2015
|
}
|
|
1774
2016
|
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
1778
|
-
throw new Error(errorMsg);
|
|
1779
|
-
}
|
|
2017
|
+
return validationErrors;
|
|
2018
|
+
}
|
|
1780
2019
|
|
|
1781
|
-
|
|
1782
|
-
// Collect all agents that would exist after operations complete
|
|
1783
|
-
const existingAgentConfigs = cluster.config.agents || [];
|
|
2020
|
+
_buildProposedAgentConfigs(existingAgentConfigs, operations) {
|
|
1784
2021
|
const proposedAgentConfigs = [...existingAgentConfigs];
|
|
1785
2022
|
|
|
1786
2023
|
for (const op of operations) {
|
|
1787
2024
|
if (op.action === 'add_agents' && op.agents) {
|
|
1788
2025
|
for (const agentConfig of op.agents) {
|
|
1789
|
-
// Check for duplicate before adding
|
|
1790
2026
|
const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
|
|
1791
2027
|
if (existingIdx === -1) {
|
|
1792
2028
|
proposedAgentConfigs.push(agentConfig);
|
|
@@ -1807,7 +2043,10 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1807
2043
|
}
|
|
1808
2044
|
}
|
|
1809
2045
|
|
|
1810
|
-
|
|
2046
|
+
return proposedAgentConfigs;
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
_validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
|
|
1811
2050
|
const mockConfig = { agents: proposedAgentConfigs };
|
|
1812
2051
|
const validation = configValidator.validateConfig(mockConfig);
|
|
1813
2052
|
|
|
@@ -1833,62 +2072,35 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
1833
2072
|
throw new Error(errorMsg);
|
|
1834
2073
|
}
|
|
1835
2074
|
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
1839
|
-
for (const warning of validation.warnings) {
|
|
1840
|
-
this._log(` - ${warning}`);
|
|
1841
|
-
}
|
|
1842
|
-
}
|
|
1843
|
-
|
|
1844
|
-
// Phase 4: Execute validated operations
|
|
1845
|
-
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
2075
|
+
return validation;
|
|
2076
|
+
}
|
|
1846
2077
|
|
|
2078
|
+
async _executeOperations(cluster, operations, sender, context) {
|
|
1847
2079
|
for (let i = 0; i < operations.length; i++) {
|
|
1848
2080
|
const op = operations[i];
|
|
1849
2081
|
this._log(` [${i + 1}/${operations.length}] ${op.action}`);
|
|
1850
|
-
|
|
1851
|
-
switch (op.action) {
|
|
1852
|
-
case 'add_agents':
|
|
1853
|
-
await this._opAddAgents(cluster, op, context);
|
|
1854
|
-
break;
|
|
1855
|
-
|
|
1856
|
-
case 'remove_agents':
|
|
1857
|
-
await this._opRemoveAgents(cluster, op);
|
|
1858
|
-
break;
|
|
1859
|
-
|
|
1860
|
-
case 'update_agent':
|
|
1861
|
-
this._opUpdateAgent(cluster, op);
|
|
1862
|
-
break;
|
|
1863
|
-
|
|
1864
|
-
case 'publish':
|
|
1865
|
-
this._opPublish(cluster, op, sender);
|
|
1866
|
-
break;
|
|
1867
|
-
|
|
1868
|
-
case 'load_config':
|
|
1869
|
-
await this._opLoadConfig(cluster, op, context);
|
|
1870
|
-
break;
|
|
1871
|
-
}
|
|
2082
|
+
await this._executeOperation(cluster, op, sender, context);
|
|
1872
2083
|
}
|
|
2084
|
+
}
|
|
1873
2085
|
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
2086
|
+
async _executeOperation(cluster, op, sender, context) {
|
|
2087
|
+
switch (op.action) {
|
|
2088
|
+
case 'add_agents':
|
|
2089
|
+
await this._opAddAgents(cluster, op, context);
|
|
2090
|
+
break;
|
|
2091
|
+
case 'remove_agents':
|
|
2092
|
+
await this._opRemoveAgents(cluster, op);
|
|
2093
|
+
break;
|
|
2094
|
+
case 'update_agent':
|
|
2095
|
+
this._opUpdateAgent(cluster, op);
|
|
2096
|
+
break;
|
|
2097
|
+
case 'publish':
|
|
2098
|
+
this._opPublish(cluster, op, sender);
|
|
2099
|
+
break;
|
|
2100
|
+
case 'load_config':
|
|
2101
|
+
await this._opLoadConfig(cluster, op, context);
|
|
2102
|
+
break;
|
|
2103
|
+
}
|
|
1892
2104
|
}
|
|
1893
2105
|
|
|
1894
2106
|
/**
|
|
@@ -2137,22 +2349,45 @@ Continue from where you left off. Review your previous output to understand what
|
|
|
2137
2349
|
const isPrMode = cluster.autoPr || process.env.ZEROSHOT_PR === '1';
|
|
2138
2350
|
|
|
2139
2351
|
if (isPrMode) {
|
|
2140
|
-
//
|
|
2141
|
-
|
|
2142
|
-
|
|
2352
|
+
// Detect platform from stored cluster metadata OR git context
|
|
2353
|
+
let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
|
|
2354
|
+
|
|
2355
|
+
// Fallback to git context detection if cluster metadata missing
|
|
2356
|
+
if (!platform) {
|
|
2357
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
2358
|
+
try {
|
|
2359
|
+
platform = getPlatformForPR(cluster.cwd || process.cwd());
|
|
2360
|
+
} catch (error) {
|
|
2361
|
+
throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
|
|
2362
|
+
}
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
// Generate platform-specific git-pusher agent from template
|
|
2366
|
+
const {
|
|
2367
|
+
generateGitPusherAgent,
|
|
2368
|
+
isPlatformSupported,
|
|
2369
|
+
} = require('./agents/git-pusher-template');
|
|
2370
|
+
|
|
2371
|
+
if (!isPlatformSupported(platform)) {
|
|
2372
|
+
throw new Error(
|
|
2373
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
2374
|
+
);
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
const gitPusherConfig = generateGitPusherAgent(platform);
|
|
2143
2378
|
|
|
2144
2379
|
// Get issue context from ledger
|
|
2145
2380
|
const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
|
|
2146
2381
|
const issueNumber = issueMsg?.content?.data?.number || 'unknown';
|
|
2147
2382
|
const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
|
|
2148
2383
|
|
|
2149
|
-
// Inject
|
|
2384
|
+
// Inject issue context into prompt
|
|
2150
2385
|
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
2151
2386
|
.replace(/\{\{issue_number\}\}/g, issueNumber)
|
|
2152
2387
|
.replace(/\{\{issue_title\}\}/g, issueTitle);
|
|
2153
2388
|
|
|
2154
2389
|
await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
|
|
2155
|
-
this._log(` [--pr mode] Injected git-pusher agent`);
|
|
2390
|
+
this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
|
|
2156
2391
|
} else {
|
|
2157
2392
|
// Default completion-detector
|
|
2158
2393
|
const completionDetector = {
|
|
@@ -2363,103 +2598,164 @@ return true;`,
|
|
|
2363
2598
|
md += `## Task\n\n${taskText}\n\n`;
|
|
2364
2599
|
|
|
2365
2600
|
// Group messages by agent for cleaner output
|
|
2601
|
+
const agentOutputs = this._collectAgentOutputs(messages);
|
|
2602
|
+
|
|
2603
|
+
for (const [agentId, agentMsgs] of agentOutputs) {
|
|
2604
|
+
md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
md += this._renderValidationMarkdown(messages);
|
|
2608
|
+
|
|
2609
|
+
// Final status
|
|
2610
|
+
const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
|
|
2611
|
+
if (clusterComplete) {
|
|
2612
|
+
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
2613
|
+
}
|
|
2614
|
+
|
|
2615
|
+
return md;
|
|
2616
|
+
}
|
|
2617
|
+
|
|
2618
|
+
_collectAgentOutputs(messages) {
|
|
2366
2619
|
const agentOutputs = new Map();
|
|
2367
2620
|
|
|
2368
2621
|
for (const msg of messages) {
|
|
2369
|
-
if (msg.topic
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
}
|
|
2373
|
-
agentOutputs.get(msg.sender).push(msg);
|
|
2622
|
+
if (msg.topic !== 'AGENT_OUTPUT') continue;
|
|
2623
|
+
if (!agentOutputs.has(msg.sender)) {
|
|
2624
|
+
agentOutputs.set(msg.sender, []);
|
|
2374
2625
|
}
|
|
2626
|
+
agentOutputs.get(msg.sender).push(msg);
|
|
2375
2627
|
}
|
|
2376
2628
|
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
md += `## Agent: ${agentId}\n\n`;
|
|
2629
|
+
return agentOutputs;
|
|
2630
|
+
}
|
|
2380
2631
|
|
|
2381
|
-
|
|
2382
|
-
|
|
2632
|
+
_extractAgentOutput(agentMsgs, parseProviderChunk) {
|
|
2633
|
+
let text = '';
|
|
2634
|
+
const tools = [];
|
|
2383
2635
|
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2636
|
+
for (const msg of agentMsgs) {
|
|
2637
|
+
const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
|
|
2638
|
+
if (!content) continue;
|
|
2387
2639
|
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
}
|
|
2640
|
+
const provider = normalizeProviderName(
|
|
2641
|
+
msg.content?.data?.provider || msg.sender_provider || 'claude'
|
|
2642
|
+
);
|
|
2643
|
+
const events = parseProviderChunk(provider, content);
|
|
2644
|
+
for (const event of events) {
|
|
2645
|
+
switch (event.type) {
|
|
2646
|
+
case 'text':
|
|
2647
|
+
text += event.text;
|
|
2648
|
+
break;
|
|
2649
|
+
case 'tool_call':
|
|
2650
|
+
tools.push({ name: event.toolName, input: event.input });
|
|
2651
|
+
break;
|
|
2652
|
+
case 'tool_result':
|
|
2653
|
+
if (tools.length > 0) {
|
|
2654
|
+
const lastTool = tools[tools.length - 1];
|
|
2655
|
+
lastTool.result = event.content;
|
|
2656
|
+
lastTool.isError = event.isError;
|
|
2657
|
+
}
|
|
2658
|
+
break;
|
|
2408
2659
|
}
|
|
2409
2660
|
}
|
|
2661
|
+
}
|
|
2662
|
+
|
|
2663
|
+
return { text, tools };
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
_renderToolMarkdown(tools) {
|
|
2667
|
+
let md = `### Tools Used\n\n`;
|
|
2410
2668
|
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2669
|
+
for (const tool of tools) {
|
|
2670
|
+
const status = tool.isError ? '❌' : '✓';
|
|
2671
|
+
md += `- **${tool.name}** ${status}\n`;
|
|
2672
|
+
if (tool.input) {
|
|
2673
|
+
const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
|
|
2674
|
+
if (inputStr.length < 100) {
|
|
2675
|
+
md += ` - Input: \`${inputStr}\`\n`;
|
|
2676
|
+
}
|
|
2414
2677
|
}
|
|
2678
|
+
}
|
|
2415
2679
|
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2680
|
+
return `${md}\n`;
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
_renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
|
|
2684
|
+
let md = `## Agent: ${agentId}\n\n`;
|
|
2685
|
+
const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
|
|
2686
|
+
|
|
2687
|
+
if (text.trim()) {
|
|
2688
|
+
md += `### Output\n\n${text.trim()}\n\n`;
|
|
2689
|
+
}
|
|
2690
|
+
|
|
2691
|
+
if (tools.length > 0) {
|
|
2692
|
+
md += this._renderToolMarkdown(tools);
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
return md;
|
|
2696
|
+
}
|
|
2697
|
+
|
|
2698
|
+
_renderCriteriaMarkdown(criteriaResults) {
|
|
2699
|
+
if (!Array.isArray(criteriaResults)) {
|
|
2700
|
+
return '';
|
|
2701
|
+
}
|
|
2702
|
+
|
|
2703
|
+
let md = '';
|
|
2704
|
+
const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
|
|
2705
|
+
if (cannotValidateYet.length > 0) {
|
|
2706
|
+
md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
|
|
2707
|
+
for (const cv of cannotValidateYet) {
|
|
2708
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2709
|
+
}
|
|
2710
|
+
md += '\n';
|
|
2711
|
+
}
|
|
2712
|
+
|
|
2713
|
+
const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
|
|
2714
|
+
if (cannotValidate.length > 0) {
|
|
2715
|
+
md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
|
|
2716
|
+
for (const cv of cannotValidate) {
|
|
2717
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
2718
|
+
}
|
|
2719
|
+
md += '\n';
|
|
2720
|
+
}
|
|
2721
|
+
|
|
2722
|
+
return md;
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
_renderValidationEntryMarkdown(validation) {
|
|
2726
|
+
const data = validation.content?.data || {};
|
|
2727
|
+
const approved = data.approved === true || data.approved === 'true';
|
|
2728
|
+
const icon = approved ? '✅' : '❌';
|
|
2729
|
+
let md = `### ${validation.sender} ${icon}\n\n`;
|
|
2730
|
+
|
|
2731
|
+
if (data.summary) {
|
|
2732
|
+
md += `${data.summary}\n\n`;
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
if (!approved && data.issues) {
|
|
2736
|
+
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
2737
|
+
if (Array.isArray(issues) && issues.length > 0) {
|
|
2738
|
+
md += `**Issues:**\n`;
|
|
2739
|
+
for (const issue of issues) {
|
|
2740
|
+
md += `- ${issue}\n`;
|
|
2429
2741
|
}
|
|
2430
2742
|
md += '\n';
|
|
2431
2743
|
}
|
|
2432
2744
|
}
|
|
2433
2745
|
|
|
2434
|
-
|
|
2746
|
+
md += this._renderCriteriaMarkdown(data.criteriaResults);
|
|
2747
|
+
return md;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
_renderValidationMarkdown(messages) {
|
|
2435
2751
|
const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
|
|
2436
|
-
if (validations.length
|
|
2437
|
-
|
|
2438
|
-
for (const v of validations) {
|
|
2439
|
-
const data = v.content?.data || {};
|
|
2440
|
-
const approved = data.approved === true || data.approved === 'true';
|
|
2441
|
-
const icon = approved ? '✅' : '❌';
|
|
2442
|
-
md += `### ${v.sender} ${icon}\n\n`;
|
|
2443
|
-
if (data.summary) {
|
|
2444
|
-
md += `${data.summary}\n\n`;
|
|
2445
|
-
}
|
|
2446
|
-
if (!approved && data.issues) {
|
|
2447
|
-
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
2448
|
-
if (Array.isArray(issues) && issues.length > 0) {
|
|
2449
|
-
md += `**Issues:**\n`;
|
|
2450
|
-
for (const issue of issues) {
|
|
2451
|
-
md += `- ${issue}\n`;
|
|
2452
|
-
}
|
|
2453
|
-
md += '\n';
|
|
2454
|
-
}
|
|
2455
|
-
}
|
|
2456
|
-
}
|
|
2752
|
+
if (validations.length === 0) {
|
|
2753
|
+
return '';
|
|
2457
2754
|
}
|
|
2458
2755
|
|
|
2459
|
-
|
|
2460
|
-
const
|
|
2461
|
-
|
|
2462
|
-
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
2756
|
+
let md = `## Validation Results\n\n`;
|
|
2757
|
+
for (const validation of validations) {
|
|
2758
|
+
md += this._renderValidationEntryMarkdown(validation);
|
|
2463
2759
|
}
|
|
2464
2760
|
|
|
2465
2761
|
return md;
|