@the-open-engine/zeroshot 5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +539 -0
- package/LICENSE +21 -0
- package/README.md +508 -0
- package/cli/commands/inspect-render.js +135 -0
- package/cli/commands/inspect.js +294 -0
- package/cli/commands/providers.js +149 -0
- package/cli/index.js +5431 -0
- package/cli/lib/first-run.js +211 -0
- package/cli/lib/update-checker.js +281 -0
- package/cli/message-formatter-utils.js +75 -0
- package/cli/message-formatters-normal.js +275 -0
- package/cli/message-formatters-watch.js +185 -0
- package/cluster-templates/base-templates/debug-workflow.json +422 -0
- package/cluster-templates/base-templates/full-workflow.json +727 -0
- package/cluster-templates/base-templates/heavy-validation.json +272 -0
- package/cluster-templates/base-templates/quick-validation.json +285 -0
- package/cluster-templates/base-templates/single-worker.json +71 -0
- package/cluster-templates/base-templates/worker-validator.json +230 -0
- package/cluster-templates/conductor-bootstrap.json +130 -0
- package/docker/zeroshot-cluster/Dockerfile +153 -0
- package/docker/zeroshot-cluster/pre-baked-deps.json +28 -0
- package/lib/agent-cli-provider/adapters/claude-parser.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/claude-parser.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/claude-parser.js +122 -0
- package/lib/agent-cli-provider/adapters/claude-parser.js.map +1 -0
- package/lib/agent-cli-provider/adapters/claude-recovery.d.ts +15 -0
- package/lib/agent-cli-provider/adapters/claude-recovery.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/claude-recovery.js +165 -0
- package/lib/agent-cli-provider/adapters/claude-recovery.js.map +1 -0
- package/lib/agent-cli-provider/adapters/claude.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/claude.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/claude.js +181 -0
- package/lib/agent-cli-provider/adapters/claude.js.map +1 -0
- package/lib/agent-cli-provider/adapters/codex-parser.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/codex-parser.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/codex-parser.js +192 -0
- package/lib/agent-cli-provider/adapters/codex-parser.js.map +1 -0
- package/lib/agent-cli-provider/adapters/codex.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/codex.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/codex.js +151 -0
- package/lib/agent-cli-provider/adapters/codex.js.map +1 -0
- package/lib/agent-cli-provider/adapters/common.d.ts +28 -0
- package/lib/agent-cli-provider/adapters/common.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/common.js +71 -0
- package/lib/agent-cli-provider/adapters/common.js.map +1 -0
- package/lib/agent-cli-provider/adapters/gemini.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/gemini.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/gemini.js +176 -0
- package/lib/agent-cli-provider/adapters/gemini.js.map +1 -0
- package/lib/agent-cli-provider/adapters/index.d.ts +9 -0
- package/lib/agent-cli-provider/adapters/index.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/index.js +95 -0
- package/lib/agent-cli-provider/adapters/index.js.map +1 -0
- package/lib/agent-cli-provider/adapters/opencode.d.ts +3 -0
- package/lib/agent-cli-provider/adapters/opencode.d.ts.map +1 -0
- package/lib/agent-cli-provider/adapters/opencode.js +228 -0
- package/lib/agent-cli-provider/adapters/opencode.js.map +1 -0
- package/lib/agent-cli-provider/contract-actions.d.ts +5 -0
- package/lib/agent-cli-provider/contract-actions.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-actions.js +140 -0
- package/lib/agent-cli-provider/contract-actions.js.map +1 -0
- package/lib/agent-cli-provider/contract-env.d.ts +11 -0
- package/lib/agent-cli-provider/contract-env.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-env.js +113 -0
- package/lib/agent-cli-provider/contract-env.js.map +1 -0
- package/lib/agent-cli-provider/contract-envelope.d.ts +49 -0
- package/lib/agent-cli-provider/contract-envelope.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-envelope.js +52 -0
- package/lib/agent-cli-provider/contract-envelope.js.map +1 -0
- package/lib/agent-cli-provider/contract-errors.d.ts +22 -0
- package/lib/agent-cli-provider/contract-errors.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-errors.js +71 -0
- package/lib/agent-cli-provider/contract-errors.js.map +1 -0
- package/lib/agent-cli-provider/contract-fallback.d.ts +7 -0
- package/lib/agent-cli-provider/contract-fallback.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-fallback.js +102 -0
- package/lib/agent-cli-provider/contract-fallback.js.map +1 -0
- package/lib/agent-cli-provider/contract-invoke.d.ts +5 -0
- package/lib/agent-cli-provider/contract-invoke.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-invoke.js +89 -0
- package/lib/agent-cli-provider/contract-invoke.js.map +1 -0
- package/lib/agent-cli-provider/contract-options.d.ts +3 -0
- package/lib/agent-cli-provider/contract-options.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-options.js +140 -0
- package/lib/agent-cli-provider/contract-options.js.map +1 -0
- package/lib/agent-cli-provider/contract-parse.d.ts +15 -0
- package/lib/agent-cli-provider/contract-parse.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-parse.js +81 -0
- package/lib/agent-cli-provider/contract-parse.js.map +1 -0
- package/lib/agent-cli-provider/contract-support.d.ts +18 -0
- package/lib/agent-cli-provider/contract-support.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract-support.js +154 -0
- package/lib/agent-cli-provider/contract-support.js.map +1 -0
- package/lib/agent-cli-provider/contract.d.ts +13 -0
- package/lib/agent-cli-provider/contract.d.ts.map +1 -0
- package/lib/agent-cli-provider/contract.js +30 -0
- package/lib/agent-cli-provider/contract.js.map +1 -0
- package/lib/agent-cli-provider/env-safety.d.ts +11 -0
- package/lib/agent-cli-provider/env-safety.d.ts.map +1 -0
- package/lib/agent-cli-provider/env-safety.js +83 -0
- package/lib/agent-cli-provider/env-safety.js.map +1 -0
- package/lib/agent-cli-provider/errors.d.ts +5 -0
- package/lib/agent-cli-provider/errors.d.ts.map +1 -0
- package/lib/agent-cli-provider/errors.js +115 -0
- package/lib/agent-cli-provider/errors.js.map +1 -0
- package/lib/agent-cli-provider/executable.d.ts +3 -0
- package/lib/agent-cli-provider/executable.d.ts.map +1 -0
- package/lib/agent-cli-provider/executable.js +24 -0
- package/lib/agent-cli-provider/executable.js.map +1 -0
- package/lib/agent-cli-provider/index.d.ts +8 -0
- package/lib/agent-cli-provider/index.d.ts.map +1 -0
- package/lib/agent-cli-provider/index.js +31 -0
- package/lib/agent-cli-provider/index.js.map +1 -0
- package/lib/agent-cli-provider/invoke-evidence.d.ts +4 -0
- package/lib/agent-cli-provider/invoke-evidence.d.ts.map +1 -0
- package/lib/agent-cli-provider/invoke-evidence.js +17 -0
- package/lib/agent-cli-provider/invoke-evidence.js.map +1 -0
- package/lib/agent-cli-provider/json.d.ts +16 -0
- package/lib/agent-cli-provider/json.d.ts.map +1 -0
- package/lib/agent-cli-provider/json.js +110 -0
- package/lib/agent-cli-provider/json.js.map +1 -0
- package/lib/agent-cli-provider/log-prefix.d.ts +2 -0
- package/lib/agent-cli-provider/log-prefix.d.ts.map +1 -0
- package/lib/agent-cli-provider/log-prefix.js +22 -0
- package/lib/agent-cli-provider/log-prefix.js.map +1 -0
- package/lib/agent-cli-provider/process-runner.d.ts +17 -0
- package/lib/agent-cli-provider/process-runner.d.ts.map +1 -0
- package/lib/agent-cli-provider/process-runner.js +89 -0
- package/lib/agent-cli-provider/process-runner.js.map +1 -0
- package/lib/agent-cli-provider/redaction.d.ts +9 -0
- package/lib/agent-cli-provider/redaction.d.ts.map +1 -0
- package/lib/agent-cli-provider/redaction.js +227 -0
- package/lib/agent-cli-provider/redaction.js.map +1 -0
- package/lib/agent-cli-provider/schema.d.ts +5 -0
- package/lib/agent-cli-provider/schema.d.ts.map +1 -0
- package/lib/agent-cli-provider/schema.js +136 -0
- package/lib/agent-cli-provider/schema.js.map +1 -0
- package/lib/agent-cli-provider/single-agent-runtime.d.ts +15 -0
- package/lib/agent-cli-provider/single-agent-runtime.d.ts.map +1 -0
- package/lib/agent-cli-provider/single-agent-runtime.js +228 -0
- package/lib/agent-cli-provider/single-agent-runtime.js.map +1 -0
- package/lib/agent-cli-provider/types.d.ts +194 -0
- package/lib/agent-cli-provider/types.d.ts.map +1 -0
- package/lib/agent-cli-provider/types.js +12 -0
- package/lib/agent-cli-provider/types.js.map +1 -0
- package/lib/completion.js +174 -0
- package/lib/detached-startup.js +220 -0
- package/lib/docker-config.js +220 -0
- package/lib/git-remote-utils.js +165 -0
- package/lib/id-detector.js +55 -0
- package/lib/provider-defaults.js +62 -0
- package/lib/provider-detection.js +59 -0
- package/lib/provider-names.js +57 -0
- package/lib/repo-settings.js +69 -0
- package/lib/settings/claude-auth.js +78 -0
- package/lib/settings.js +542 -0
- package/lib/start-cluster.js +321 -0
- package/lib/stream-json-parser.js +67 -0
- package/package.json +162 -0
- package/scripts/fix-node-pty-permissions.js +75 -0
- package/scripts/record-demo.sh +279 -0
- package/scripts/setup-merge-queue.sh +170 -0
- package/scripts/test-install.sh +40 -0
- package/scripts/validate-templates.js +107 -0
- package/src/agent/agent-config.js +266 -0
- package/src/agent/agent-context-builder.js +189 -0
- package/src/agent/agent-context-sections.js +338 -0
- package/src/agent/agent-context-sources.js +147 -0
- package/src/agent/agent-hook-executor.js +721 -0
- package/src/agent/agent-input-injector.js +141 -0
- package/src/agent/agent-lifecycle.js +982 -0
- package/src/agent/agent-quality-gate-schema.js +93 -0
- package/src/agent/agent-quality-gates-context.js +51 -0
- package/src/agent/agent-stuck-detector.js +256 -0
- package/src/agent/agent-task-executor.js +2028 -0
- package/src/agent/agent-trigger-evaluator.js +67 -0
- package/src/agent/context-metrics.js +160 -0
- package/src/agent/context-pack-builder.js +367 -0
- package/src/agent/context-replay-policy.js +51 -0
- package/src/agent/guidance-queue.js +77 -0
- package/src/agent/output-extraction.js +367 -0
- package/src/agent/output-reformatter.js +175 -0
- package/src/agent/pr-verification.js +653 -0
- package/src/agent/rate-limit-backoff.js +82 -0
- package/src/agent/schema-utils.js +146 -0
- package/src/agent/validation-platform.js +35 -0
- package/src/agent-cli-provider/adapters/claude-parser.ts +133 -0
- package/src/agent-cli-provider/adapters/claude-recovery.ts +203 -0
- package/src/agent-cli-provider/adapters/claude.ts +247 -0
- package/src/agent-cli-provider/adapters/codex-parser.ts +211 -0
- package/src/agent-cli-provider/adapters/codex.ts +217 -0
- package/src/agent-cli-provider/adapters/common.ts +124 -0
- package/src/agent-cli-provider/adapters/gemini.ts +243 -0
- package/src/agent-cli-provider/adapters/index.ts +126 -0
- package/src/agent-cli-provider/adapters/opencode.ts +286 -0
- package/src/agent-cli-provider/contract-actions.ts +150 -0
- package/src/agent-cli-provider/contract-env.ts +111 -0
- package/src/agent-cli-provider/contract-envelope.ts +110 -0
- package/src/agent-cli-provider/contract-errors.ts +66 -0
- package/src/agent-cli-provider/contract-fallback.ts +121 -0
- package/src/agent-cli-provider/contract-invoke.ts +104 -0
- package/src/agent-cli-provider/contract-options.ts +173 -0
- package/src/agent-cli-provider/contract-parse.ts +94 -0
- package/src/agent-cli-provider/contract-support.ts +167 -0
- package/src/agent-cli-provider/contract.ts +56 -0
- package/src/agent-cli-provider/env-safety.ts +82 -0
- package/src/agent-cli-provider/errors.ts +122 -0
- package/src/agent-cli-provider/executable.ts +24 -0
- package/src/agent-cli-provider/index.ts +83 -0
- package/src/agent-cli-provider/invoke-evidence.ts +18 -0
- package/src/agent-cli-provider/json.ts +114 -0
- package/src/agent-cli-provider/log-prefix.ts +20 -0
- package/src/agent-cli-provider/process-runner.ts +145 -0
- package/src/agent-cli-provider/redaction.ts +282 -0
- package/src/agent-cli-provider/schema.ts +115 -0
- package/src/agent-cli-provider/single-agent-runtime.ts +311 -0
- package/src/agent-cli-provider/types.ts +237 -0
- package/src/agent-wrapper.js +615 -0
- package/src/agents/git-pusher-template.js +705 -0
- package/src/attach/attach-client.js +438 -0
- package/src/attach/attach-server.js +543 -0
- package/src/attach/index.js +37 -0
- package/src/attach/protocol.js +220 -0
- package/src/attach/ring-buffer.js +121 -0
- package/src/attach/send-input.js +88 -0
- package/src/attach/socket-discovery.js +267 -0
- package/src/claude-task-runner.js +661 -0
- package/src/config-router.js +89 -0
- package/src/config-validator.js +2202 -0
- package/src/copy-worker.js +43 -0
- package/src/guidance-topics.js +10 -0
- package/src/input-helpers.js +65 -0
- package/src/isolation-manager.js +1734 -0
- package/src/issue-providers/README.md +305 -0
- package/src/issue-providers/azure-devops-provider.js +307 -0
- package/src/issue-providers/base-provider.js +232 -0
- package/src/issue-providers/github-provider.js +210 -0
- package/src/issue-providers/gitlab-provider.js +262 -0
- package/src/issue-providers/index.js +196 -0
- package/src/issue-providers/jira-provider.js +260 -0
- package/src/ledger.js +692 -0
- package/src/lib/gc.js +232 -0
- package/src/lib/safe-exec.js +88 -0
- package/src/logic-engine.js +201 -0
- package/src/message-buffer.js +81 -0
- package/src/message-bus-bridge.js +144 -0
- package/src/message-bus.js +256 -0
- package/src/name-generator.js +232 -0
- package/src/orchestrator.js +3924 -0
- package/src/preflight.js +712 -0
- package/src/process-metrics.js +608 -0
- package/src/providers/anthropic/index.js +3 -0
- package/src/providers/base-provider.js +355 -0
- package/src/providers/capabilities.js +60 -0
- package/src/providers/google/index.js +3 -0
- package/src/providers/index.js +293 -0
- package/src/providers/openai/index.js +3 -0
- package/src/providers/opencode/index.js +3 -0
- package/src/quality-gates.js +143 -0
- package/src/schemas/sub-cluster.js +208 -0
- package/src/state-snapshot.js +398 -0
- package/src/state-snapshotter.js +142 -0
- package/src/status-footer.js +1026 -0
- package/src/sub-cluster-wrapper.js +693 -0
- package/src/task-runner.js +30 -0
- package/src/template-resolver.js +425 -0
- package/src/template-validation/index.js +338 -0
- package/src/template-validation/simulate-consensus-gates.js +324 -0
- package/src/template-validation/simulate-random-topology.js +541 -0
- package/src/template-validation/simulate-two-stage-validation.js +270 -0
- package/src/worktree-claude-config.js +135 -0
- package/src/worktree-tooling-env.js +150 -0
- package/task-lib/attachable-watcher.js +381 -0
- package/task-lib/commands/clean.js +50 -0
- package/task-lib/commands/episodes.js +105 -0
- package/task-lib/commands/get-log-path.js +23 -0
- package/task-lib/commands/kill.js +32 -0
- package/task-lib/commands/list.js +105 -0
- package/task-lib/commands/logs.js +439 -0
- package/task-lib/commands/resume.js +42 -0
- package/task-lib/commands/run.js +57 -0
- package/task-lib/commands/schedule.js +105 -0
- package/task-lib/commands/scheduler-cmd.js +96 -0
- package/task-lib/commands/schedules.js +148 -0
- package/task-lib/commands/status.js +44 -0
- package/task-lib/commands/unschedule.js +16 -0
- package/task-lib/completion.js +9 -0
- package/task-lib/config.js +11 -0
- package/task-lib/name-generator.js +230 -0
- package/task-lib/package.json +3 -0
- package/task-lib/provider-helper-runtime.js +29 -0
- package/task-lib/runner.js +190 -0
- package/task-lib/scheduler.js +252 -0
- package/task-lib/store.js +529 -0
- package/task-lib/watcher.js +305 -0
|
@@ -0,0 +1,3924 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Orchestrator - Manages cluster lifecycle
|
|
3
|
+
*
|
|
4
|
+
* Provides:
|
|
5
|
+
* - Cluster initialization and configuration
|
|
6
|
+
* - Agent lifecycle management
|
|
7
|
+
* - GitHub issue integration
|
|
8
|
+
* - Cluster state tracking
|
|
9
|
+
* - Crash recovery
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const os = require('os');
|
|
15
|
+
const lockfile = require('proper-lockfile');
|
|
16
|
+
|
|
17
|
+
// Stale lock timeout in ms - if lock file is older than this, delete it
|
|
18
|
+
const LOCK_STALE_MS = 5000;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Remove lock file if it's stale (older than LOCK_STALE_MS)
|
|
22
|
+
* Handles crashes that leave orphaned lock files
|
|
23
|
+
*/
|
|
24
|
+
function cleanStaleLock(lockPath) {
|
|
25
|
+
try {
|
|
26
|
+
if (fs.existsSync(lockPath)) {
|
|
27
|
+
const age = Date.now() - fs.statSync(lockPath).mtimeMs;
|
|
28
|
+
if (age > LOCK_STALE_MS) {
|
|
29
|
+
fs.unlinkSync(lockPath);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
} catch {
|
|
33
|
+
// Ignore - another process may have cleaned it
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const AgentWrapper = require('./agent-wrapper');
|
|
37
|
+
const SubClusterWrapper = require('./sub-cluster-wrapper');
|
|
38
|
+
const MessageBus = require('./message-bus');
|
|
39
|
+
const Ledger = require('./ledger');
|
|
40
|
+
const InputHelpers = require('./input-helpers');
|
|
41
|
+
const { USER_GUIDANCE_AGENT, USER_GUIDANCE_CLUSTER } = require('./guidance-topics');
|
|
42
|
+
const { detectProvider } = require('./issue-providers');
|
|
43
|
+
const IsolationManager = require('./isolation-manager');
|
|
44
|
+
const { generateName } = require('./name-generator');
|
|
45
|
+
const configValidator = require('./config-validator');
|
|
46
|
+
const TemplateResolver = require('./template-resolver');
|
|
47
|
+
const { loadSettings } = require('../lib/settings');
|
|
48
|
+
const { normalizeProviderName } = require('../lib/provider-names');
|
|
49
|
+
const { getProvider } = require('./providers');
|
|
50
|
+
const StateSnapshotter = require('./state-snapshotter');
|
|
51
|
+
const { resolveClusterRequiredQualityGates } = require('./quality-gates');
|
|
52
|
+
const crypto = require('crypto');
|
|
53
|
+
|
|
54
|
+
function applyModelOverride(agentConfig, modelOverride) {
|
|
55
|
+
if (!modelOverride) return;
|
|
56
|
+
|
|
57
|
+
agentConfig.model = modelOverride;
|
|
58
|
+
if (agentConfig.modelRules) {
|
|
59
|
+
delete agentConfig.modelRules;
|
|
60
|
+
}
|
|
61
|
+
if (agentConfig.modelConfig) {
|
|
62
|
+
delete agentConfig.modelConfig;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Operation Chain Schema
|
|
68
|
+
* Conductor (or any agent) can publish CLUSTER_OPERATIONS to dynamically modify cluster
|
|
69
|
+
*
|
|
70
|
+
* Supported operations:
|
|
71
|
+
* - add_agents: Spawn new agents with given configs
|
|
72
|
+
* - remove_agents: Stop and remove agents by ID
|
|
73
|
+
* - update_agent: Modify existing agent config
|
|
74
|
+
* - publish: Publish a message to the bus
|
|
75
|
+
* - load_config: Load agents from a named cluster config template
|
|
76
|
+
*/
|
|
77
|
+
const VALID_OPERATIONS = ['add_agents', 'remove_agents', 'update_agent', 'publish', 'load_config'];
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Workflow-triggering topics that indicate cluster state progression
|
|
81
|
+
* These are the topics that MATTER for resume - not AGENT_OUTPUT noise
|
|
82
|
+
*/
|
|
83
|
+
const WORKFLOW_TRIGGERS = Object.freeze([
|
|
84
|
+
'ISSUE_OPENED',
|
|
85
|
+
'PLAN_READY',
|
|
86
|
+
'IMPLEMENTATION_READY',
|
|
87
|
+
'VALIDATION_RESULT',
|
|
88
|
+
'PUSH_BLOCKED',
|
|
89
|
+
'CONDUCTOR_ESCALATE',
|
|
90
|
+
]);
|
|
91
|
+
|
|
92
|
+
const PUSH_BLOCKED_REPAIR_TRIGGER = Object.freeze({
|
|
93
|
+
topic: 'PUSH_BLOCKED',
|
|
94
|
+
action: 'execute_task',
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
function applyRequiredQualityGatesToAgent(agentConfig, requiredQualityGates) {
|
|
98
|
+
if (!Array.isArray(requiredQualityGates) || requiredQualityGates.length === 0) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (agentConfig.role !== 'validator') {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (!Array.isArray(agentConfig.requiredQualityGates)) {
|
|
107
|
+
agentConfig.requiredQualityGates = requiredQualityGates;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function applyRequiredQualityGatesToValidators(config, requiredQualityGates) {
|
|
112
|
+
if (!Array.isArray(requiredQualityGates) || requiredQualityGates.length === 0) {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
for (const agentConfig of config.agents || []) {
|
|
117
|
+
applyRequiredQualityGatesToAgent(agentConfig, requiredQualityGates);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function getTriggerTopic(trigger) {
|
|
122
|
+
return typeof trigger === 'string' ? trigger : trigger?.topic;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function shouldHandlePushBlockedRepair(agentConfig) {
|
|
126
|
+
return agentConfig?.role === 'implementation';
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function applyPushBlockedRepairTrigger(agentConfig) {
|
|
130
|
+
if (!shouldHandlePushBlockedRepair(agentConfig)) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (!Array.isArray(agentConfig.triggers)) {
|
|
135
|
+
agentConfig.triggers = [];
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if (agentConfig.triggers.some((trigger) => getTriggerTopic(trigger) === 'PUSH_BLOCKED')) {
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
agentConfig.triggers.push({ ...PUSH_BLOCKED_REPAIR_TRIGGER });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function applyPushBlockedRepairTriggers(config) {
|
|
146
|
+
for (const agentConfig of config?.agents || []) {
|
|
147
|
+
applyPushBlockedRepairTrigger(agentConfig);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function buildPrOptions(options, requiredQualityGates) {
|
|
152
|
+
if (
|
|
153
|
+
!options.prBase &&
|
|
154
|
+
!options.mergeQueue &&
|
|
155
|
+
!options.closeIssue &&
|
|
156
|
+
requiredQualityGates.length === 0
|
|
157
|
+
) {
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
prBase: options.prBase || null,
|
|
163
|
+
mergeQueue: options.mergeQueue || false,
|
|
164
|
+
closeIssue: options.closeIssue || null,
|
|
165
|
+
...(requiredQualityGates.length > 0 ? { requiredQualityGates } : {}),
|
|
166
|
+
cwd: options.cwd || process.cwd(),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
class Orchestrator {
|
|
171
|
+
constructor(options = {}) {
|
|
172
|
+
this.clusters = new Map(); // cluster_id -> cluster object
|
|
173
|
+
this.quiet = options.quiet || false; // Suppress verbose logging
|
|
174
|
+
|
|
175
|
+
// TaskRunner DI - allows injecting MockTaskRunner for testing
|
|
176
|
+
// When set, passed to all AgentWrappers to control task execution
|
|
177
|
+
this.taskRunner = options.taskRunner || null;
|
|
178
|
+
|
|
179
|
+
// Set up persistent storage directory (can be overridden for testing)
|
|
180
|
+
this.storageDir = options.storageDir || path.join(os.homedir(), '.zeroshot');
|
|
181
|
+
if (!fs.existsSync(this.storageDir)) {
|
|
182
|
+
fs.mkdirSync(this.storageDir, { recursive: true });
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Track if orchestrator is closed (prevents _saveClusters race conditions during cleanup)
|
|
186
|
+
this.closed = false;
|
|
187
|
+
|
|
188
|
+
// Track if clusters are loaded (for lazy loading pattern)
|
|
189
|
+
this._clustersLoaded = options.skipLoad === true;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Factory method for async initialization
|
|
194
|
+
* Use this instead of `new Orchestrator()` for proper async cluster loading
|
|
195
|
+
* @param {Object} options - Same options as constructor
|
|
196
|
+
* @returns {Promise<Orchestrator>}
|
|
197
|
+
*/
|
|
198
|
+
static async create(options = {}) {
|
|
199
|
+
const instance = new Orchestrator({ ...options, skipLoad: true });
|
|
200
|
+
if (options.skipLoad !== true) {
|
|
201
|
+
await instance._loadClusters();
|
|
202
|
+
instance._clustersLoaded = true;
|
|
203
|
+
}
|
|
204
|
+
return instance;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Log message (respects quiet mode)
|
|
209
|
+
* @private
|
|
210
|
+
*/
|
|
211
|
+
_log(...args) {
|
|
212
|
+
if (!this.quiet) {
|
|
213
|
+
console.log(...args);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Resolve provider for a cluster config using the standard precedence.
|
|
219
|
+
* @param {Object} clusterConfig
|
|
220
|
+
* @param {Object} settings
|
|
221
|
+
* @returns {string}
|
|
222
|
+
* @private
|
|
223
|
+
*/
|
|
224
|
+
_resolveClusterProvider(clusterConfig = {}, settings = loadSettings()) {
|
|
225
|
+
const resolved =
|
|
226
|
+
clusterConfig.forceProvider ||
|
|
227
|
+
clusterConfig.defaultProvider ||
|
|
228
|
+
settings.defaultProvider ||
|
|
229
|
+
'claude';
|
|
230
|
+
return normalizeProviderName(resolved) || 'claude';
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Resolve the model level for internal completion agents.
|
|
235
|
+
* Uses provider minLevel when configured, otherwise provider default min level.
|
|
236
|
+
* @param {Object} clusterConfig
|
|
237
|
+
* @returns {string}
|
|
238
|
+
* @private
|
|
239
|
+
*/
|
|
240
|
+
_resolveCompletionDetectorLevel(clusterConfig = {}) {
|
|
241
|
+
const settings = loadSettings();
|
|
242
|
+
const providerName = this._resolveClusterProvider(clusterConfig, settings);
|
|
243
|
+
const provider = getProvider(providerName);
|
|
244
|
+
const providerSettings = settings.providerSettings?.[providerName] || {};
|
|
245
|
+
|
|
246
|
+
return providerSettings.minLevel || provider.getDefaultMinLevel();
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Get input source type for metadata
|
|
251
|
+
* @param {Object} input - Input object
|
|
252
|
+
* @returns {string} Source type: 'github', 'file', or 'text'
|
|
253
|
+
* @private
|
|
254
|
+
*/
|
|
255
|
+
_getInputSource(input) {
|
|
256
|
+
if (input.issue) return 'github';
|
|
257
|
+
if (input.file) return 'file';
|
|
258
|
+
return 'text';
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Load clusters from persistent storage
|
|
263
|
+
* Uses file locking for consistent reads
|
|
264
|
+
* @private
|
|
265
|
+
*/
|
|
266
|
+
async _loadClusters() {
|
|
267
|
+
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
268
|
+
this._log(`[Orchestrator] Loading clusters from: ${clustersFile}`);
|
|
269
|
+
|
|
270
|
+
if (!fs.existsSync(clustersFile)) {
|
|
271
|
+
this._log(`[Orchestrator] No clusters file found at ${clustersFile}`);
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
276
|
+
let release;
|
|
277
|
+
|
|
278
|
+
try {
|
|
279
|
+
// Clean stale locks from crashed processes
|
|
280
|
+
cleanStaleLock(lockfilePath);
|
|
281
|
+
|
|
282
|
+
// Acquire lock with async API (proper retries without CPU spin-wait)
|
|
283
|
+
release = await lockfile.lock(clustersFile, {
|
|
284
|
+
lockfilePath,
|
|
285
|
+
stale: LOCK_STALE_MS,
|
|
286
|
+
retries: {
|
|
287
|
+
retries: 20,
|
|
288
|
+
minTimeout: 100,
|
|
289
|
+
maxTimeout: 200,
|
|
290
|
+
randomize: true,
|
|
291
|
+
},
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
|
|
295
|
+
const clusterIds = Object.keys(data);
|
|
296
|
+
this._log(`[Orchestrator] Found ${clusterIds.length} clusters in file:`, clusterIds);
|
|
297
|
+
|
|
298
|
+
// Track clusters to remove (missing .db files or 0 messages)
|
|
299
|
+
const clustersToRemove = [];
|
|
300
|
+
// Track clusters with 0 messages (corrupted from SIGINT race condition)
|
|
301
|
+
const corruptedClusters = [];
|
|
302
|
+
|
|
303
|
+
for (const [clusterId, clusterData] of Object.entries(data)) {
|
|
304
|
+
if (clusterData?.state === 'setup' || clusterData?.provisional === true) {
|
|
305
|
+
this.clusters.set(clusterId, this._loadSetupCluster(clusterId, clusterData));
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Skip clusters whose .db file doesn't exist (orphaned registry entries)
|
|
310
|
+
const dbPath = path.join(this.storageDir, `${clusterId}.db`);
|
|
311
|
+
if (!fs.existsSync(dbPath)) {
|
|
312
|
+
console.warn(
|
|
313
|
+
`[Orchestrator] Cluster ${clusterId} has no database file, removing from registry`
|
|
314
|
+
);
|
|
315
|
+
clustersToRemove.push(clusterId);
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
this._log(`[Orchestrator] Loading cluster: ${clusterId}`);
|
|
320
|
+
let cluster;
|
|
321
|
+
try {
|
|
322
|
+
cluster = this._loadSingleCluster(clusterId, clusterData);
|
|
323
|
+
} catch (error) {
|
|
324
|
+
console.warn(
|
|
325
|
+
`[Orchestrator] Skipping cluster ${clusterId}: ${error.message || String(error)}`
|
|
326
|
+
);
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// VALIDATION: Detect 0-message clusters (corrupted from SIGINT during initialization)
|
|
331
|
+
// These clusters were created before the initCompletePromise fix was applied
|
|
332
|
+
if (cluster && cluster.messageBus) {
|
|
333
|
+
const messageCount = cluster.messageBus.count({ cluster_id: clusterId });
|
|
334
|
+
if (messageCount === 0) {
|
|
335
|
+
console.warn(`[Orchestrator] ⚠️ Cluster ${clusterId} has 0 messages (corrupted)`);
|
|
336
|
+
console.warn(
|
|
337
|
+
`[Orchestrator] This likely occurred from SIGINT during initialization.`
|
|
338
|
+
);
|
|
339
|
+
console.warn(
|
|
340
|
+
`[Orchestrator] Marking as 'corrupted' - use 'zeroshot kill ${clusterId}' to remove.`
|
|
341
|
+
);
|
|
342
|
+
corruptedClusters.push(clusterId);
|
|
343
|
+
// Mark cluster as corrupted for visibility in status/list commands
|
|
344
|
+
cluster.state = 'corrupted';
|
|
345
|
+
cluster.corruptedReason = 'SIGINT during initialization (0 messages in ledger)';
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Clean up orphaned entries from clusters.json
|
|
351
|
+
if (clustersToRemove.length > 0) {
|
|
352
|
+
for (const clusterId of clustersToRemove) {
|
|
353
|
+
delete data[clusterId];
|
|
354
|
+
}
|
|
355
|
+
fs.writeFileSync(clustersFile, JSON.stringify(data, null, 2));
|
|
356
|
+
this._log(
|
|
357
|
+
`[Orchestrator] Removed ${clustersToRemove.length} orphaned cluster(s) from registry`
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Log summary of corrupted clusters
|
|
362
|
+
if (corruptedClusters.length > 0) {
|
|
363
|
+
console.warn(
|
|
364
|
+
`\n[Orchestrator] ⚠️ Found ${corruptedClusters.length} corrupted cluster(s):`
|
|
365
|
+
);
|
|
366
|
+
for (const clusterId of corruptedClusters) {
|
|
367
|
+
console.warn(` - ${clusterId}`);
|
|
368
|
+
}
|
|
369
|
+
console.warn(`[Orchestrator] Run 'zeroshot clear' to remove all corrupted clusters.\n`);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
this._log(`[Orchestrator] Total clusters loaded: ${this.clusters.size}`);
|
|
373
|
+
} catch (error) {
|
|
374
|
+
console.error('[Orchestrator] Failed to load clusters:', error.message);
|
|
375
|
+
console.error(error.stack);
|
|
376
|
+
} finally {
|
|
377
|
+
if (release) {
|
|
378
|
+
await release();
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
_loadSetupCluster(clusterId, clusterData) {
|
|
384
|
+
const ledger = this._createSetupLedgerStub();
|
|
385
|
+
const messageBus = this._createSetupMessageBusStub(ledger);
|
|
386
|
+
|
|
387
|
+
return {
|
|
388
|
+
...clusterData,
|
|
389
|
+
id: clusterId,
|
|
390
|
+
config: clusterData.config || { agents: [] },
|
|
391
|
+
state: clusterData.state || 'setup',
|
|
392
|
+
createdAt: clusterData.createdAt || Date.now(),
|
|
393
|
+
pid: clusterData.pid || null,
|
|
394
|
+
agents: [],
|
|
395
|
+
messageBus,
|
|
396
|
+
ledger,
|
|
397
|
+
setupLogPath: clusterData.setupLogPath || null,
|
|
398
|
+
setupStage: clusterData.setupStage || null,
|
|
399
|
+
failureInfo: clusterData.failureInfo || null,
|
|
400
|
+
provisional: true,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
_createSetupLedgerStub() {
|
|
405
|
+
const emptyArray = () => [];
|
|
406
|
+
const noop = () => {};
|
|
407
|
+
|
|
408
|
+
return {
|
|
409
|
+
append: () => null,
|
|
410
|
+
batchAppend: () => [],
|
|
411
|
+
query: emptyArray,
|
|
412
|
+
queryGuidanceMailbox: emptyArray,
|
|
413
|
+
findLast: () => null,
|
|
414
|
+
count: () => 0,
|
|
415
|
+
since: emptyArray,
|
|
416
|
+
getAll: emptyArray,
|
|
417
|
+
getTokensByRole: () => ({
|
|
418
|
+
_total: { count: 0, inputTokens: 0, outputTokens: 0, totalCostUsd: 0 },
|
|
419
|
+
}),
|
|
420
|
+
pollForMessages: () => noop,
|
|
421
|
+
on: noop,
|
|
422
|
+
off: noop,
|
|
423
|
+
close: noop,
|
|
424
|
+
clear: noop,
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
_createSetupMessageBusStub(ledger) {
|
|
429
|
+
const noop = () => {};
|
|
430
|
+
const unsubscribe = () => {};
|
|
431
|
+
|
|
432
|
+
return {
|
|
433
|
+
ledger,
|
|
434
|
+
publish: () => null,
|
|
435
|
+
batchPublish: () => [],
|
|
436
|
+
subscribe: () => unsubscribe,
|
|
437
|
+
subscribeTopic: () => unsubscribe,
|
|
438
|
+
subscribeTopics: () => unsubscribe,
|
|
439
|
+
query: (criteria) => ledger.query(criteria),
|
|
440
|
+
queryGuidanceMailbox: (criteria) => ledger.queryGuidanceMailbox(criteria),
|
|
441
|
+
findLast: (criteria) => ledger.findLast(criteria),
|
|
442
|
+
count: (criteria) => ledger.count(criteria),
|
|
443
|
+
since: (params) => ledger.since(params),
|
|
444
|
+
getAll: (clusterId) => ledger.getAll(clusterId),
|
|
445
|
+
getTokensByRole: (clusterId) => ledger.getTokensByRole(clusterId),
|
|
446
|
+
addWebSocketClient: noop,
|
|
447
|
+
removeWebSocketClient: noop,
|
|
448
|
+
on: noop,
|
|
449
|
+
off: noop,
|
|
450
|
+
emit: () => false,
|
|
451
|
+
close: noop,
|
|
452
|
+
clear: noop,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
_isSetupCluster(cluster) {
|
|
457
|
+
return Boolean(cluster?.provisional === true || cluster?.state === 'setup');
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
_resolveStartClusterId(requestedClusterId, dbPath) {
|
|
461
|
+
if (!requestedClusterId) {
|
|
462
|
+
return this._generateUniqueClusterId(null, dbPath || null);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
const existingCluster = this.clusters.get(requestedClusterId);
|
|
466
|
+
const candidateDbPath = dbPath || path.join(this.storageDir, `${requestedClusterId}.db`);
|
|
467
|
+
if (this._isSetupCluster(existingCluster) && !fs.existsSync(candidateDbPath)) {
|
|
468
|
+
this.clusters.delete(requestedClusterId);
|
|
469
|
+
return requestedClusterId;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return this._generateUniqueClusterId(requestedClusterId, dbPath || null);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Load a single cluster from data
|
|
477
|
+
* @private
|
|
478
|
+
*/
|
|
479
|
+
_loadSingleCluster(clusterId, clusterData) {
|
|
480
|
+
// Skip if already loaded
|
|
481
|
+
if (this.clusters.has(clusterId)) {
|
|
482
|
+
return this.clusters.get(clusterId);
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// Restore ledger and message bus
|
|
486
|
+
const dbPath = path.join(this.storageDir, `${clusterId}.db`);
|
|
487
|
+
const ledger = new Ledger(dbPath);
|
|
488
|
+
const messageBus = new MessageBus(ledger);
|
|
489
|
+
|
|
490
|
+
// Restore isolation manager FIRST if cluster was running in isolation mode
|
|
491
|
+
const { isolation, isolationManager } = this._restoreClusterIsolation(clusterId, clusterData);
|
|
492
|
+
|
|
493
|
+
// Create mutable cluster context for reload path (used by AgentWrapper/SubClusterWrapper)
|
|
494
|
+
const clusterContext = {
|
|
495
|
+
...clusterData,
|
|
496
|
+
id: clusterId,
|
|
497
|
+
isolation,
|
|
498
|
+
};
|
|
499
|
+
|
|
500
|
+
let agents;
|
|
501
|
+
try {
|
|
502
|
+
// Reconstruct agent metadata from config (processes are ephemeral)
|
|
503
|
+
// CRITICAL: Pass isolation context to agents if cluster was running in isolation
|
|
504
|
+
agents = this._rebuildClusterAgents(clusterContext, messageBus, isolation, isolationManager);
|
|
505
|
+
} catch (error) {
|
|
506
|
+
try {
|
|
507
|
+
ledger.close();
|
|
508
|
+
} catch (closeError) {
|
|
509
|
+
console.warn(
|
|
510
|
+
`[Orchestrator] Failed to close ledger for ${clusterId}: ${closeError.message || String(closeError)}`
|
|
511
|
+
);
|
|
512
|
+
}
|
|
513
|
+
throw error;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const cluster = {
|
|
517
|
+
...clusterContext,
|
|
518
|
+
ledger,
|
|
519
|
+
messageBus,
|
|
520
|
+
agents,
|
|
521
|
+
isolation,
|
|
522
|
+
autoPr: clusterData.autoPr || false,
|
|
523
|
+
prOptions: clusterData.prOptions || null,
|
|
524
|
+
issue: clusterData.issue || null,
|
|
525
|
+
};
|
|
526
|
+
|
|
527
|
+
Object.assign(clusterContext, cluster);
|
|
528
|
+
|
|
529
|
+
this.clusters.set(clusterId, clusterContext);
|
|
530
|
+
this._startSnapshotter(clusterContext);
|
|
531
|
+
this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`);
|
|
532
|
+
|
|
533
|
+
return clusterContext;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
_restoreClusterIsolation(clusterId, clusterData) {
|
|
537
|
+
let isolation = clusterData.isolation || null;
|
|
538
|
+
let isolationManager = null;
|
|
539
|
+
|
|
540
|
+
if (isolation?.enabled && isolation.containerId) {
|
|
541
|
+
isolationManager = new IsolationManager({ image: isolation.image });
|
|
542
|
+
// Restore the container mapping so cleanup works
|
|
543
|
+
isolationManager.containers.set(clusterId, isolation.containerId);
|
|
544
|
+
// Restore isolated dir mapping for workspace preservation during cleanup
|
|
545
|
+
if (isolation.workDir) {
|
|
546
|
+
isolationManager.isolatedDirs.set(clusterId, {
|
|
547
|
+
path: path.join(os.tmpdir(), 'zeroshot-isolated', clusterId),
|
|
548
|
+
originalDir: isolation.workDir,
|
|
549
|
+
});
|
|
550
|
+
}
|
|
551
|
+
isolation = {
|
|
552
|
+
...isolation,
|
|
553
|
+
manager: isolationManager,
|
|
554
|
+
};
|
|
555
|
+
this._log(
|
|
556
|
+
`[Orchestrator] Restored isolation manager for ${clusterId} (container: ${isolation.containerId}, workDir: ${isolation.workDir || 'unknown'})`
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
return { isolation, isolationManager };
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
_resolveAgentCwd(clusterData) {
|
|
564
|
+
const worktreePath = clusterData.worktree?.path;
|
|
565
|
+
const isolationWorkDir = clusterData.isolation?.workDir;
|
|
566
|
+
return worktreePath || isolationWorkDir || null;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
_buildAgentOptions(clusterId, clusterData, isolation, isolationManager) {
|
|
570
|
+
const agentOptions = {
|
|
571
|
+
id: clusterId,
|
|
572
|
+
quiet: this.quiet,
|
|
573
|
+
modelOverride: clusterData.modelOverride || null,
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
if (isolation?.enabled && isolationManager) {
|
|
577
|
+
agentOptions.isolation = {
|
|
578
|
+
enabled: true,
|
|
579
|
+
manager: isolationManager,
|
|
580
|
+
clusterId,
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
return agentOptions;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
_instantiateAgent(agentConfig, messageBus, clusterContext, agentOptions) {
|
|
588
|
+
if (agentConfig.type === 'subcluster') {
|
|
589
|
+
return new SubClusterWrapper(agentConfig, messageBus, clusterContext, agentOptions);
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
return new AgentWrapper(agentConfig, messageBus, clusterContext, agentOptions);
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
_restoreAgentState(agent, agentConfig, clusterData) {
|
|
596
|
+
if (!clusterData.agentStates) return;
|
|
597
|
+
|
|
598
|
+
const savedState = clusterData.agentStates.find((state) => state.id === agentConfig.id);
|
|
599
|
+
if (!savedState) return;
|
|
600
|
+
|
|
601
|
+
agent.state = savedState.state || 'idle';
|
|
602
|
+
agent.iteration = savedState.iteration || 0;
|
|
603
|
+
// currentTask is a live in-process handle, not durable state.
|
|
604
|
+
// Restoring a serialized boolean here revives a fake runtime handle and
|
|
605
|
+
// makes stopped/resumed agents look like they still own a running task.
|
|
606
|
+
agent.currentTask = null;
|
|
607
|
+
agent.currentTaskId = savedState.currentTaskId || null;
|
|
608
|
+
agent.processPid = savedState.processPid || null;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
_rebuildClusterAgents(clusterContext, messageBus, isolation, isolationManager) {
|
|
612
|
+
const agents = [];
|
|
613
|
+
const clusterId = clusterContext.id;
|
|
614
|
+
const clusterData = clusterContext;
|
|
615
|
+
const agentCwd = this._resolveAgentCwd(clusterData);
|
|
616
|
+
|
|
617
|
+
if (!clusterData.config?.agents) {
|
|
618
|
+
return agents;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
for (const agentConfig of clusterData.config.agents) {
|
|
622
|
+
if (!agentConfig.cwd && agentCwd) {
|
|
623
|
+
agentConfig.cwd = agentCwd;
|
|
624
|
+
this._log(`[Orchestrator] Fixed missing cwd for agent ${agentConfig.id}: ${agentCwd}`);
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
if (clusterData.modelOverride) {
|
|
628
|
+
applyModelOverride(agentConfig, clusterData.modelOverride);
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
const agentOptions = this._buildAgentOptions(
|
|
632
|
+
clusterId,
|
|
633
|
+
clusterData,
|
|
634
|
+
isolation,
|
|
635
|
+
isolationManager
|
|
636
|
+
);
|
|
637
|
+
const agent = this._instantiateAgent(agentConfig, messageBus, clusterContext, agentOptions);
|
|
638
|
+
this._restoreAgentState(agent, agentConfig, clusterData);
|
|
639
|
+
agents.push(agent);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
return agents;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
_startSnapshotter(cluster) {
|
|
646
|
+
if (cluster.snapshotter) {
|
|
647
|
+
cluster.snapshotter.start();
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
const snapshotter = new StateSnapshotter({
|
|
652
|
+
messageBus: cluster.messageBus,
|
|
653
|
+
clusterId: cluster.id,
|
|
654
|
+
});
|
|
655
|
+
snapshotter.start();
|
|
656
|
+
cluster.snapshotter = snapshotter;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
/**
|
|
660
|
+
* Ensure clusters file exists (required for file locking)
|
|
661
|
+
* @private
|
|
662
|
+
*/
|
|
663
|
+
_ensureClustersFile() {
|
|
664
|
+
if (!fs.existsSync(this.storageDir)) {
|
|
665
|
+
try {
|
|
666
|
+
fs.mkdirSync(this.storageDir, { recursive: true });
|
|
667
|
+
} catch (error) {
|
|
668
|
+
console.warn(
|
|
669
|
+
`[Orchestrator] Failed to create storage directory ${this.storageDir}: ${error.message}`
|
|
670
|
+
);
|
|
671
|
+
return null;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
676
|
+
if (!fs.existsSync(clustersFile)) {
|
|
677
|
+
fs.writeFileSync(clustersFile, '{}');
|
|
678
|
+
}
|
|
679
|
+
return clustersFile;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* Find active clusters for a given issue number
|
|
684
|
+
* Used to prevent duplicate runs on the same issue
|
|
685
|
+
* @param {number|string} issueNumber - Issue number to check
|
|
686
|
+
* @returns {Array<{id: string, state: string, createdAt: number}>} Active clusters for this issue
|
|
687
|
+
* @private
|
|
688
|
+
*/
|
|
689
|
+
_getActiveClustersForIssue(issueNumber, excludeClusterId = null) {
|
|
690
|
+
const activeClusters = [];
|
|
691
|
+
const issueNum = Number(issueNumber);
|
|
692
|
+
|
|
693
|
+
for (const [clusterId, cluster] of this.clusters) {
|
|
694
|
+
if (excludeClusterId && clusterId === excludeClusterId) continue;
|
|
695
|
+
// Skip clusters without issue numbers
|
|
696
|
+
if (!cluster.issue) continue;
|
|
697
|
+
|
|
698
|
+
// Check if same issue number
|
|
699
|
+
if (Number(cluster.issue) !== issueNum) continue;
|
|
700
|
+
|
|
701
|
+
// Check if cluster is still active (not completed/failed/stopped)
|
|
702
|
+
const inactiveStates = ['completed', 'failed', 'stopped', 'corrupted'];
|
|
703
|
+
if (inactiveStates.includes(cluster.state)) continue;
|
|
704
|
+
|
|
705
|
+
// Check if process is still running (zombie detection)
|
|
706
|
+
if (cluster.pid) {
|
|
707
|
+
try {
|
|
708
|
+
// process.kill with signal 0 checks if process exists
|
|
709
|
+
process.kill(cluster.pid, 0);
|
|
710
|
+
// Process exists - cluster is active
|
|
711
|
+
activeClusters.push({
|
|
712
|
+
id: clusterId,
|
|
713
|
+
state: cluster.state,
|
|
714
|
+
createdAt: cluster.createdAt,
|
|
715
|
+
pid: cluster.pid,
|
|
716
|
+
});
|
|
717
|
+
} catch {
|
|
718
|
+
// Process doesn't exist - cluster is a zombie (stale entry)
|
|
719
|
+
// Don't include in active list
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
return activeClusters;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/**
|
|
728
|
+
* Save clusters to persistent storage
|
|
729
|
+
* Uses file locking to prevent race conditions with other processes
|
|
730
|
+
* @private
|
|
731
|
+
*/
|
|
732
|
+
async _saveClusters() {
|
|
733
|
+
// Skip saving if orchestrator is closed (prevents race conditions during cleanup)
|
|
734
|
+
if (this.closed) {
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
const clustersFile = this._ensureClustersFile();
|
|
739
|
+
if (!clustersFile) {
|
|
740
|
+
return;
|
|
741
|
+
}
|
|
742
|
+
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
743
|
+
let release;
|
|
744
|
+
|
|
745
|
+
try {
|
|
746
|
+
// Clean stale locks from crashed processes
|
|
747
|
+
cleanStaleLock(lockfilePath);
|
|
748
|
+
|
|
749
|
+
// Acquire exclusive lock with async API (proper retries without CPU spin-wait)
|
|
750
|
+
release = await lockfile.lock(clustersFile, {
|
|
751
|
+
lockfilePath,
|
|
752
|
+
stale: LOCK_STALE_MS,
|
|
753
|
+
retries: {
|
|
754
|
+
retries: 50,
|
|
755
|
+
minTimeout: 100,
|
|
756
|
+
maxTimeout: 300,
|
|
757
|
+
randomize: true,
|
|
758
|
+
},
|
|
759
|
+
});
|
|
760
|
+
|
|
761
|
+
// Read existing clusters from file (other processes may have added clusters)
|
|
762
|
+
let existingClusters = {};
|
|
763
|
+
try {
|
|
764
|
+
const content = fs.readFileSync(clustersFile, 'utf8');
|
|
765
|
+
existingClusters = JSON.parse(content);
|
|
766
|
+
} catch (error) {
|
|
767
|
+
console.error('[Orchestrator] Failed to read existing clusters:', error.message);
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// Merge: update/add clusters from this process
|
|
771
|
+
for (const [clusterId, cluster] of this.clusters.entries()) {
|
|
772
|
+
// CRITICAL: Only update clusters this process actually owns or has modified
|
|
773
|
+
// A process owns a cluster if: it started it (pid matches) OR it explicitly stopped/killed it
|
|
774
|
+
const isOwnedByThisProcess = cluster.pid === process.pid;
|
|
775
|
+
const wasModifiedByThisProcess = cluster.state === 'stopped' || cluster.state === 'killed';
|
|
776
|
+
|
|
777
|
+
// Skip clusters we don't own and haven't modified - prevents race condition
|
|
778
|
+
// where a running cluster overwrites another process's stop/kill operation
|
|
779
|
+
if (!isOwnedByThisProcess && !wasModifiedByThisProcess) {
|
|
780
|
+
// Preserve existing state from file for clusters we don't own
|
|
781
|
+
continue;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// CRITICAL: Killed clusters are DELETED from disk, not persisted
|
|
785
|
+
// This ensures they can't be accidentally resumed
|
|
786
|
+
if (cluster.state === 'killed') {
|
|
787
|
+
delete existingClusters[clusterId];
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
existingClusters[clusterId] = {
|
|
792
|
+
id: cluster.id,
|
|
793
|
+
config: cluster.config,
|
|
794
|
+
state: cluster.state,
|
|
795
|
+
createdAt: cluster.createdAt,
|
|
796
|
+
// Track PID for zombie detection (null if cluster is stopped/killed)
|
|
797
|
+
pid: cluster.state === 'running' ? cluster.pid : null,
|
|
798
|
+
// Persist failure info for resume capability
|
|
799
|
+
failureInfo: cluster.failureInfo || null,
|
|
800
|
+
// Persist PR mode for completion agent selection
|
|
801
|
+
autoPr: cluster.autoPr || false,
|
|
802
|
+
// Persist PR options for resume
|
|
803
|
+
prOptions: cluster.prOptions || null,
|
|
804
|
+
// Persist model override for consistent agent spawning on resume
|
|
805
|
+
modelOverride: cluster.modelOverride || null,
|
|
806
|
+
// Persist issue number for heroshot/external tools
|
|
807
|
+
issue: cluster.issue || null,
|
|
808
|
+
// Persist isolation info (excluding manager instance which can't be serialized)
|
|
809
|
+
// CRITICAL: workDir is required for resume() to recreate container with same workspace
|
|
810
|
+
isolation: cluster.isolation
|
|
811
|
+
? {
|
|
812
|
+
enabled: cluster.isolation.enabled,
|
|
813
|
+
containerId: cluster.isolation.containerId,
|
|
814
|
+
image: cluster.isolation.image,
|
|
815
|
+
workDir: cluster.isolation.workDir, // Required for resume
|
|
816
|
+
}
|
|
817
|
+
: null,
|
|
818
|
+
// Persist agent runtime states for accurate status display from other processes
|
|
819
|
+
agentStates: cluster.agents
|
|
820
|
+
? cluster.agents.map((a) => ({
|
|
821
|
+
id: a.id,
|
|
822
|
+
state: a.state,
|
|
823
|
+
iteration: a.iteration,
|
|
824
|
+
currentTask: a.currentTask ? true : false,
|
|
825
|
+
currentTaskId: a.currentTaskId,
|
|
826
|
+
processPid: a.processPid,
|
|
827
|
+
}))
|
|
828
|
+
: null,
|
|
829
|
+
setupLogPath: cluster.setupLogPath || null,
|
|
830
|
+
setupStage: cluster.setupStage || null,
|
|
831
|
+
provisional: cluster.provisional || false,
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// Write merged data
|
|
836
|
+
fs.writeFileSync(clustersFile, JSON.stringify(existingClusters, null, 2));
|
|
837
|
+
this._log(
|
|
838
|
+
`[Orchestrator] Saved ${this.clusters.size} cluster(s), file now has ${Object.keys(existingClusters).length} total`
|
|
839
|
+
);
|
|
840
|
+
} catch (error) {
|
|
841
|
+
if (error.code === 'ENOENT') {
|
|
842
|
+
console.warn(
|
|
843
|
+
`[Orchestrator] Skipping cluster save; storage directory missing: ${this.storageDir}`
|
|
844
|
+
);
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
throw error;
|
|
848
|
+
} finally {
|
|
849
|
+
// Always release lock
|
|
850
|
+
if (release) {
|
|
851
|
+
await release();
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/**
|
|
857
|
+
* Watch for new clusters and call callback when found
|
|
858
|
+
* Polls the clusters file for changes with file locking
|
|
859
|
+
* @param {Function} onNewCluster - Callback(cluster) for each new cluster
|
|
860
|
+
* @param {Number} intervalMs - Poll interval in ms (default: 2000)
|
|
861
|
+
* @returns {Function} Stop function to cancel watching
|
|
862
|
+
*/
|
|
863
|
+
watchForNewClusters(onNewCluster, intervalMs = 2000) {
|
|
864
|
+
const clustersFile = path.join(this.storageDir, 'clusters.json');
|
|
865
|
+
const lockfilePath = path.join(this.storageDir, 'clusters.json.lock');
|
|
866
|
+
const knownClusterIds = new Set(this.clusters.keys());
|
|
867
|
+
|
|
868
|
+
const intervalId = setInterval(() => {
|
|
869
|
+
let release;
|
|
870
|
+
try {
|
|
871
|
+
if (!fs.existsSync(clustersFile)) return;
|
|
872
|
+
|
|
873
|
+
// Clean stale locks from crashed processes
|
|
874
|
+
cleanStaleLock(lockfilePath);
|
|
875
|
+
|
|
876
|
+
// Try to acquire lock once (polling is best-effort, will retry on next cycle)
|
|
877
|
+
try {
|
|
878
|
+
release = lockfile.lockSync(clustersFile, {
|
|
879
|
+
lockfilePath,
|
|
880
|
+
stale: LOCK_STALE_MS,
|
|
881
|
+
});
|
|
882
|
+
} catch (lockErr) {
|
|
883
|
+
// Lock busy - skip this poll cycle, try again next interval
|
|
884
|
+
if (lockErr.code === 'ELOCKED') return;
|
|
885
|
+
throw lockErr;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
const data = JSON.parse(fs.readFileSync(clustersFile, 'utf8'));
|
|
889
|
+
|
|
890
|
+
for (const [clusterId, clusterData] of Object.entries(data)) {
|
|
891
|
+
if (!knownClusterIds.has(clusterId)) {
|
|
892
|
+
// New cluster found
|
|
893
|
+
knownClusterIds.add(clusterId);
|
|
894
|
+
const cluster = this._loadSingleCluster(clusterId, clusterData);
|
|
895
|
+
if (cluster && onNewCluster) {
|
|
896
|
+
onNewCluster(cluster);
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
} catch (error) {
|
|
901
|
+
// File access during polling can fail transiently - log and continue
|
|
902
|
+
console.error(`[Orchestrator] watchForNewClusters error (will retry): ${error.message}`);
|
|
903
|
+
} finally {
|
|
904
|
+
if (release) {
|
|
905
|
+
release();
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
}, intervalMs);
|
|
909
|
+
|
|
910
|
+
// Return stop function
|
|
911
|
+
return () => clearInterval(intervalId);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
/**
|
|
915
|
+
* Start a new cluster with mocked agent executors (TESTING ONLY)
|
|
916
|
+
*
|
|
917
|
+
* CRITICAL: This method PREVENTS real Claude API calls.
|
|
918
|
+
* All agent behaviors must be defined in mockExecutor.
|
|
919
|
+
*
|
|
920
|
+
* @param {Object} config - Cluster configuration
|
|
921
|
+
* @param {Object} input - Input source { issue, text, or config }
|
|
922
|
+
* @param {MockAgentExecutor} mockExecutor - Mock executor with agent behaviors
|
|
923
|
+
* @returns {Object} Cluster object
|
|
924
|
+
*/
|
|
925
|
+
startWithMock(config, input, mockExecutor) {
|
|
926
|
+
if (!mockExecutor) {
|
|
927
|
+
throw new Error('Orchestrator.startWithMock: mockExecutor is required');
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// Validate all agents that execute tasks have mock behaviors defined
|
|
931
|
+
// Orchestrator agents (action: 'stop_cluster') don't execute tasks, so don't need mocks
|
|
932
|
+
for (const agentConfig of config.agents) {
|
|
933
|
+
const agentId = agentConfig.id;
|
|
934
|
+
|
|
935
|
+
// Check if agent has any triggers that execute tasks
|
|
936
|
+
const executesTask = agentConfig.triggers?.some(
|
|
937
|
+
(trigger) => !trigger.action || trigger.action === 'execute_task'
|
|
938
|
+
);
|
|
939
|
+
|
|
940
|
+
if (executesTask && !mockExecutor.behaviors[agentId]) {
|
|
941
|
+
throw new Error(
|
|
942
|
+
`Orchestrator.startWithMock: No behavior defined for agent '${agentId}'. ` +
|
|
943
|
+
`This would cause real Claude API calls. ABORTING.\n` +
|
|
944
|
+
`Available behaviors: ${Object.keys(mockExecutor.behaviors).join(', ')}`
|
|
945
|
+
);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
return this._startInternal(config, input, {
|
|
950
|
+
mockExecutor,
|
|
951
|
+
testMode: true,
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Start a new cluster
|
|
957
|
+
* @param {Object} config - Cluster configuration
|
|
958
|
+
* @param {Object} input - Input source { issue, text, or config }
|
|
959
|
+
* @param {Object} options - Start options
|
|
960
|
+
* @param {boolean} options.isolation - Run in Docker container
|
|
961
|
+
* @param {string} options.isolationImage - Docker image to use
|
|
962
|
+
* @param {boolean} options.worktree - Run in git worktree isolation (lightweight, no Docker)
|
|
963
|
+
* @returns {Object} Cluster object
|
|
964
|
+
*/
|
|
965
|
+
start(config, input = {}, options = {}) {
|
|
966
|
+
const testMode = options.testMode || !!this.taskRunner;
|
|
967
|
+
const autoPr = options.autoPr ?? (testMode ? false : process.env.ZEROSHOT_PR === '1');
|
|
968
|
+
return this._startInternal(config, input, {
|
|
969
|
+
...options,
|
|
970
|
+
testMode,
|
|
971
|
+
cwd: options.cwd || process.cwd(), // Target working directory for agents
|
|
972
|
+
isolation: options.isolation || false,
|
|
973
|
+
isolationImage: options.isolationImage,
|
|
974
|
+
worktree: options.worktree || false,
|
|
975
|
+
autoPr,
|
|
976
|
+
modelOverride: options.modelOverride, // Model override for all agents
|
|
977
|
+
clusterId: options.clusterId, // Explicit ID from CLI/daemon parent
|
|
978
|
+
settings: options.settings, // User settings for issue provider detection
|
|
979
|
+
forceProvider: options.forceProvider, // Force specific issue provider
|
|
980
|
+
force: options.force || false, // Skip duplicate issue check
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* Internal start implementation (shared by start and startWithMock)
|
|
986
|
+
* @private
|
|
987
|
+
*/
|
|
988
|
+
async _startInternal(config, input = {}, options = {}) {
|
|
989
|
+
// Generate a unique cluster ID for this process call.
|
|
990
|
+
// IMPORTANT: Do NOT implicitly reuse ZEROSHOT_CLUSTER_ID, because:
|
|
991
|
+
// - test harnesses may set it globally (breaking multi-start tests)
|
|
992
|
+
// - callers may start multiple clusters in one process
|
|
993
|
+
// Use it only when explicitly passed (CLI/daemon parent) via options.clusterId.
|
|
994
|
+
const clusterId = this._resolveStartClusterId(
|
|
995
|
+
options.clusterId || null,
|
|
996
|
+
config?.dbPath || null
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
// Create ledger and message bus with persistent storage
|
|
1000
|
+
const dbPath = config.dbPath || path.join(this.storageDir, `${clusterId}.db`);
|
|
1001
|
+
const ledger = new Ledger(dbPath);
|
|
1002
|
+
const messageBus = new MessageBus(ledger);
|
|
1003
|
+
|
|
1004
|
+
// Handle isolation mode (Docker container OR git worktree)
|
|
1005
|
+
const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation(
|
|
1006
|
+
options,
|
|
1007
|
+
config,
|
|
1008
|
+
clusterId
|
|
1009
|
+
);
|
|
1010
|
+
const requiredQualityGates = resolveClusterRequiredQualityGates(config, options);
|
|
1011
|
+
options.requiredQualityGates = requiredQualityGates;
|
|
1012
|
+
applyRequiredQualityGatesToValidators(config, requiredQualityGates);
|
|
1013
|
+
|
|
1014
|
+
// Build cluster object
|
|
1015
|
+
// CRITICAL: initComplete promise ensures ISSUE_OPENED is published before stop() completes
|
|
1016
|
+
// This prevents 0-message clusters from SIGINT during async initialization
|
|
1017
|
+
let resolveInitComplete;
|
|
1018
|
+
const initCompletePromise = new Promise((resolve) => {
|
|
1019
|
+
resolveInitComplete = resolve;
|
|
1020
|
+
});
|
|
1021
|
+
|
|
1022
|
+
const cluster = {
|
|
1023
|
+
id: clusterId,
|
|
1024
|
+
config,
|
|
1025
|
+
state: 'initializing',
|
|
1026
|
+
messageBus,
|
|
1027
|
+
ledger,
|
|
1028
|
+
agents: [],
|
|
1029
|
+
createdAt: Date.now(),
|
|
1030
|
+
// Track PID for zombie detection (this process owns the cluster)
|
|
1031
|
+
pid: process.pid,
|
|
1032
|
+
// Initialization completion tracking (for safe SIGINT handling)
|
|
1033
|
+
initCompletePromise,
|
|
1034
|
+
_resolveInitComplete: resolveInitComplete,
|
|
1035
|
+
autoPr: options.autoPr || false,
|
|
1036
|
+
requiredQualityGates,
|
|
1037
|
+
// PR configuration options (persisted for resume)
|
|
1038
|
+
prOptions: buildPrOptions(options, requiredQualityGates),
|
|
1039
|
+
// Model override for all agents (applied to dynamically added agents)
|
|
1040
|
+
modelOverride: options.modelOverride || null,
|
|
1041
|
+
// Issue provider tracking (where issue was fetched from)
|
|
1042
|
+
issueProvider: null, // Set after fetching issue (github, gitlab, jira, azure-devops)
|
|
1043
|
+
// Git platform tracking (where PR/MR will be created - independent of issue provider)
|
|
1044
|
+
gitPlatform: null, // Set when --pr mode is active
|
|
1045
|
+
// Isolation state (only if enabled)
|
|
1046
|
+
// CRITICAL: Store workDir for resume capability - without this, resume() can't recreate container
|
|
1047
|
+
isolation: options.isolation
|
|
1048
|
+
? {
|
|
1049
|
+
enabled: true,
|
|
1050
|
+
containerId,
|
|
1051
|
+
image: options.isolationImage || 'zeroshot-cluster-base',
|
|
1052
|
+
manager: isolationManager,
|
|
1053
|
+
workDir: options.cwd || process.cwd(), // Persisted for resume
|
|
1054
|
+
}
|
|
1055
|
+
: null,
|
|
1056
|
+
// Worktree isolation state (lightweight alternative to Docker)
|
|
1057
|
+
worktree: options.worktree
|
|
1058
|
+
? {
|
|
1059
|
+
enabled: true,
|
|
1060
|
+
path: worktreeInfo.path,
|
|
1061
|
+
branch: worktreeInfo.branch,
|
|
1062
|
+
repoRoot: worktreeInfo.repoRoot,
|
|
1063
|
+
manager: isolationManager,
|
|
1064
|
+
workDir: options.cwd || process.cwd(), // Persisted for resume
|
|
1065
|
+
}
|
|
1066
|
+
: null,
|
|
1067
|
+
};
|
|
1068
|
+
|
|
1069
|
+
this.clusters.set(clusterId, cluster);
|
|
1070
|
+
this._startSnapshotter(cluster);
|
|
1071
|
+
|
|
1072
|
+
// Persist the cluster immediately so detached runs can't create "invisible" initializing clusters.
|
|
1073
|
+
// Without this, an early startup failure (before ISSUE_OPENED / TASK_STARTED) may never be saved,
|
|
1074
|
+
// and external supervisors (e.g., heroshot) can't detect/cleanup the stuck state.
|
|
1075
|
+
await this._saveClusters().catch((err) => {
|
|
1076
|
+
console.warn(`[Orchestrator] Failed to persist initial cluster state for ${clusterId}:`, err);
|
|
1077
|
+
});
|
|
1078
|
+
|
|
1079
|
+
try {
|
|
1080
|
+
// Fetch input (issue from provider, file, or text)
|
|
1081
|
+
let inputData;
|
|
1082
|
+
if (input.issue) {
|
|
1083
|
+
// Detect provider and fetch issue
|
|
1084
|
+
const ProviderClass = detectProvider(
|
|
1085
|
+
input.issue,
|
|
1086
|
+
options.settings || {},
|
|
1087
|
+
options.forceProvider
|
|
1088
|
+
);
|
|
1089
|
+
if (!ProviderClass) {
|
|
1090
|
+
throw new Error(`No issue provider matched input: ${input.issue}`);
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
const provider = new ProviderClass();
|
|
1094
|
+
inputData = await provider.fetchIssue(input.issue, options.settings || {});
|
|
1095
|
+
|
|
1096
|
+
// Store issue provider for logging/debugging and cross-provider workflows
|
|
1097
|
+
cluster.issueProvider = ProviderClass.id;
|
|
1098
|
+
|
|
1099
|
+
// Store issue number for heroshot/external tools (avoids log parsing)
|
|
1100
|
+
cluster.issue = inputData.number || null;
|
|
1101
|
+
|
|
1102
|
+
// Persist issue number early so supervisors can associate this cluster with the issue even if start fails.
|
|
1103
|
+
await this._saveClusters().catch((err) => {
|
|
1104
|
+
console.warn(`[Orchestrator] Failed to persist issue number for ${clusterId}:`, err);
|
|
1105
|
+
});
|
|
1106
|
+
|
|
1107
|
+
// Check for duplicate active runs on same issue (unless --force)
|
|
1108
|
+
if (cluster.issue && !options.force) {
|
|
1109
|
+
const activeClusters = this._getActiveClustersForIssue(cluster.issue, clusterId);
|
|
1110
|
+
if (activeClusters.length > 0) {
|
|
1111
|
+
const existing = activeClusters[0];
|
|
1112
|
+
const age = Math.round((Date.now() - existing.createdAt) / 1000 / 60);
|
|
1113
|
+
throw new Error(
|
|
1114
|
+
`Issue #${cluster.issue} already has an active cluster:\n` +
|
|
1115
|
+
` Cluster: ${existing.id} (state: ${existing.state}, running for ${age}min, pid: ${existing.pid})\n\n` +
|
|
1116
|
+
`Options:\n` +
|
|
1117
|
+
` 1. Kill existing: zeroshot kill ${existing.id}\n` +
|
|
1118
|
+
` 2. Override check: zeroshot run ${input.issue} --force\n` +
|
|
1119
|
+
` 3. View status: zeroshot status ${existing.id}`
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
// Log clickable issue link
|
|
1125
|
+
if (inputData.url) {
|
|
1126
|
+
this._log(`[Orchestrator] Issue (${ProviderClass.displayName}): ${inputData.url}`);
|
|
1127
|
+
}
|
|
1128
|
+
} else if (input.file) {
|
|
1129
|
+
inputData = InputHelpers.createFileInput(input.file);
|
|
1130
|
+
this._log(`[Orchestrator] File: ${input.file}`);
|
|
1131
|
+
} else if (input.text) {
|
|
1132
|
+
inputData = InputHelpers.createTextInput(input.text);
|
|
1133
|
+
} else {
|
|
1134
|
+
throw new Error('Either issue, file, or text input is required');
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
// Detect git platform for --pr mode (independent of issue provider)
|
|
1138
|
+
if (options.autoPr) {
|
|
1139
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
1140
|
+
const gitContext = detectGitContext(options.cwd);
|
|
1141
|
+
cluster.gitPlatform = gitContext?.provider || null;
|
|
1142
|
+
|
|
1143
|
+
if (cluster.gitPlatform) {
|
|
1144
|
+
this._log(`[Orchestrator] Git platform detected: ${cluster.gitPlatform.toUpperCase()}`);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
// Inject git-pusher agent if --pr is set (replaces completion-detector)
|
|
1149
|
+
this._applyAutoPrConfig(config, inputData, options);
|
|
1150
|
+
if (options.autoPr) {
|
|
1151
|
+
applyPushBlockedRepairTriggers(config);
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
// Inject workers instruction if --workers explicitly provided and > 1
|
|
1155
|
+
this._applyWorkerInstruction(config);
|
|
1156
|
+
|
|
1157
|
+
// Initialize agents with optional mock injection
|
|
1158
|
+
// Check agent type: regular agent or subcluster
|
|
1159
|
+
// CRITICAL: Inject cwd into each agent config for proper working directory
|
|
1160
|
+
// In worktree mode, agents run in the worktree path (not original cwd)
|
|
1161
|
+
this._initializeClusterAgents({
|
|
1162
|
+
config,
|
|
1163
|
+
cluster,
|
|
1164
|
+
messageBus,
|
|
1165
|
+
options,
|
|
1166
|
+
isolationManager,
|
|
1167
|
+
clusterId,
|
|
1168
|
+
});
|
|
1169
|
+
|
|
1170
|
+
// ========================================================================
|
|
1171
|
+
// CRITICAL ORDERING INVARIANT (Issue #31 - Subscription Race Condition)
|
|
1172
|
+
// ========================================================================
|
|
1173
|
+
// MUST register ALL subscriptions BEFORE starting agents.
|
|
1174
|
+
//
|
|
1175
|
+
// WHY: EventEmitter is synchronous and doesn't replay past events to new
|
|
1176
|
+
// listeners. If an agent completes and publishes CLUSTER_COMPLETE before
|
|
1177
|
+
// we register the subscription, the orchestrator will never receive the
|
|
1178
|
+
// message and the cluster will appear stuck in 'executing_task' forever.
|
|
1179
|
+
//
|
|
1180
|
+
// ORDER:
|
|
1181
|
+
// 1. Register subscriptions (lines below)
|
|
1182
|
+
// 2. Start agents
|
|
1183
|
+
// 3. Publish ISSUE_OPENED
|
|
1184
|
+
//
|
|
1185
|
+
// DO NOT move subscriptions after agent.start() - this will reintroduce
|
|
1186
|
+
// the race condition fixed in issue #31.
|
|
1187
|
+
// ========================================================================
|
|
1188
|
+
this._registerClusterSubscriptions({
|
|
1189
|
+
messageBus,
|
|
1190
|
+
clusterId,
|
|
1191
|
+
isolationManager,
|
|
1192
|
+
containerId,
|
|
1193
|
+
});
|
|
1194
|
+
|
|
1195
|
+
// Start all agents
|
|
1196
|
+
for (const agent of cluster.agents) {
|
|
1197
|
+
await agent.start();
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
cluster.state = 'running';
|
|
1201
|
+
|
|
1202
|
+
// Publish ISSUE_OPENED message to bootstrap workflow
|
|
1203
|
+
messageBus.publish({
|
|
1204
|
+
cluster_id: clusterId,
|
|
1205
|
+
topic: 'ISSUE_OPENED',
|
|
1206
|
+
sender: 'system',
|
|
1207
|
+
receiver: 'broadcast',
|
|
1208
|
+
content: {
|
|
1209
|
+
text: inputData.context,
|
|
1210
|
+
data: {
|
|
1211
|
+
issue_number: inputData.number,
|
|
1212
|
+
title: inputData.title,
|
|
1213
|
+
},
|
|
1214
|
+
},
|
|
1215
|
+
metadata: {
|
|
1216
|
+
source: this._getInputSource(input),
|
|
1217
|
+
},
|
|
1218
|
+
});
|
|
1219
|
+
|
|
1220
|
+
// CRITICAL: Mark initialization complete AFTER ISSUE_OPENED is published
|
|
1221
|
+
// This ensures stop() waits for at least 1 message before stopping
|
|
1222
|
+
if (cluster._resolveInitComplete) {
|
|
1223
|
+
cluster._resolveInitComplete();
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
this._log(`Cluster ${clusterId} started with ${cluster.agents.length} agents`);
|
|
1227
|
+
|
|
1228
|
+
// DISABLED: Idle timeout auto-stop mechanism
|
|
1229
|
+
// WHY DISABLED: Clusters should only stop on explicit signals:
|
|
1230
|
+
// - User `zeroshot kill` command
|
|
1231
|
+
// - CLUSTER_COMPLETE message (successful completion)
|
|
1232
|
+
// - CLUSTER_FAILED message (failure/abort)
|
|
1233
|
+
// Being "idle" is NOT a reason to auto-stop - agents may be legitimately
|
|
1234
|
+
// waiting for external events, user input (in interactive mode), or
|
|
1235
|
+
// processing that doesn't show as "executing" (e.g., polling, monitoring).
|
|
1236
|
+
//
|
|
1237
|
+
// Previous behavior: Stopped cluster after 2 minutes of all agents idle
|
|
1238
|
+
// Result: Clusters were killed while legitimately waiting, causing confusion
|
|
1239
|
+
//
|
|
1240
|
+
// cluster.idleCheckInterval = setInterval(() => { ... }, 30000);
|
|
1241
|
+
// ^^^^^^ REMOVED - clusters run until explicitly stopped or completed
|
|
1242
|
+
|
|
1243
|
+
// Save cluster to disk
|
|
1244
|
+
await this._saveClusters();
|
|
1245
|
+
|
|
1246
|
+
return {
|
|
1247
|
+
id: clusterId,
|
|
1248
|
+
state: cluster.state,
|
|
1249
|
+
agents: cluster.agents.map((a) => a.getState()),
|
|
1250
|
+
ledger: cluster.ledger, // Expose ledger for testing
|
|
1251
|
+
messageBus: cluster.messageBus, // Expose messageBus for testing
|
|
1252
|
+
};
|
|
1253
|
+
} catch (error) {
|
|
1254
|
+
cluster.state = 'failed';
|
|
1255
|
+
// CRITICAL: Resolve the promise on failure too, so stop() doesn't hang
|
|
1256
|
+
if (cluster._resolveInitComplete) {
|
|
1257
|
+
cluster._resolveInitComplete();
|
|
1258
|
+
}
|
|
1259
|
+
// Persist the failure state (prevents "invisible" failures that supervisors can't detect/cleanup).
|
|
1260
|
+
await this._saveClusters().catch((err) => {
|
|
1261
|
+
console.warn(
|
|
1262
|
+
`[Orchestrator] Failed to persist failed cluster state for ${clusterId}:`,
|
|
1263
|
+
err
|
|
1264
|
+
);
|
|
1265
|
+
});
|
|
1266
|
+
|
|
1267
|
+
// Best-effort cleanup of partially initialized resources (prevents orphaned worktrees/containers).
|
|
1268
|
+
try {
|
|
1269
|
+
if (cluster.snapshotter) {
|
|
1270
|
+
cluster.snapshotter.stop();
|
|
1271
|
+
}
|
|
1272
|
+
} catch {
|
|
1273
|
+
// ignore
|
|
1274
|
+
}
|
|
1275
|
+
try {
|
|
1276
|
+
for (const agent of cluster.agents || []) {
|
|
1277
|
+
// Agent start may have partially succeeded.
|
|
1278
|
+
// Stop is best-effort and must not mask the original error.
|
|
1279
|
+
await agent.stop();
|
|
1280
|
+
}
|
|
1281
|
+
} catch {
|
|
1282
|
+
// ignore
|
|
1283
|
+
}
|
|
1284
|
+
try {
|
|
1285
|
+
if (cluster.isolation?.manager) {
|
|
1286
|
+
// Preserve workspace for inspection; callers can `zeroshot kill` for full cleanup.
|
|
1287
|
+
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
|
|
1288
|
+
}
|
|
1289
|
+
} catch {
|
|
1290
|
+
// ignore
|
|
1291
|
+
}
|
|
1292
|
+
try {
|
|
1293
|
+
if (cluster.worktree?.manager) {
|
|
1294
|
+
cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
|
|
1295
|
+
}
|
|
1296
|
+
} catch {
|
|
1297
|
+
// ignore
|
|
1298
|
+
}
|
|
1299
|
+
try {
|
|
1300
|
+
cluster.messageBus?.close?.();
|
|
1301
|
+
} catch {
|
|
1302
|
+
// ignore
|
|
1303
|
+
}
|
|
1304
|
+
try {
|
|
1305
|
+
cluster.ledger?.close?.();
|
|
1306
|
+
} catch {
|
|
1307
|
+
// ignore
|
|
1308
|
+
}
|
|
1309
|
+
console.error(`Cluster ${clusterId} failed to start:`, error);
|
|
1310
|
+
throw error;
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
_initializeClusterAgents({ config, cluster, messageBus, options, isolationManager, clusterId }) {
|
|
1315
|
+
const agentCwd = cluster.worktree ? cluster.worktree.path : options.cwd || process.cwd();
|
|
1316
|
+
|
|
1317
|
+
for (const agentConfig of config.agents) {
|
|
1318
|
+
if (!agentConfig.cwd) {
|
|
1319
|
+
agentConfig.cwd = agentCwd;
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
if (options.modelOverride) {
|
|
1323
|
+
applyModelOverride(agentConfig, options.modelOverride);
|
|
1324
|
+
this._log(` [model] Overridden model for ${agentConfig.id}: ${options.modelOverride}`);
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
const agentOptions = {
|
|
1328
|
+
testMode: options.testMode || !!this.taskRunner,
|
|
1329
|
+
quiet: this.quiet,
|
|
1330
|
+
modelOverride: options.modelOverride || null,
|
|
1331
|
+
};
|
|
1332
|
+
|
|
1333
|
+
if (options.mockExecutor) {
|
|
1334
|
+
agentOptions.mockSpawnFn = options.mockExecutor.createMockSpawnFn(agentConfig.id);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
if (this.taskRunner) {
|
|
1338
|
+
agentOptions.taskRunner = this.taskRunner;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
if (cluster.isolation) {
|
|
1342
|
+
agentOptions.isolation = {
|
|
1343
|
+
enabled: true,
|
|
1344
|
+
manager: isolationManager,
|
|
1345
|
+
clusterId,
|
|
1346
|
+
};
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
if (cluster.worktree) {
|
|
1350
|
+
agentOptions.worktree = {
|
|
1351
|
+
enabled: true,
|
|
1352
|
+
path: cluster.worktree.path,
|
|
1353
|
+
branch: cluster.worktree.branch,
|
|
1354
|
+
repoRoot: cluster.worktree.repoRoot,
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
const agent =
|
|
1359
|
+
agentConfig.type === 'subcluster'
|
|
1360
|
+
? new SubClusterWrapper(agentConfig, messageBus, cluster, agentOptions)
|
|
1361
|
+
: new AgentWrapper(agentConfig, messageBus, cluster, agentOptions);
|
|
1362
|
+
|
|
1363
|
+
cluster.agents.push(agent);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
_subscribeToClusterTopic(messageBus, clusterId, topic, handler) {
|
|
1368
|
+
messageBus.subscribe((message) => {
|
|
1369
|
+
if (message.topic === topic && message.cluster_id === clusterId) {
|
|
1370
|
+
handler(message);
|
|
1371
|
+
}
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
_registerClusterCompletionHandlers(messageBus, clusterId) {
|
|
1376
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_COMPLETE', (message) => {
|
|
1377
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1378
|
+
this._log(`✅ CLUSTER COMPLETED SUCCESSFULLY: ${clusterId}`);
|
|
1379
|
+
this._log(`${'='.repeat(80)}`);
|
|
1380
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
1381
|
+
this._log(`Initiated by: ${message.sender}`);
|
|
1382
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1383
|
+
|
|
1384
|
+
this.stop(clusterId, { completedSuccessfully: true }).catch((err) => {
|
|
1385
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
1386
|
+
});
|
|
1387
|
+
});
|
|
1388
|
+
|
|
1389
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_FAILED', (message) => {
|
|
1390
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1391
|
+
this._log(`❌ CLUSTER FAILED: ${clusterId}`);
|
|
1392
|
+
this._log(`${'='.repeat(80)}`);
|
|
1393
|
+
this._log(`Reason: ${message.content?.data?.reason || 'unknown'}`);
|
|
1394
|
+
this._log(`Agent: ${message.sender}`);
|
|
1395
|
+
if (message.content?.text) {
|
|
1396
|
+
this._log(`Details: ${message.content.text}`);
|
|
1397
|
+
}
|
|
1398
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1399
|
+
|
|
1400
|
+
this.stop(clusterId).catch((err) => {
|
|
1401
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
1402
|
+
});
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
_registerAgentErrorHandler(messageBus, clusterId) {
|
|
1407
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_ERROR', async (message) => {
|
|
1408
|
+
const agentRole = message.content?.data?.role;
|
|
1409
|
+
const attempts = message.content?.data?.attempts || 1;
|
|
1410
|
+
const hookFailure = message.content?.data?.hookFailure === true;
|
|
1411
|
+
|
|
1412
|
+
await this._saveClusters();
|
|
1413
|
+
|
|
1414
|
+
const shouldStopForRole =
|
|
1415
|
+
agentRole === 'implementation' ||
|
|
1416
|
+
agentRole === 'coordinator' ||
|
|
1417
|
+
message.sender === 'consensus-coordinator';
|
|
1418
|
+
const shouldStop = shouldStopForRole && (hookFailure || attempts >= 3);
|
|
1419
|
+
|
|
1420
|
+
if (shouldStop) {
|
|
1421
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1422
|
+
this._log(`❌ CRITICAL AGENT FAILED: ${clusterId}`);
|
|
1423
|
+
this._log(`${'='.repeat(80)}`);
|
|
1424
|
+
this._log(
|
|
1425
|
+
`${message.sender} (${agentRole || 'unknown role'}) failed` +
|
|
1426
|
+
(hookFailure ? ` (hookFailure=true)` : ``) +
|
|
1427
|
+
` after ${attempts} attempts`
|
|
1428
|
+
);
|
|
1429
|
+
this._log(`Error: ${message.content?.data?.error || 'unknown'}`);
|
|
1430
|
+
this._log(`Stopping cluster - critical agent cannot continue`);
|
|
1431
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1432
|
+
|
|
1433
|
+
this.stop(clusterId).catch((err) => {
|
|
1434
|
+
console.error(`Failed to auto-stop cluster ${clusterId}:`, err.message);
|
|
1435
|
+
});
|
|
1436
|
+
}
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
_registerPushBlockedHandler(messageBus, clusterId) {
|
|
1441
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'PUSH_BLOCKED', async (message) => {
|
|
1442
|
+
const reason = message.content?.data?.blocked_reason || 'unknown';
|
|
1443
|
+
const prNumber = message.content?.data?.pr_number || message.content?.data?.mr_number;
|
|
1444
|
+
this._log(`🔁 git-pusher blocked${prNumber ? ` (${prNumber})` : ''}: ${reason}`);
|
|
1445
|
+
await this._saveClusters();
|
|
1446
|
+
});
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
_registerAgentLifecycleHandlers(messageBus, clusterId) {
|
|
1450
|
+
messageBus.on('topic:AGENT_LIFECYCLE', async (message) => {
|
|
1451
|
+
const event = message.content?.data?.event;
|
|
1452
|
+
if (
|
|
1453
|
+
[
|
|
1454
|
+
'TASK_STARTED',
|
|
1455
|
+
'TASK_COMPLETED',
|
|
1456
|
+
'PROCESS_SPAWNED',
|
|
1457
|
+
'TASK_ID_ASSIGNED',
|
|
1458
|
+
'STARTED',
|
|
1459
|
+
].includes(event)
|
|
1460
|
+
) {
|
|
1461
|
+
await this._saveClusters();
|
|
1462
|
+
}
|
|
1463
|
+
});
|
|
1464
|
+
|
|
1465
|
+
messageBus.on('topic:AGENT_LIFECYCLE', (message) => {
|
|
1466
|
+
if (message.content?.data?.event !== 'AGENT_STALE_WARNING') return;
|
|
1467
|
+
|
|
1468
|
+
const agentId = message.content?.data?.agent;
|
|
1469
|
+
const timeSinceLastOutput = message.content?.data?.timeSinceLastOutput;
|
|
1470
|
+
const analysis = message.content?.data?.analysis || 'No analysis available';
|
|
1471
|
+
|
|
1472
|
+
this._log(
|
|
1473
|
+
`⚠️ Orchestrator: Agent ${agentId} appears stale (${Math.round(timeSinceLastOutput / 1000)}s no output) but will NOT be killed`
|
|
1474
|
+
);
|
|
1475
|
+
this._log(` Analysis: ${analysis}`);
|
|
1476
|
+
this._log(
|
|
1477
|
+
` Manual intervention may be needed - use 'zeroshot resume ${clusterId}' if stuck`
|
|
1478
|
+
);
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
_registerConductorWatchdog(messageBus, clusterId) {
|
|
1483
|
+
const timeoutMs = 30000;
|
|
1484
|
+
let watchdogTimer = null;
|
|
1485
|
+
let completedAt = null;
|
|
1486
|
+
|
|
1487
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'AGENT_LIFECYCLE', (message) => {
|
|
1488
|
+
const event = message.content?.data?.event;
|
|
1489
|
+
const role = message.content?.data?.role;
|
|
1490
|
+
|
|
1491
|
+
if (event === 'TASK_COMPLETED' && role === 'conductor') {
|
|
1492
|
+
completedAt = Date.now();
|
|
1493
|
+
this._log(
|
|
1494
|
+
`⏱️ Conductor completed. Watchdog started - expecting CLUSTER_OPERATIONS within ${timeoutMs / 1000}s`
|
|
1495
|
+
);
|
|
1496
|
+
|
|
1497
|
+
watchdogTimer = setTimeout(() => {
|
|
1498
|
+
const clusterOps = messageBus.query({
|
|
1499
|
+
cluster_id: clusterId,
|
|
1500
|
+
topic: 'CLUSTER_OPERATIONS',
|
|
1501
|
+
limit: 1,
|
|
1502
|
+
});
|
|
1503
|
+
if (clusterOps.length === 0) {
|
|
1504
|
+
console.error(`\n${'='.repeat(80)}`);
|
|
1505
|
+
console.error(`🔴 CONDUCTOR WATCHDOG TRIGGERED - CLUSTER_OPERATIONS NEVER RECEIVED`);
|
|
1506
|
+
console.error(`${'='.repeat(80)}`);
|
|
1507
|
+
console.error(`Conductor completed ${timeoutMs / 1000}s ago but no CLUSTER_OPERATIONS`);
|
|
1508
|
+
console.error(`This indicates the conductor's onComplete hook FAILED SILENTLY`);
|
|
1509
|
+
console.error(
|
|
1510
|
+
`Check: 1) Result parsing 2) Transform script errors 3) Schema validation`
|
|
1511
|
+
);
|
|
1512
|
+
console.error(`${'='.repeat(80)}\n`);
|
|
1513
|
+
|
|
1514
|
+
messageBus.publish({
|
|
1515
|
+
cluster_id: clusterId,
|
|
1516
|
+
topic: 'CLUSTER_FAILED',
|
|
1517
|
+
sender: 'orchestrator',
|
|
1518
|
+
content: {
|
|
1519
|
+
text: `Conductor completed but CLUSTER_OPERATIONS never published - hook failure`,
|
|
1520
|
+
data: {
|
|
1521
|
+
reason: 'CONDUCTOR_WATCHDOG_TIMEOUT',
|
|
1522
|
+
conductorCompletedAt: completedAt,
|
|
1523
|
+
timeoutMs: timeoutMs,
|
|
1524
|
+
},
|
|
1525
|
+
},
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
}, timeoutMs);
|
|
1529
|
+
}
|
|
1530
|
+
});
|
|
1531
|
+
|
|
1532
|
+
return {
|
|
1533
|
+
clear: () => {
|
|
1534
|
+
if (!watchdogTimer) {
|
|
1535
|
+
return;
|
|
1536
|
+
}
|
|
1537
|
+
clearTimeout(watchdogTimer);
|
|
1538
|
+
watchdogTimer = null;
|
|
1539
|
+
const elapsed = completedAt ? Date.now() - completedAt : 0;
|
|
1540
|
+
this._log(
|
|
1541
|
+
`✅ CLUSTER_OPERATIONS received (${elapsed}ms after conductor completed) - watchdog cleared`
|
|
1542
|
+
);
|
|
1543
|
+
},
|
|
1544
|
+
};
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
_registerClusterOperationsHandler(
|
|
1548
|
+
messageBus,
|
|
1549
|
+
clusterId,
|
|
1550
|
+
isolationManager,
|
|
1551
|
+
containerId,
|
|
1552
|
+
watchdog
|
|
1553
|
+
) {
|
|
1554
|
+
this._subscribeToClusterTopic(messageBus, clusterId, 'CLUSTER_OPERATIONS', (message) => {
|
|
1555
|
+
watchdog?.clear?.();
|
|
1556
|
+
|
|
1557
|
+
let operations = message.content?.data?.operations;
|
|
1558
|
+
if (typeof operations === 'string') {
|
|
1559
|
+
try {
|
|
1560
|
+
operations = JSON.parse(operations);
|
|
1561
|
+
} catch (e) {
|
|
1562
|
+
this._log(`⚠️ CLUSTER_OPERATIONS has invalid operations JSON: ${e.message}`);
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
if (!operations || !Array.isArray(operations)) {
|
|
1568
|
+
this._log(`⚠️ CLUSTER_OPERATIONS missing operations array, ignoring`);
|
|
1569
|
+
return;
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1573
|
+
this._log(`🔧 CLUSTER_OPERATIONS received from ${message.sender}`);
|
|
1574
|
+
this._log(`${'='.repeat(80)}`);
|
|
1575
|
+
if (message.content?.data?.reasoning) {
|
|
1576
|
+
this._log(`Reasoning: ${message.content.data.reasoning}`);
|
|
1577
|
+
}
|
|
1578
|
+
this._log(`Operations: ${operations.length}`);
|
|
1579
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1580
|
+
|
|
1581
|
+
this._handleOperations(clusterId, operations, message.sender, {
|
|
1582
|
+
isolationManager,
|
|
1583
|
+
containerId,
|
|
1584
|
+
}).catch((err) => {
|
|
1585
|
+
console.error(`Failed to execute CLUSTER_OPERATIONS:`, err.message);
|
|
1586
|
+
messageBus.publish({
|
|
1587
|
+
cluster_id: clusterId,
|
|
1588
|
+
topic: 'CLUSTER_OPERATIONS_FAILED',
|
|
1589
|
+
sender: 'orchestrator',
|
|
1590
|
+
content: {
|
|
1591
|
+
text: `Operation chain failed: ${err.message}`,
|
|
1592
|
+
data: {
|
|
1593
|
+
error: err.message,
|
|
1594
|
+
operations: operations,
|
|
1595
|
+
},
|
|
1596
|
+
},
|
|
1597
|
+
});
|
|
1598
|
+
|
|
1599
|
+
this._log(`\n${'='.repeat(80)}`);
|
|
1600
|
+
this._log(`❌ CLUSTER_OPERATIONS FAILED - STOPPING CLUSTER`);
|
|
1601
|
+
this._log(`${'='.repeat(80)}`);
|
|
1602
|
+
this._log(`Error: ${err.message}`);
|
|
1603
|
+
this._log(`${'='.repeat(80)}\n`);
|
|
1604
|
+
|
|
1605
|
+
this.stop(clusterId).catch((stopErr) => {
|
|
1606
|
+
console.error(`Failed to stop cluster after operation failure:`, stopErr.message);
|
|
1607
|
+
});
|
|
1608
|
+
});
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
_registerClusterSubscriptions({ messageBus, clusterId, isolationManager, containerId }) {
|
|
1613
|
+
this._registerClusterCompletionHandlers(messageBus, clusterId);
|
|
1614
|
+
this._registerAgentErrorHandler(messageBus, clusterId);
|
|
1615
|
+
this._registerPushBlockedHandler(messageBus, clusterId);
|
|
1616
|
+
this._registerAgentLifecycleHandlers(messageBus, clusterId);
|
|
1617
|
+
|
|
1618
|
+
const watchdog = this._registerConductorWatchdog(messageBus, clusterId);
|
|
1619
|
+
this._registerClusterOperationsHandler(
|
|
1620
|
+
messageBus,
|
|
1621
|
+
clusterId,
|
|
1622
|
+
isolationManager,
|
|
1623
|
+
containerId,
|
|
1624
|
+
watchdog
|
|
1625
|
+
);
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
async _initializeIsolation(options, config, clusterId) {
|
|
1629
|
+
let isolationManager = null;
|
|
1630
|
+
let containerId = null;
|
|
1631
|
+
let worktreeInfo = null;
|
|
1632
|
+
|
|
1633
|
+
if (options.isolation) {
|
|
1634
|
+
if (!IsolationManager.isDockerAvailable()) {
|
|
1635
|
+
throw new Error('Docker is not available. Install Docker to use --docker mode.');
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
const image = options.isolationImage || 'zeroshot-cluster-base';
|
|
1639
|
+
await IsolationManager.ensureImage(image);
|
|
1640
|
+
|
|
1641
|
+
isolationManager = new IsolationManager({ image });
|
|
1642
|
+
this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`);
|
|
1643
|
+
|
|
1644
|
+
const workDir = options.cwd || process.cwd();
|
|
1645
|
+
const providerName = this._resolveClusterProvider(config);
|
|
1646
|
+
containerId = await isolationManager.createContainer(clusterId, {
|
|
1647
|
+
workDir,
|
|
1648
|
+
image,
|
|
1649
|
+
noMounts: options.noMounts,
|
|
1650
|
+
mounts: options.mounts,
|
|
1651
|
+
containerHome: options.containerHome,
|
|
1652
|
+
provider: providerName,
|
|
1653
|
+
});
|
|
1654
|
+
this._log(`[Orchestrator] Container created: ${containerId} (workDir: ${workDir})`);
|
|
1655
|
+
} else if (options.worktree) {
|
|
1656
|
+
const workDir = options.cwd || process.cwd();
|
|
1657
|
+
|
|
1658
|
+
isolationManager = new IsolationManager({});
|
|
1659
|
+
// `prBase` is the PR target branch. Use that same local ref as the
|
|
1660
|
+
// worktree base so repo-local integration commits are not skipped.
|
|
1661
|
+
const worktreeOptions = {};
|
|
1662
|
+
if (options.prBase) {
|
|
1663
|
+
worktreeOptions.baseRef = options.prBase;
|
|
1664
|
+
this._log(`[Orchestrator] Using worktree base ref: ${options.prBase}`);
|
|
1665
|
+
}
|
|
1666
|
+
worktreeInfo = isolationManager.createWorktreeIsolation(clusterId, workDir, worktreeOptions);
|
|
1667
|
+
|
|
1668
|
+
this._log(`[Orchestrator] Starting cluster in worktree isolation mode`);
|
|
1669
|
+
this._log(`[Orchestrator] Worktree: ${worktreeInfo.path}`);
|
|
1670
|
+
this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`);
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
return { isolationManager, containerId, worktreeInfo };
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
_applyAutoPrConfig(config, inputData, options) {
|
|
1677
|
+
if (!options.autoPr) {
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
config.agents = config.agents.filter((a) => a.id !== 'completion-detector');
|
|
1682
|
+
|
|
1683
|
+
// Detect git platform (independent of issue provider)
|
|
1684
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
1685
|
+
const { detectGitContext } = require('../lib/git-remote-utils');
|
|
1686
|
+
|
|
1687
|
+
let platform;
|
|
1688
|
+
try {
|
|
1689
|
+
// ALWAYS use git context, regardless of issue provider
|
|
1690
|
+
// This allows Jira issues in GitHub repos, etc.
|
|
1691
|
+
platform = getPlatformForPR(options.cwd);
|
|
1692
|
+
} catch (error) {
|
|
1693
|
+
throw new Error(`--pr mode failed: ${error.message}`);
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
// Check for repo mismatch (issue URL repo vs current git remote)
|
|
1697
|
+
// This catches cases like: running `zeroshot run https://gitlab.com/foo/bar/-/issues/1`
|
|
1698
|
+
// from a different repo directory
|
|
1699
|
+
let skipCloseIssue = false;
|
|
1700
|
+
const gitContext = detectGitContext(options.cwd);
|
|
1701
|
+
if (inputData.url && gitContext) {
|
|
1702
|
+
const issueHost = this._extractHostFromUrl(inputData.url);
|
|
1703
|
+
const gitHost = gitContext.host;
|
|
1704
|
+
|
|
1705
|
+
// Compare hosts - if different platforms, warn and don't close issue
|
|
1706
|
+
if (issueHost && gitHost && issueHost !== gitHost) {
|
|
1707
|
+
this._log(
|
|
1708
|
+
`[Orchestrator] ⚠️ WARNING: Issue is from ${issueHost} but PR will be created on ${gitHost}`
|
|
1709
|
+
);
|
|
1710
|
+
this._log(
|
|
1711
|
+
`[Orchestrator] ⚠️ The PR will NOT auto-close the issue (different repositories)`
|
|
1712
|
+
);
|
|
1713
|
+
this._log(`[Orchestrator] ⚠️ To fix: run zeroshot from the target repository directory`);
|
|
1714
|
+
skipCloseIssue = true;
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
// Generate platform-specific git-pusher agent from template
|
|
1719
|
+
const { generateGitPusherAgent, isPlatformSupported } = require('./agents/git-pusher-template');
|
|
1720
|
+
|
|
1721
|
+
if (!isPlatformSupported(platform)) {
|
|
1722
|
+
throw new Error(
|
|
1723
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
1724
|
+
);
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
const gitPusherConfig = generateGitPusherAgent(platform, {
|
|
1728
|
+
prBase: options.prBase,
|
|
1729
|
+
mergeQueue: options.mergeQueue,
|
|
1730
|
+
closeIssue: options.closeIssue,
|
|
1731
|
+
requiredQualityGates: options.requiredQualityGates,
|
|
1732
|
+
cwd: options.cwd,
|
|
1733
|
+
});
|
|
1734
|
+
|
|
1735
|
+
// Template replacement for issue context
|
|
1736
|
+
const issueRef = skipCloseIssue ? '' : `Closes #${inputData.number || 'unknown'}`;
|
|
1737
|
+
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
1738
|
+
.replace(/\{\{issue_number\}\}/g, inputData.number || 'unknown')
|
|
1739
|
+
.replace(/\{\{issue_title\}\}/g, inputData.title || 'Implementation')
|
|
1740
|
+
.replace(/Closes #\{\{issue_number\}\}/g, issueRef);
|
|
1741
|
+
|
|
1742
|
+
config.agents.push(gitPusherConfig);
|
|
1743
|
+
this._log(
|
|
1744
|
+
`[Orchestrator] Injected ${platform}-git-pusher agent (issue: ${inputData.number || 'N/A'}, PR platform: ${platform.toUpperCase()})`
|
|
1745
|
+
);
|
|
1746
|
+
}
|
|
1747
|
+
|
|
1748
|
+
/**
|
|
1749
|
+
* Extract hostname from a URL
|
|
1750
|
+
* @private
|
|
1751
|
+
*/
|
|
1752
|
+
_extractHostFromUrl(url) {
|
|
1753
|
+
try {
|
|
1754
|
+
const match = url.match(/https?:\/\/([^/]+)/);
|
|
1755
|
+
return match ? match[1] : null;
|
|
1756
|
+
} catch {
|
|
1757
|
+
return null;
|
|
1758
|
+
}
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
_applyWorkerInstruction(config) {
|
|
1762
|
+
const workersCount = process.env.ZEROSHOT_WORKERS ? parseInt(process.env.ZEROSHOT_WORKERS) : 0;
|
|
1763
|
+
if (workersCount <= 1) {
|
|
1764
|
+
return;
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
const workerAgent = config.agents.find((a) => a.id === 'worker');
|
|
1768
|
+
if (!workerAgent) {
|
|
1769
|
+
return;
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
const instruction = `PARALLELIZATION: Use up to ${workersCount} sub-agents to parallelize your work where appropriate.\n\n`;
|
|
1773
|
+
|
|
1774
|
+
if (!workerAgent.prompt) {
|
|
1775
|
+
workerAgent.prompt = instruction;
|
|
1776
|
+
} else if (typeof workerAgent.prompt === 'string') {
|
|
1777
|
+
workerAgent.prompt = instruction + workerAgent.prompt;
|
|
1778
|
+
} else if (workerAgent.prompt.system) {
|
|
1779
|
+
workerAgent.prompt.system = instruction + workerAgent.prompt.system;
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
this._log(`[Orchestrator] Injected parallelization instruction (workers=${workersCount})`);
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
/**
|
|
1786
|
+
* Generate a unique cluster ID, safe for concurrent starts in-process.
|
|
1787
|
+
* If an explicit ID is provided, uses it as a base and suffixes on collision.
|
|
1788
|
+
* @private
|
|
1789
|
+
*/
|
|
1790
|
+
_generateUniqueClusterId(explicitId, explicitDbPath) {
|
|
1791
|
+
const baseId = explicitId || generateName('cluster');
|
|
1792
|
+
const baseDbPath = explicitDbPath || path.join(this.storageDir, `${baseId}.db`);
|
|
1793
|
+
|
|
1794
|
+
// Fast path: base is unused.
|
|
1795
|
+
if (!this.clusters.has(baseId) && !fs.existsSync(baseDbPath)) {
|
|
1796
|
+
return baseId;
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
// Collision: suffix with random bytes to avoid race conditions under concurrency.
|
|
1800
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1801
|
+
const suffix = crypto.randomBytes(3).toString('hex');
|
|
1802
|
+
const candidateId = `${baseId}-${suffix}`;
|
|
1803
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1804
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1805
|
+
return candidateId;
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
// Last resort: new generated name (should never happen).
|
|
1810
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
1811
|
+
const candidateId = generateName('cluster');
|
|
1812
|
+
const candidateDbPath = explicitDbPath || path.join(this.storageDir, `${candidateId}.db`);
|
|
1813
|
+
if (!this.clusters.has(candidateId) && !fs.existsSync(candidateDbPath)) {
|
|
1814
|
+
return candidateId;
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
|
|
1818
|
+
throw new Error(`Failed to generate unique cluster ID in ${this.storageDir} after 50 attempts`);
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
/**
|
|
1822
|
+
* Wait for a process to exit
|
|
1823
|
+
* @param {Number} pid - Process ID
|
|
1824
|
+
* @param {Number} timeoutMs - Timeout in milliseconds
|
|
1825
|
+
* @param {Number} intervalMs - Poll interval in milliseconds
|
|
1826
|
+
* @returns {Promise<Boolean>} True if process exited
|
|
1827
|
+
* @private
|
|
1828
|
+
*/
|
|
1829
|
+
async _waitForProcessExit(pid, timeoutMs, intervalMs = 100) {
|
|
1830
|
+
const deadline = Date.now() + timeoutMs;
|
|
1831
|
+
while (Date.now() < deadline) {
|
|
1832
|
+
if (!this._isProcessRunning(pid)) {
|
|
1833
|
+
return true;
|
|
1834
|
+
}
|
|
1835
|
+
await new Promise((resolve) => setTimeout(resolve, intervalMs));
|
|
1836
|
+
}
|
|
1837
|
+
return !this._isProcessRunning(pid);
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
/**
|
|
1841
|
+
* Tear down Docker Compose services in a worktree directory to free host ports.
|
|
1842
|
+
* Best-effort — silently ignores failures (compose may not have been started, Docker may not be running).
|
|
1843
|
+
* @param {string} worktreePath - Path to the worktree directory
|
|
1844
|
+
* @private
|
|
1845
|
+
*/
|
|
1846
|
+
_teardownWorktreeCompose(worktreePath) {
|
|
1847
|
+
const composePath = path.join(worktreePath, 'docker-compose.yml');
|
|
1848
|
+
if (!fs.existsSync(composePath)) return;
|
|
1849
|
+
|
|
1850
|
+
try {
|
|
1851
|
+
this._log(`[Orchestrator] Tearing down Docker Compose services in ${worktreePath}...`);
|
|
1852
|
+
const { spawnSync } = require('child_process');
|
|
1853
|
+
const result = spawnSync(
|
|
1854
|
+
'docker',
|
|
1855
|
+
['compose', 'down', '--remove-orphans', '--volumes', '--timeout', '10'],
|
|
1856
|
+
{
|
|
1857
|
+
cwd: worktreePath,
|
|
1858
|
+
encoding: 'utf8',
|
|
1859
|
+
stdio: 'pipe',
|
|
1860
|
+
timeout: 30000,
|
|
1861
|
+
}
|
|
1862
|
+
);
|
|
1863
|
+
if (result.status !== 0 || result.error) {
|
|
1864
|
+
const detail = result.error?.message || result.stderr || 'no stderr';
|
|
1865
|
+
throw new Error(`Docker Compose teardown failed in ${worktreePath}: ${detail}`);
|
|
1866
|
+
}
|
|
1867
|
+
this._log(`[Orchestrator] Docker Compose services torn down`);
|
|
1868
|
+
} catch {
|
|
1869
|
+
// Best-effort: compose project may not have been started
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
/**
|
|
1874
|
+
* Signal a remote daemon that owns the cluster and wait for exit
|
|
1875
|
+
* @param {Object} cluster - Cluster object
|
|
1876
|
+
* @param {Object} options - { action, timeoutMs, killTimeoutMs, signal }
|
|
1877
|
+
* @returns {Promise<Object>} { handled, remotePid, exited, forced }
|
|
1878
|
+
* @private
|
|
1879
|
+
*/
|
|
1880
|
+
async _signalRemoteCluster(cluster, options) {
|
|
1881
|
+
const remotePid = cluster.pid;
|
|
1882
|
+
if (!remotePid || remotePid === process.pid) {
|
|
1883
|
+
return { handled: false };
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
if (!this._isProcessRunning(remotePid)) {
|
|
1887
|
+
return { handled: false, remotePid, alreadyExited: true };
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
const action = options?.action || 'stop';
|
|
1891
|
+
const timeoutMs = options?.timeoutMs ?? 10000;
|
|
1892
|
+
const killTimeoutMs = options?.killTimeoutMs ?? 5000;
|
|
1893
|
+
const signal = options?.signal || 'SIGTERM';
|
|
1894
|
+
|
|
1895
|
+
this._log(`[Orchestrator] Sending ${signal} to daemon PID ${remotePid} for ${cluster.id}...`);
|
|
1896
|
+
try {
|
|
1897
|
+
process.kill(remotePid, signal);
|
|
1898
|
+
} catch (error) {
|
|
1899
|
+
if (error.code === 'ESRCH') {
|
|
1900
|
+
return { handled: true, remotePid, exited: true };
|
|
1901
|
+
}
|
|
1902
|
+
throw error;
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
const exited = await this._waitForProcessExit(remotePid, timeoutMs);
|
|
1906
|
+
if (exited) {
|
|
1907
|
+
return { handled: true, remotePid, exited: true };
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
if (action !== 'kill') {
|
|
1911
|
+
throw new Error(
|
|
1912
|
+
`Timed out waiting for daemon PID ${remotePid} to stop cluster ${cluster.id}`
|
|
1913
|
+
);
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
this._log(
|
|
1917
|
+
`[Orchestrator] Daemon PID ${remotePid} still running after ${timeoutMs}ms, sending SIGKILL...`
|
|
1918
|
+
);
|
|
1919
|
+
try {
|
|
1920
|
+
process.kill(remotePid, 'SIGKILL');
|
|
1921
|
+
} catch (error) {
|
|
1922
|
+
if (error.code !== 'ESRCH') {
|
|
1923
|
+
throw error;
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
const killed = await this._waitForProcessExit(remotePid, killTimeoutMs);
|
|
1928
|
+
if (!killed) {
|
|
1929
|
+
throw new Error(
|
|
1930
|
+
`Failed to kill daemon PID ${remotePid} for cluster ${cluster.id} after SIGKILL`
|
|
1931
|
+
);
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
return { handled: true, remotePid, exited: true, forced: true };
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
async _signalSetupProcess(clusterId, cluster) {
|
|
1938
|
+
const setupPid = cluster.pid;
|
|
1939
|
+
if (!setupPid || setupPid === process.pid || !this._isProcessRunning(setupPid)) {
|
|
1940
|
+
return { handled: false };
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
const signalTarget = process.platform === 'win32' ? setupPid : -setupPid;
|
|
1944
|
+
this._log(`[Orchestrator] Killing setup daemon PID ${setupPid} for ${clusterId}...`);
|
|
1945
|
+
|
|
1946
|
+
this._sendSetupSignal(signalTarget, setupPid, 'SIGTERM');
|
|
1947
|
+
const exited = await this._waitForProcessExit(setupPid, 5000);
|
|
1948
|
+
if (exited) {
|
|
1949
|
+
return { handled: true, remotePid: setupPid, exited: true };
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
this._log(`[Orchestrator] Setup daemon PID ${setupPid} still running, sending SIGKILL...`);
|
|
1953
|
+
this._sendSetupSignal(signalTarget, setupPid, 'SIGKILL');
|
|
1954
|
+
const killed = await this._waitForProcessExit(setupPid, 5000);
|
|
1955
|
+
if (!killed) {
|
|
1956
|
+
throw new Error(`Failed to kill setup daemon PID ${setupPid} for cluster ${clusterId}`);
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
return { handled: true, remotePid: setupPid, exited: true, forced: true };
|
|
1960
|
+
}
|
|
1961
|
+
|
|
1962
|
+
_sendSetupSignal(signalTarget, fallbackPid, signal) {
|
|
1963
|
+
try {
|
|
1964
|
+
process.kill(signalTarget, signal);
|
|
1965
|
+
return;
|
|
1966
|
+
} catch (error) {
|
|
1967
|
+
if (error.code !== 'ESRCH') {
|
|
1968
|
+
throw error;
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
if (signalTarget === fallbackPid) {
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
try {
|
|
1977
|
+
process.kill(fallbackPid, signal);
|
|
1978
|
+
} catch (error) {
|
|
1979
|
+
if (error.code !== 'ESRCH') {
|
|
1980
|
+
throw error;
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
async _killSetupCluster(clusterId, cluster) {
|
|
1986
|
+
await this._signalSetupProcess(clusterId, cluster);
|
|
1987
|
+
|
|
1988
|
+
if (cluster.worktree?.path) {
|
|
1989
|
+
this._teardownWorktreeCompose(cluster.worktree.path);
|
|
1990
|
+
try {
|
|
1991
|
+
const manager = cluster.worktree.manager || new IsolationManager();
|
|
1992
|
+
manager.worktrees.set(clusterId, {
|
|
1993
|
+
path: cluster.worktree.path,
|
|
1994
|
+
branch: cluster.worktree.branch,
|
|
1995
|
+
repoRoot: cluster.worktree.repoRoot,
|
|
1996
|
+
});
|
|
1997
|
+
manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
|
|
1998
|
+
} catch (error) {
|
|
1999
|
+
this._log(`[Orchestrator] Warning: setup worktree cleanup failed: ${error.message}`);
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
cluster.state = 'killed';
|
|
2004
|
+
cluster.pid = null;
|
|
2005
|
+
await this._saveClusters();
|
|
2006
|
+
this.clusters.delete(clusterId);
|
|
2007
|
+
this._log(`Setup cluster ${clusterId} killed`);
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
/**
|
|
2011
|
+
* Stop a cluster
|
|
2012
|
+
* @param {String} clusterId - Cluster ID
|
|
2013
|
+
* @param {Object} [options] - Stop options
|
|
2014
|
+
* @param {boolean} [options.completedSuccessfully=false] - Whether cluster completed successfully (vs user-initiated stop)
|
|
2015
|
+
*/
|
|
2016
|
+
async stop(clusterId, options = {}) {
|
|
2017
|
+
const cluster = this.clusters.get(clusterId);
|
|
2018
|
+
if (!cluster) {
|
|
2019
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
await this._signalRemoteCluster(cluster, { action: 'stop' });
|
|
2023
|
+
|
|
2024
|
+
// CRITICAL: Wait for initialization to complete before stopping
|
|
2025
|
+
// This ensures ISSUE_OPENED is published, preventing 0-message clusters
|
|
2026
|
+
// Timeout after 30s to prevent infinite hang if init truly fails
|
|
2027
|
+
if (cluster.initCompletePromise && cluster.state === 'initializing') {
|
|
2028
|
+
this._log(`[Orchestrator] Waiting for initialization to complete before stopping...`);
|
|
2029
|
+
await Promise.race([
|
|
2030
|
+
cluster.initCompletePromise,
|
|
2031
|
+
new Promise((resolve) => setTimeout(resolve, 30000)),
|
|
2032
|
+
]);
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
cluster.state = 'stopping';
|
|
2036
|
+
|
|
2037
|
+
// Stop all agents (including subclusters which handle their own children)
|
|
2038
|
+
for (const agent of cluster.agents) {
|
|
2039
|
+
await agent.stop();
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
if (cluster.snapshotter) {
|
|
2043
|
+
cluster.snapshotter.stop();
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
// Clean up isolation container if enabled
|
|
2047
|
+
// CRITICAL: Preserve workspace for resume capability - only delete on kill()
|
|
2048
|
+
if (cluster.isolation?.manager) {
|
|
2049
|
+
this._log(
|
|
2050
|
+
`[Orchestrator] Stopping isolation container for ${clusterId} (preserving workspace for resume)...`
|
|
2051
|
+
);
|
|
2052
|
+
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: true });
|
|
2053
|
+
this._log(`[Orchestrator] Container stopped, workspace preserved`);
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
if (cluster.validatorIsolation?.manager) {
|
|
2057
|
+
this._log(`[Orchestrator] Cleaning up validator isolation container for ${clusterId}...`);
|
|
2058
|
+
await cluster.validatorIsolation.manager.cleanup(cluster.validatorIsolation.clusterId, {
|
|
2059
|
+
preserveWorkspace: false,
|
|
2060
|
+
});
|
|
2061
|
+
cluster.validatorIsolation = null;
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
// Worktree cleanup: auto-remove if completed successfully with --pr/--ship
|
|
2065
|
+
// (no reason to resume after PR is created/merged), otherwise preserve for resume
|
|
2066
|
+
const shouldAutoCleanWorktree =
|
|
2067
|
+
options.completedSuccessfully && cluster.autoPr && cluster.worktree?.manager;
|
|
2068
|
+
|
|
2069
|
+
if (shouldAutoCleanWorktree) {
|
|
2070
|
+
this._log(`[Orchestrator] Auto-cleaning worktree (completed successfully with --pr/--ship)`);
|
|
2071
|
+
// Tear down Docker Compose first, then remove worktree entirely
|
|
2072
|
+
if (cluster.worktree.path) {
|
|
2073
|
+
this._teardownWorktreeCompose(cluster.worktree.path);
|
|
2074
|
+
}
|
|
2075
|
+
try {
|
|
2076
|
+
cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
|
|
2077
|
+
this._log(`[Orchestrator] Worktree removed, branch ${cluster.worktree.branch} preserved`);
|
|
2078
|
+
} catch (err) {
|
|
2079
|
+
// Best-effort cleanup — don't fail the stop operation
|
|
2080
|
+
this._log(`[Orchestrator] Warning: worktree cleanup failed: ${err.message}`);
|
|
2081
|
+
}
|
|
2082
|
+
} else if (cluster.worktree?.manager) {
|
|
2083
|
+
this._log(`[Orchestrator] Worktree preserved at ${cluster.worktree.path} for resume`);
|
|
2084
|
+
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
2085
|
+
// Hook daemons run detached inside the worktree and survive agent shutdown
|
|
2086
|
+
// unless we explicitly reap worktree-scoped processes on stop.
|
|
2087
|
+
if (cluster.worktree.path) {
|
|
2088
|
+
cluster.worktree.manager.cleanupWorktreeProcesses(cluster.worktree.path);
|
|
2089
|
+
}
|
|
2090
|
+
// Tear down Docker Compose services in the worktree to free host ports.
|
|
2091
|
+
// Without this, stopped worktrees hold ports (5433, 6379, etc.) indefinitely.
|
|
2092
|
+
if (cluster.worktree.path) {
|
|
2093
|
+
this._teardownWorktreeCompose(cluster.worktree.path);
|
|
2094
|
+
}
|
|
2095
|
+
// Don't cleanup worktree itself - it will be reused on resume
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
cluster.state = shouldAutoCleanWorktree ? 'killed' : 'stopped';
|
|
2099
|
+
cluster.pid = null; // Clear PID - cluster is no longer running
|
|
2100
|
+
|
|
2101
|
+
if (shouldAutoCleanWorktree) {
|
|
2102
|
+
// Close message bus and ledger (same as kill() path)
|
|
2103
|
+
cluster.messageBus.close();
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
this._log(`Cluster ${clusterId} ${cluster.state}`);
|
|
2107
|
+
|
|
2108
|
+
// Save updated state
|
|
2109
|
+
// Note: 'killed' state causes _saveClusters to DELETE from disk (no stale entries)
|
|
2110
|
+
await this._saveClusters();
|
|
2111
|
+
|
|
2112
|
+
if (shouldAutoCleanWorktree) {
|
|
2113
|
+
// Remove from memory after persisting (same as kill() path)
|
|
2114
|
+
this.clusters.delete(clusterId);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
/**
|
|
2119
|
+
* Kill a cluster (force stop)
|
|
2120
|
+
* @param {String} clusterId - Cluster ID
|
|
2121
|
+
*/
|
|
2122
|
+
async kill(clusterId) {
|
|
2123
|
+
const cluster = this.clusters.get(clusterId);
|
|
2124
|
+
if (!cluster) {
|
|
2125
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
if (this._isSetupCluster(cluster)) {
|
|
2129
|
+
await this._killSetupCluster(clusterId, cluster);
|
|
2130
|
+
return;
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
await this._signalRemoteCluster(cluster, { action: 'kill' });
|
|
2134
|
+
|
|
2135
|
+
cluster.state = 'stopping';
|
|
2136
|
+
|
|
2137
|
+
// Force stop all agents
|
|
2138
|
+
for (const agent of cluster.agents) {
|
|
2139
|
+
await agent.stop();
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
if (cluster.snapshotter) {
|
|
2143
|
+
cluster.snapshotter.stop();
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
// Force remove isolation container AND workspace (full cleanup, no resume)
|
|
2147
|
+
if (cluster.isolation?.manager) {
|
|
2148
|
+
this._log(
|
|
2149
|
+
`[Orchestrator] Force removing isolation container and workspace for ${clusterId}...`
|
|
2150
|
+
);
|
|
2151
|
+
await cluster.isolation.manager.cleanup(clusterId, { preserveWorkspace: false });
|
|
2152
|
+
this._log(`[Orchestrator] Container and workspace removed`);
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
if (cluster.validatorIsolation?.manager) {
|
|
2156
|
+
this._log(`[Orchestrator] Force removing validator isolation container for ${clusterId}...`);
|
|
2157
|
+
await cluster.validatorIsolation.manager.cleanup(cluster.validatorIsolation.clusterId, {
|
|
2158
|
+
preserveWorkspace: false,
|
|
2159
|
+
});
|
|
2160
|
+
cluster.validatorIsolation = null;
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
// Force remove worktree (full cleanup, no resume)
|
|
2164
|
+
// Note: Branch is preserved for potential PR creation / inspection
|
|
2165
|
+
if (cluster.worktree?.manager) {
|
|
2166
|
+
this._log(`[Orchestrator] Force removing worktree for ${clusterId}...`);
|
|
2167
|
+
cluster.worktree.manager.cleanupWorktreeIsolation(clusterId, { preserveBranch: true });
|
|
2168
|
+
this._log(`[Orchestrator] Worktree removed, branch ${cluster.worktree.branch} preserved`);
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
// Close message bus and ledger
|
|
2172
|
+
cluster.messageBus.close();
|
|
2173
|
+
|
|
2174
|
+
cluster.state = 'killed';
|
|
2175
|
+
cluster.pid = null; // Clear PID - cluster is no longer running
|
|
2176
|
+
// DON'T delete from memory - keep it so it gets saved with 'killed' state
|
|
2177
|
+
// this.clusters.delete(clusterId);
|
|
2178
|
+
|
|
2179
|
+
this._log(`Cluster ${clusterId} killed`);
|
|
2180
|
+
|
|
2181
|
+
// Save updated state (will be marked as 'killed' in file)
|
|
2182
|
+
await this._saveClusters();
|
|
2183
|
+
|
|
2184
|
+
// Now remove from memory after persisting
|
|
2185
|
+
this.clusters.delete(clusterId);
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
/**
|
|
2189
|
+
* Kill all running clusters
|
|
2190
|
+
* @returns {Object} { killed: Array<string>, errors: Array<{id, error}> }
|
|
2191
|
+
*/
|
|
2192
|
+
async killAll() {
|
|
2193
|
+
const results = { killed: [], errors: [] };
|
|
2194
|
+
const runningClusters = Array.from(this.clusters.values()).filter(
|
|
2195
|
+
(c) => c.state === 'running' || c.state === 'initializing' || this._isSetupCluster(c)
|
|
2196
|
+
);
|
|
2197
|
+
|
|
2198
|
+
for (const cluster of runningClusters) {
|
|
2199
|
+
try {
|
|
2200
|
+
await this.kill(cluster.id);
|
|
2201
|
+
results.killed.push(cluster.id);
|
|
2202
|
+
} catch (error) {
|
|
2203
|
+
results.errors.push({ id: cluster.id, error: error.message });
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
return results;
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
/**
|
|
2211
|
+
* Close the orchestrator (prevents further _saveClusters operations)
|
|
2212
|
+
* Call before deleting storageDir to prevent ENOENT race conditions during cleanup
|
|
2213
|
+
*/
|
|
2214
|
+
close() {
|
|
2215
|
+
this.closed = true;
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
/**
|
|
2219
|
+
* Find the last workflow-triggering message in the ledger
|
|
2220
|
+
* Workflow triggers indicate cluster state progression (not AGENT_OUTPUT noise)
|
|
2221
|
+
* @param {Array} messages - All messages from ledger
|
|
2222
|
+
* @returns {Object|null} - Last workflow trigger message or null
|
|
2223
|
+
* @private
|
|
2224
|
+
*/
|
|
2225
|
+
_findLastWorkflowTrigger(messages) {
|
|
2226
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
2227
|
+
if (WORKFLOW_TRIGGERS.includes(messages[i].topic)) {
|
|
2228
|
+
return messages[i];
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
return null;
|
|
2232
|
+
}
|
|
2233
|
+
|
|
2234
|
+
/**
|
|
2235
|
+
* Resume a stopped cluster from where it left off
|
|
2236
|
+
* Handles both failed clusters (with error context) and cleanly stopped clusters
|
|
2237
|
+
* @param {String} clusterId - Cluster ID
|
|
2238
|
+
* @param {String} prompt - Optional custom resume prompt
|
|
2239
|
+
* @returns {Object} Resumed cluster info
|
|
2240
|
+
*/
|
|
2241
|
+
async resume(clusterId, prompt) {
|
|
2242
|
+
const cluster = this.clusters.get(clusterId);
|
|
2243
|
+
if (!cluster) {
|
|
2244
|
+
throw new Error(`Cluster not found: ${clusterId}`);
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
if (cluster.state === 'running') {
|
|
2248
|
+
throw new Error(
|
|
2249
|
+
`Cluster ${clusterId} is still running. Use 'zeroshot stop' first if you want to restart it.`
|
|
2250
|
+
);
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
if (this._isSetupCluster(cluster)) {
|
|
2254
|
+
const setupError = cluster.failureInfo?.error
|
|
2255
|
+
? ` Previous setup error: ${cluster.failureInfo.error}.`
|
|
2256
|
+
: '';
|
|
2257
|
+
const logHint = cluster.setupLogPath ? ` Check setup log: ${cluster.setupLogPath}.` : '';
|
|
2258
|
+
throw new Error(
|
|
2259
|
+
`Cluster ${clusterId} never finished setup and cannot be resumed.` +
|
|
2260
|
+
setupError +
|
|
2261
|
+
` Use 'zeroshot run' to start a fresh cluster.` +
|
|
2262
|
+
logHint
|
|
2263
|
+
);
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
const failureInfo = this._resolveFailureInfo(cluster, clusterId);
|
|
2267
|
+
|
|
2268
|
+
await this._ensureIsolationForResume(clusterId, cluster);
|
|
2269
|
+
this._ensureWorktreeForResume(clusterId, cluster);
|
|
2270
|
+
this._startSnapshotter(cluster);
|
|
2271
|
+
await this._restartClusterAgents(cluster);
|
|
2272
|
+
|
|
2273
|
+
const recentMessages = this._loadRecentMessages(cluster, clusterId, 50);
|
|
2274
|
+
|
|
2275
|
+
if (failureInfo) {
|
|
2276
|
+
return this._resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt);
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
return this._resumeCleanCluster(clusterId, cluster, recentMessages, prompt);
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
_validateGuidanceAgentArgs(clusterId, agentId, text) {
|
|
2283
|
+
if (!clusterId) {
|
|
2284
|
+
throw new Error('sendGuidanceToAgent: clusterId is required');
|
|
2285
|
+
}
|
|
2286
|
+
if (!agentId) {
|
|
2287
|
+
throw new Error('sendGuidanceToAgent: agentId is required');
|
|
2288
|
+
}
|
|
2289
|
+
if (typeof text !== 'string' || !text.trim()) {
|
|
2290
|
+
throw new Error('sendGuidanceToAgent: text must be a non-empty string');
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
_validateGuidanceClusterArgs(clusterId, text) {
|
|
2295
|
+
if (!clusterId) {
|
|
2296
|
+
throw new Error('sendGuidanceToCluster: clusterId is required');
|
|
2297
|
+
}
|
|
2298
|
+
if (typeof text !== 'string' || !text.trim()) {
|
|
2299
|
+
throw new Error('sendGuidanceToCluster: text must be a non-empty string');
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
_getClusterOrThrow(clusterId, caller = 'sendGuidanceToAgent') {
|
|
2304
|
+
const cluster = this.clusters.get(clusterId);
|
|
2305
|
+
if (!cluster) {
|
|
2306
|
+
throw new Error(`${caller}: cluster not found: ${clusterId}`);
|
|
2307
|
+
}
|
|
2308
|
+
return cluster;
|
|
2309
|
+
}
|
|
2310
|
+
|
|
2311
|
+
_getAgentOrThrow(cluster, agentId) {
|
|
2312
|
+
const agent = cluster.agents.find((candidate) => candidate.id === agentId);
|
|
2313
|
+
if (!agent) {
|
|
2314
|
+
throw new Error(`sendGuidanceToAgent: agent not found: ${agentId}`);
|
|
2315
|
+
}
|
|
2316
|
+
return agent;
|
|
2317
|
+
}
|
|
2318
|
+
|
|
2319
|
+
_buildDefaultDelivery(agent) {
|
|
2320
|
+
return {
|
|
2321
|
+
status: 'unsupported',
|
|
2322
|
+
reason: 'unknown',
|
|
2323
|
+
method: null,
|
|
2324
|
+
taskId: agent.currentTaskId || null,
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
async _attemptGuidanceInjection(agent, text, timeoutMs) {
|
|
2329
|
+
try {
|
|
2330
|
+
const result = await agent.injectInput(text, { timeoutMs });
|
|
2331
|
+
return {
|
|
2332
|
+
status: result.status === 'injected' ? 'injected' : 'unsupported',
|
|
2333
|
+
reason: result.status === 'injected' ? null : result.reason || 'unsupported',
|
|
2334
|
+
method: result.status === 'injected' ? result.method || 'pty' : result.method || null,
|
|
2335
|
+
taskId: result.taskId || agent.currentTaskId || null,
|
|
2336
|
+
};
|
|
2337
|
+
} catch (error) {
|
|
2338
|
+
return {
|
|
2339
|
+
status: 'unsupported',
|
|
2340
|
+
reason: error.message,
|
|
2341
|
+
method: null,
|
|
2342
|
+
taskId: agent.currentTaskId || null,
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
_buildGuidanceMetadata(options, delivery) {
|
|
2348
|
+
return {
|
|
2349
|
+
...(options.metadata || {}),
|
|
2350
|
+
delivery: {
|
|
2351
|
+
status: delivery.status,
|
|
2352
|
+
reason: delivery.reason,
|
|
2353
|
+
method: delivery.method,
|
|
2354
|
+
taskId: delivery.taskId,
|
|
2355
|
+
timestamp: Date.now(),
|
|
2356
|
+
},
|
|
2357
|
+
};
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
_summarizeGuidanceDeliveries(deliveries) {
|
|
2361
|
+
const agentIds = Object.keys(deliveries);
|
|
2362
|
+
const summary = {
|
|
2363
|
+
injected: 0,
|
|
2364
|
+
queued: 0,
|
|
2365
|
+
total: agentIds.length,
|
|
2366
|
+
};
|
|
2367
|
+
|
|
2368
|
+
for (const agentId of agentIds) {
|
|
2369
|
+
const delivery = deliveries[agentId];
|
|
2370
|
+
if (delivery?.status === 'injected') {
|
|
2371
|
+
summary.injected += 1;
|
|
2372
|
+
} else {
|
|
2373
|
+
summary.queued += 1;
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
return summary;
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
_buildClusterGuidanceMetadata(options, deliveries) {
|
|
2381
|
+
return {
|
|
2382
|
+
...(options.metadata || {}),
|
|
2383
|
+
delivery: {
|
|
2384
|
+
summary: this._summarizeGuidanceDeliveries(deliveries),
|
|
2385
|
+
agents: deliveries,
|
|
2386
|
+
timestamp: Date.now(),
|
|
2387
|
+
},
|
|
2388
|
+
};
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
_publishGuidance(cluster, clusterId, agentId, text, metadata, sender) {
|
|
2392
|
+
cluster.messageBus.publish({
|
|
2393
|
+
cluster_id: clusterId,
|
|
2394
|
+
topic: USER_GUIDANCE_AGENT,
|
|
2395
|
+
sender,
|
|
2396
|
+
target_agent_id: agentId,
|
|
2397
|
+
content: { text },
|
|
2398
|
+
metadata,
|
|
2399
|
+
});
|
|
2400
|
+
}
|
|
2401
|
+
|
|
2402
|
+
_publishClusterGuidance(cluster, clusterId, text, metadata, sender) {
|
|
2403
|
+
cluster.messageBus.publish({
|
|
2404
|
+
cluster_id: clusterId,
|
|
2405
|
+
topic: USER_GUIDANCE_CLUSTER,
|
|
2406
|
+
sender,
|
|
2407
|
+
content: { text },
|
|
2408
|
+
metadata,
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
/**
|
|
2413
|
+
* Send guidance to a specific agent with optional live injection.
|
|
2414
|
+
* Always persists USER_GUIDANCE_AGENT in the ledger with delivery metadata.
|
|
2415
|
+
* @param {string} clusterId
|
|
2416
|
+
* @param {string} agentId
|
|
2417
|
+
* @param {string} text
|
|
2418
|
+
* @param {object} [options]
|
|
2419
|
+
* @param {string} [options.sender='user']
|
|
2420
|
+
* @param {object} [options.metadata]
|
|
2421
|
+
* @param {number} [options.timeoutMs]
|
|
2422
|
+
* @returns {Promise<{status: string, reason: string|null, method: string|null, taskId: string|null}>}
|
|
2423
|
+
*/
|
|
2424
|
+
async sendGuidanceToAgent(clusterId, agentId, text, options = {}) {
|
|
2425
|
+
this._validateGuidanceAgentArgs(clusterId, agentId, text);
|
|
2426
|
+
|
|
2427
|
+
const cluster = this._getClusterOrThrow(clusterId, 'sendGuidanceToAgent');
|
|
2428
|
+
const agent = this._getAgentOrThrow(cluster, agentId);
|
|
2429
|
+
const defaultDelivery = this._buildDefaultDelivery(agent);
|
|
2430
|
+
const delivery =
|
|
2431
|
+
(await this._attemptGuidanceInjection(agent, text, options.timeoutMs)) || defaultDelivery;
|
|
2432
|
+
const metadata = this._buildGuidanceMetadata(options, delivery);
|
|
2433
|
+
|
|
2434
|
+
this._publishGuidance(cluster, clusterId, agentId, text, metadata, options.sender || 'user');
|
|
2435
|
+
|
|
2436
|
+
return delivery;
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
/**
|
|
2440
|
+
* Send guidance to every agent in a cluster with optional live injection.
|
|
2441
|
+
* Always persists a single USER_GUIDANCE_CLUSTER in the ledger with delivery metadata.
|
|
2442
|
+
* @param {string} clusterId
|
|
2443
|
+
* @param {string} text
|
|
2444
|
+
* @param {object} [options]
|
|
2445
|
+
* @param {string} [options.sender='user']
|
|
2446
|
+
* @param {object} [options.metadata]
|
|
2447
|
+
* @param {number} [options.timeoutMs]
|
|
2448
|
+
* @returns {Promise<{summary: {injected: number, queued: number, total: number}, agents: Record<string, {status: string, reason: string|null, method: string|null, taskId: string|null}>, timestamp: number}>}
|
|
2449
|
+
*/
|
|
2450
|
+
async sendGuidanceToCluster(clusterId, text, options = {}) {
|
|
2451
|
+
this._validateGuidanceClusterArgs(clusterId, text);
|
|
2452
|
+
|
|
2453
|
+
const cluster = this._getClusterOrThrow(clusterId, 'sendGuidanceToCluster');
|
|
2454
|
+
const agents = Array.isArray(cluster.agents) ? cluster.agents : [];
|
|
2455
|
+
const deliveries = {};
|
|
2456
|
+
|
|
2457
|
+
await Promise.all(
|
|
2458
|
+
agents.map(async (agent) => {
|
|
2459
|
+
const defaultDelivery = this._buildDefaultDelivery(agent);
|
|
2460
|
+
const delivery =
|
|
2461
|
+
(await this._attemptGuidanceInjection(agent, text, options.timeoutMs)) || defaultDelivery;
|
|
2462
|
+
deliveries[agent.id] = delivery;
|
|
2463
|
+
})
|
|
2464
|
+
);
|
|
2465
|
+
|
|
2466
|
+
const metadata = this._buildClusterGuidanceMetadata(options, deliveries);
|
|
2467
|
+
|
|
2468
|
+
this._publishClusterGuidance(cluster, clusterId, text, metadata, options.sender || 'user');
|
|
2469
|
+
|
|
2470
|
+
return metadata.delivery;
|
|
2471
|
+
}
|
|
2472
|
+
|
|
2473
|
+
_resolveFailureInfo(cluster, clusterId) {
|
|
2474
|
+
if (cluster.failureInfo) {
|
|
2475
|
+
return cluster.failureInfo;
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
const errors = cluster.messageBus.query({
|
|
2479
|
+
cluster_id: clusterId,
|
|
2480
|
+
topic: 'AGENT_ERROR',
|
|
2481
|
+
limit: 10,
|
|
2482
|
+
order: 'desc',
|
|
2483
|
+
});
|
|
2484
|
+
|
|
2485
|
+
if (errors.length === 0) {
|
|
2486
|
+
return null;
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
const firstError = errors[0];
|
|
2490
|
+
const failureInfo = {
|
|
2491
|
+
agentId: firstError.sender,
|
|
2492
|
+
taskId: firstError.content?.data?.taskId || null,
|
|
2493
|
+
iteration: firstError.content?.data?.iteration || 0,
|
|
2494
|
+
error: firstError.content?.data?.error || firstError.content?.text,
|
|
2495
|
+
timestamp: firstError.timestamp,
|
|
2496
|
+
};
|
|
2497
|
+
this._log(`[Orchestrator] Found failure from ledger: ${failureInfo.agentId}`);
|
|
2498
|
+
return failureInfo;
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
_checkContainerExists(containerId) {
|
|
2502
|
+
const { spawn } = require('child_process');
|
|
2503
|
+
const checkContainer = spawn('docker', ['inspect', containerId], { stdio: 'ignore' });
|
|
2504
|
+
return new Promise((resolve) => {
|
|
2505
|
+
checkContainer.on('close', (code) => resolve(code === 0));
|
|
2506
|
+
});
|
|
2507
|
+
}
|
|
2508
|
+
|
|
2509
|
+
async _ensureIsolationForResume(clusterId, cluster) {
|
|
2510
|
+
if (!cluster.isolation?.enabled) {
|
|
2511
|
+
return;
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
const oldContainerId = cluster.isolation.containerId;
|
|
2515
|
+
const containerExists = await this._checkContainerExists(oldContainerId);
|
|
2516
|
+
if (containerExists) {
|
|
2517
|
+
this._log(`[Orchestrator] Container ${oldContainerId} still exists, reusing`);
|
|
2518
|
+
return;
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2521
|
+
this._log(`[Orchestrator] Container ${oldContainerId} not found, recreating...`);
|
|
2522
|
+
|
|
2523
|
+
const workDir = cluster.isolation.workDir;
|
|
2524
|
+
if (!workDir) {
|
|
2525
|
+
throw new Error(`Cannot resume cluster ${clusterId}: workDir not saved in isolation state`);
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
const isolatedPath = path.join(os.tmpdir(), 'zeroshot-isolated', clusterId);
|
|
2529
|
+
if (!fs.existsSync(isolatedPath)) {
|
|
2530
|
+
throw new Error(
|
|
2531
|
+
`Cannot resume cluster ${clusterId}: isolated workspace deleted. ` +
|
|
2532
|
+
`Was the cluster killed (not stopped)? Use 'zeroshot run' to start fresh.`
|
|
2533
|
+
);
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
const providerName = this._resolveClusterProvider(cluster.config);
|
|
2537
|
+
const newContainerId = await cluster.isolation.manager.createContainer(clusterId, {
|
|
2538
|
+
workDir,
|
|
2539
|
+
image: cluster.isolation.image,
|
|
2540
|
+
reuseExistingWorkspace: true,
|
|
2541
|
+
provider: providerName,
|
|
2542
|
+
});
|
|
2543
|
+
|
|
2544
|
+
this._log(`[Orchestrator] New container created: ${newContainerId}`);
|
|
2545
|
+
cluster.isolation.containerId = newContainerId;
|
|
2546
|
+
|
|
2547
|
+
for (const agent of cluster.agents) {
|
|
2548
|
+
if (agent.isolation?.enabled) {
|
|
2549
|
+
agent.isolation.containerId = newContainerId;
|
|
2550
|
+
agent.isolation.manager = cluster.isolation.manager;
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
|
|
2554
|
+
this._log(`[Orchestrator] All agents updated with new container ID`);
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2557
|
+
_ensureWorktreeForResume(clusterId, cluster) {
|
|
2558
|
+
if (!cluster.worktree?.enabled) {
|
|
2559
|
+
return;
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2562
|
+
const worktreePath = cluster.worktree.path;
|
|
2563
|
+
if (!fs.existsSync(worktreePath)) {
|
|
2564
|
+
throw new Error(
|
|
2565
|
+
`Cannot resume cluster ${clusterId}: worktree at ${worktreePath} no longer exists. ` +
|
|
2566
|
+
`Was the worktree manually removed? Use 'zeroshot run' to start fresh.`
|
|
2567
|
+
);
|
|
2568
|
+
}
|
|
2569
|
+
|
|
2570
|
+
this._log(`[Orchestrator] Worktree at ${worktreePath} exists, reusing`);
|
|
2571
|
+
this._log(`[Orchestrator] Branch: ${cluster.worktree.branch}`);
|
|
2572
|
+
}
|
|
2573
|
+
|
|
2574
|
+
async _restartClusterAgents(cluster) {
|
|
2575
|
+
cluster.state = 'running';
|
|
2576
|
+
cluster.pid = process.pid;
|
|
2577
|
+
for (const agent of cluster.agents) {
|
|
2578
|
+
if (!agent.running) {
|
|
2579
|
+
await agent.start();
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2584
|
+
_loadRecentMessages(cluster, clusterId, limit) {
|
|
2585
|
+
return cluster.messageBus
|
|
2586
|
+
.query({
|
|
2587
|
+
cluster_id: clusterId,
|
|
2588
|
+
limit,
|
|
2589
|
+
order: 'desc',
|
|
2590
|
+
})
|
|
2591
|
+
.reverse();
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
_buildResumeContext(recentMessages, prompt, options) {
|
|
2595
|
+
const resumePrompt = prompt || 'Continue from where you left off. Complete the task.';
|
|
2596
|
+
let context = `${options.header}\n\n`;
|
|
2597
|
+
|
|
2598
|
+
if (options.error) {
|
|
2599
|
+
context += `Previous error: ${options.error}\n\n`;
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
context += `## Recent Context\n\n`;
|
|
2603
|
+
for (const msg of recentMessages.slice(-10)) {
|
|
2604
|
+
if (options.topics.includes(msg.topic)) {
|
|
2605
|
+
context += `[${msg.sender}] ${msg.content?.text?.slice(0, 200) || ''}\n`;
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
|
|
2609
|
+
context += `\n## Resume Instructions\n\n${resumePrompt}\n`;
|
|
2610
|
+
return context;
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
_triggerMatchesTopic(triggerTopic, topic) {
|
|
2614
|
+
if (triggerTopic === topic) return true;
|
|
2615
|
+
if (triggerTopic === '*') return true;
|
|
2616
|
+
if (triggerTopic.endsWith('*')) {
|
|
2617
|
+
const prefix = triggerTopic.slice(0, -1);
|
|
2618
|
+
return topic.startsWith(prefix);
|
|
2619
|
+
}
|
|
2620
|
+
return false;
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
_selectAgentsToResume(cluster, lastTrigger) {
|
|
2624
|
+
const agentsToResume = [];
|
|
2625
|
+
|
|
2626
|
+
for (const agent of cluster.agents) {
|
|
2627
|
+
if (!agent.config.triggers) continue;
|
|
2628
|
+
|
|
2629
|
+
const matchingTrigger = agent.config.triggers.find((trigger) =>
|
|
2630
|
+
this._triggerMatchesTopic(trigger.topic, lastTrigger.topic)
|
|
2631
|
+
);
|
|
2632
|
+
if (!matchingTrigger) continue;
|
|
2633
|
+
|
|
2634
|
+
if (matchingTrigger.logic?.script) {
|
|
2635
|
+
const shouldTrigger = agent._evaluateTrigger(matchingTrigger, lastTrigger);
|
|
2636
|
+
if (!shouldTrigger) continue;
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
agentsToResume.push({ agent, message: lastTrigger, trigger: matchingTrigger });
|
|
2640
|
+
}
|
|
2641
|
+
|
|
2642
|
+
return agentsToResume;
|
|
2643
|
+
}
|
|
2644
|
+
|
|
2645
|
+
_resumeAgents(agentsToResume, context) {
|
|
2646
|
+
for (const { agent, message } of agentsToResume) {
|
|
2647
|
+
this._log(`[Orchestrator] - Resuming agent ${agent.id} (triggered by ${message.topic})`);
|
|
2648
|
+
agent.resume(context).catch((err) => {
|
|
2649
|
+
console.error(`[Orchestrator] Resume failed for agent ${agent.id}:`, err.message);
|
|
2650
|
+
});
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
_republishIssue(cluster, clusterId, recentMessages) {
|
|
2655
|
+
const issueMessage = recentMessages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
2656
|
+
if (!issueMessage) {
|
|
2657
|
+
return false;
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
cluster.messageBus.publish({
|
|
2661
|
+
cluster_id: clusterId,
|
|
2662
|
+
topic: 'ISSUE_OPENED',
|
|
2663
|
+
sender: 'system',
|
|
2664
|
+
receiver: 'broadcast',
|
|
2665
|
+
content: issueMessage.content,
|
|
2666
|
+
metadata: { _resumed: true, _originalId: issueMessage.id },
|
|
2667
|
+
});
|
|
2668
|
+
|
|
2669
|
+
return true;
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
async _resumeFailedCluster(clusterId, cluster, failureInfo, recentMessages, prompt) {
|
|
2673
|
+
const { agentId, iteration, error } = failureInfo;
|
|
2674
|
+
this._log(
|
|
2675
|
+
`[Orchestrator] Resuming failed cluster ${clusterId} from agent ${agentId} iteration ${iteration}`
|
|
2676
|
+
);
|
|
2677
|
+
this._log(`[Orchestrator] Previous error: ${error}`);
|
|
2678
|
+
|
|
2679
|
+
const failedAgent = cluster.agents.find((agent) => agent.id === agentId);
|
|
2680
|
+
if (!failedAgent) {
|
|
2681
|
+
throw new Error(`Failed agent '${agentId}' not found in cluster`);
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
2685
|
+
header: 'You are resuming from a previous failed attempt.',
|
|
2686
|
+
error,
|
|
2687
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT'],
|
|
2688
|
+
});
|
|
2689
|
+
|
|
2690
|
+
cluster.failureInfo = null;
|
|
2691
|
+
await this._saveClusters();
|
|
2692
|
+
|
|
2693
|
+
failedAgent.resume(context).catch((err) => {
|
|
2694
|
+
console.error(`[Orchestrator] Resume failed for agent ${agentId}:`, err.message);
|
|
2695
|
+
});
|
|
2696
|
+
|
|
2697
|
+
this._log(`[Orchestrator] Cluster ${clusterId} resumed from failure`);
|
|
2698
|
+
|
|
2699
|
+
return {
|
|
2700
|
+
id: clusterId,
|
|
2701
|
+
state: cluster.state,
|
|
2702
|
+
resumeType: 'failure',
|
|
2703
|
+
resumedAgent: agentId,
|
|
2704
|
+
previousError: error,
|
|
2705
|
+
};
|
|
2706
|
+
}
|
|
2707
|
+
|
|
2708
|
+
async _resumeCleanCluster(clusterId, cluster, recentMessages, prompt) {
|
|
2709
|
+
this._log(`[Orchestrator] Resuming stopped cluster ${clusterId} (no failure)`);
|
|
2710
|
+
|
|
2711
|
+
const context = this._buildResumeContext(recentMessages, prompt, {
|
|
2712
|
+
header: 'Resuming cluster from previous session.',
|
|
2713
|
+
topics: ['AGENT_OUTPUT', 'VALIDATION_RESULT', 'ISSUE_OPENED'],
|
|
2714
|
+
});
|
|
2715
|
+
|
|
2716
|
+
const lastTrigger = this._findLastWorkflowTrigger(recentMessages);
|
|
2717
|
+
if (lastTrigger) {
|
|
2718
|
+
this._log(
|
|
2719
|
+
`[Orchestrator] Last workflow trigger: ${lastTrigger.topic} (${new Date(lastTrigger.timestamp).toISOString()})`
|
|
2720
|
+
);
|
|
2721
|
+
} else {
|
|
2722
|
+
this._log(`[Orchestrator] No workflow triggers found in ledger`);
|
|
2723
|
+
}
|
|
2724
|
+
|
|
2725
|
+
const agentsToResume = lastTrigger ? this._selectAgentsToResume(cluster, lastTrigger) : [];
|
|
2726
|
+
if (agentsToResume.length === 0) {
|
|
2727
|
+
if (!lastTrigger) {
|
|
2728
|
+
this._log(
|
|
2729
|
+
`[Orchestrator] WARNING: No workflow triggers in ledger. Cluster may not have started properly.`
|
|
2730
|
+
);
|
|
2731
|
+
this._log(`[Orchestrator] Publishing ISSUE_OPENED to bootstrap workflow...`);
|
|
2732
|
+
|
|
2733
|
+
if (!this._republishIssue(cluster, clusterId, recentMessages)) {
|
|
2734
|
+
throw new Error(
|
|
2735
|
+
`Cannot resume cluster ${clusterId}: No workflow triggers found and no ISSUE_OPENED message. ` +
|
|
2736
|
+
`The cluster may not have started properly. Try: zeroshot run <issue> instead.`
|
|
2737
|
+
);
|
|
2738
|
+
}
|
|
2739
|
+
} else {
|
|
2740
|
+
throw new Error(
|
|
2741
|
+
`Cannot resume cluster ${clusterId}: Found trigger ${lastTrigger.topic} but no agents handle it. ` +
|
|
2742
|
+
`Check agent trigger configurations.`
|
|
2743
|
+
);
|
|
2744
|
+
}
|
|
2745
|
+
} else {
|
|
2746
|
+
this._log(`[Orchestrator] Resuming ${agentsToResume.length} agent(s) based on ledger state`);
|
|
2747
|
+
this._resumeAgents(agentsToResume, context);
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
await this._saveClusters();
|
|
2751
|
+
this._log(`[Orchestrator] Cluster ${clusterId} resumed`);
|
|
2752
|
+
|
|
2753
|
+
return {
|
|
2754
|
+
id: clusterId,
|
|
2755
|
+
state: cluster.state,
|
|
2756
|
+
resumeType: 'clean',
|
|
2757
|
+
resumedAgents: agentsToResume.map((a) => a.agent.id),
|
|
2758
|
+
};
|
|
2759
|
+
}
|
|
2760
|
+
|
|
2761
|
+
/**
|
|
2762
|
+
* Force restart a stale agent with imperative prompt injection
|
|
2763
|
+
* @param {string} clusterId - Cluster ID
|
|
2764
|
+
* @param {string} agentId - Agent to restart
|
|
2765
|
+
* @param {number} staleDuration - How long agent was stale (ms)
|
|
2766
|
+
* @private
|
|
2767
|
+
*/
|
|
2768
|
+
async _forceRestartAgent(clusterId, agentId, staleDuration) {
|
|
2769
|
+
const cluster = this.clusters.get(clusterId);
|
|
2770
|
+
if (!cluster) {
|
|
2771
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
2772
|
+
}
|
|
2773
|
+
|
|
2774
|
+
const agent = cluster.agents.find((a) => a.id === agentId);
|
|
2775
|
+
if (!agent) {
|
|
2776
|
+
throw new Error(`Agent ${agentId} not found in cluster ${clusterId}`);
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2779
|
+
// Kill current task
|
|
2780
|
+
try {
|
|
2781
|
+
agent._killTask();
|
|
2782
|
+
} catch (err) {
|
|
2783
|
+
this._log(`⚠️ Failed to kill agent ${agentId} task:`, err.message);
|
|
2784
|
+
}
|
|
2785
|
+
|
|
2786
|
+
// Build imperative restart context
|
|
2787
|
+
const staleMinutes = Math.round(staleDuration / 60000);
|
|
2788
|
+
const imperativePrompt = `
|
|
2789
|
+
🔴 CRITICAL: Your previous session STOPPED PRODUCING OUTPUT for ${staleMinutes} minutes and was detected as STUCK.
|
|
2790
|
+
|
|
2791
|
+
## What Happened
|
|
2792
|
+
- Last output timestamp: ${new Date(Date.now() - staleDuration).toISOString()}
|
|
2793
|
+
- Detected as stale after ${staleMinutes} minutes of silence
|
|
2794
|
+
- Process was forcefully restarted
|
|
2795
|
+
|
|
2796
|
+
## Your Instructions
|
|
2797
|
+
You MUST complete your current task. DO NOT STOP until you either:
|
|
2798
|
+
1. Successfully complete the task and publish your completion message, OR
|
|
2799
|
+
2. Explicitly state WHY you cannot complete the task (missing files, impossible requirements, etc.)
|
|
2800
|
+
|
|
2801
|
+
If you discovered that files you need to modify don't exist:
|
|
2802
|
+
- CREATE them from scratch with the expected implementation
|
|
2803
|
+
- DO NOT silently give up
|
|
2804
|
+
- DO NOT stop working without explicit explanation
|
|
2805
|
+
|
|
2806
|
+
If you are stuck in an impossible situation:
|
|
2807
|
+
- EXPLAIN the problem clearly
|
|
2808
|
+
- PROPOSE alternative solutions
|
|
2809
|
+
- WAIT for guidance - do not exit
|
|
2810
|
+
|
|
2811
|
+
## Resume Your Work
|
|
2812
|
+
Continue from where you left off. Review your previous output to understand what you were working on.
|
|
2813
|
+
`.trim();
|
|
2814
|
+
|
|
2815
|
+
// Get recent context from ledger
|
|
2816
|
+
const recentMessages = cluster.messageBus
|
|
2817
|
+
.query({
|
|
2818
|
+
cluster_id: cluster.id,
|
|
2819
|
+
limit: 10,
|
|
2820
|
+
order: 'desc',
|
|
2821
|
+
})
|
|
2822
|
+
.reverse();
|
|
2823
|
+
|
|
2824
|
+
const contextText = recentMessages
|
|
2825
|
+
.map((m) => `[${m.sender}] ${m.content?.text || JSON.stringify(m.content)}`)
|
|
2826
|
+
.join('\n\n');
|
|
2827
|
+
|
|
2828
|
+
const fullContext = `${imperativePrompt}\n\n## Recent Context\n${contextText}`;
|
|
2829
|
+
|
|
2830
|
+
// Resume agent with imperative prompt
|
|
2831
|
+
this._log(
|
|
2832
|
+
`🔄 Restarting agent ${agentId} with imperative prompt (${imperativePrompt.length} chars)`
|
|
2833
|
+
);
|
|
2834
|
+
|
|
2835
|
+
try {
|
|
2836
|
+
await agent.resume(fullContext);
|
|
2837
|
+
this._log(`✅ Agent ${agentId} successfully restarted`);
|
|
2838
|
+
} catch (err) {
|
|
2839
|
+
this._log(`❌ Failed to resume agent ${agentId}:`, err.message);
|
|
2840
|
+
throw err;
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
|
|
2844
|
+
/**
|
|
2845
|
+
* Handle operation chain from CLUSTER_OPERATIONS message
|
|
2846
|
+
* Executes operations sequentially: add_agents, remove_agents, update_agent, publish
|
|
2847
|
+
*
|
|
2848
|
+
* Validation strategy:
|
|
2849
|
+
* 1. Pre-validate all agent configs before executing any operations
|
|
2850
|
+
* 2. Build a mock cluster config with proposed changes
|
|
2851
|
+
* 3. Run config-validator on the mock to catch structural issues
|
|
2852
|
+
* 4. Only execute operations if validation passes
|
|
2853
|
+
*
|
|
2854
|
+
* @param {string} clusterId - Cluster ID
|
|
2855
|
+
* @param {Array} operations - Array of operation objects
|
|
2856
|
+
* @param {string} sender - Who sent the operations (for attribution)
|
|
2857
|
+
* @param {Object} context - Isolation context { isolationManager, containerId }
|
|
2858
|
+
* @private
|
|
2859
|
+
*/
|
|
2860
|
+
async _handleOperations(clusterId, operations, sender, context = {}) {
|
|
2861
|
+
const cluster = this.clusters.get(clusterId);
|
|
2862
|
+
if (!cluster) {
|
|
2863
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
2864
|
+
}
|
|
2865
|
+
|
|
2866
|
+
this._log(`[Orchestrator] Validating ${operations.length} operation(s) from ${sender}`);
|
|
2867
|
+
|
|
2868
|
+
// Phase 1: Pre-validate operation structure
|
|
2869
|
+
const validationErrors = this._validateOperationChain(operations);
|
|
2870
|
+
|
|
2871
|
+
if (validationErrors.length > 0) {
|
|
2872
|
+
const errorMsg = `Operation chain validation failed:\n - ${validationErrors.join('\n - ')}`;
|
|
2873
|
+
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
2874
|
+
throw new Error(errorMsg);
|
|
2875
|
+
}
|
|
2876
|
+
|
|
2877
|
+
// Phase 2: Build mock cluster config with proposed agents
|
|
2878
|
+
// Collect all agents that would exist after operations complete
|
|
2879
|
+
const existingAgentConfigs = cluster.config.agents || [];
|
|
2880
|
+
const proposedAgentConfigs = this._buildProposedAgentConfigs(existingAgentConfigs, operations);
|
|
2881
|
+
|
|
2882
|
+
// Phase 3: Validate proposed cluster config
|
|
2883
|
+
const validation = this._validateProposedConfig(
|
|
2884
|
+
clusterId,
|
|
2885
|
+
cluster,
|
|
2886
|
+
proposedAgentConfigs,
|
|
2887
|
+
operations
|
|
2888
|
+
);
|
|
2889
|
+
|
|
2890
|
+
// Log warnings but proceed
|
|
2891
|
+
if (validation.warnings.length > 0) {
|
|
2892
|
+
this._log(`[Orchestrator] ⚠️ Warnings (proceeding anyway):`);
|
|
2893
|
+
for (const warning of validation.warnings) {
|
|
2894
|
+
this._log(` - ${warning}`);
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
|
|
2898
|
+
// Phase 4: Execute validated operations
|
|
2899
|
+
this._log(`[Orchestrator] ✓ Validation passed, executing ${operations.length} operation(s)`);
|
|
2900
|
+
|
|
2901
|
+
await this._executeOperations(cluster, operations, sender, context);
|
|
2902
|
+
|
|
2903
|
+
this._log(`[Orchestrator] All ${operations.length} operation(s) executed successfully`);
|
|
2904
|
+
|
|
2905
|
+
// Publish success notification
|
|
2906
|
+
cluster.messageBus.publish({
|
|
2907
|
+
cluster_id: clusterId,
|
|
2908
|
+
topic: 'CLUSTER_OPERATIONS_SUCCESS',
|
|
2909
|
+
sender: 'orchestrator',
|
|
2910
|
+
content: {
|
|
2911
|
+
text: `Executed ${operations.length} operation(s)`,
|
|
2912
|
+
data: {
|
|
2913
|
+
operationCount: operations.length,
|
|
2914
|
+
agentCount: cluster.agents.length,
|
|
2915
|
+
},
|
|
2916
|
+
},
|
|
2917
|
+
});
|
|
2918
|
+
|
|
2919
|
+
// Save updated cluster state to disk
|
|
2920
|
+
await this._saveClusters();
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
_validateOperationChain(operations) {
|
|
2924
|
+
const validationErrors = [];
|
|
2925
|
+
|
|
2926
|
+
for (let i = 0; i < operations.length; i++) {
|
|
2927
|
+
const op = operations[i];
|
|
2928
|
+
if (!op.action) {
|
|
2929
|
+
validationErrors.push(`Operation ${i}: missing 'action' field`);
|
|
2930
|
+
continue;
|
|
2931
|
+
}
|
|
2932
|
+
if (!VALID_OPERATIONS.includes(op.action)) {
|
|
2933
|
+
validationErrors.push(
|
|
2934
|
+
`Operation ${i}: unknown action '${op.action}'. Valid: ${VALID_OPERATIONS.join(', ')}`
|
|
2935
|
+
);
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
|
|
2939
|
+
return validationErrors;
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2942
|
+
_buildProposedAgentConfigs(existingAgentConfigs, operations) {
|
|
2943
|
+
const proposedAgentConfigs = [...existingAgentConfigs];
|
|
2944
|
+
|
|
2945
|
+
for (const op of operations) {
|
|
2946
|
+
if (op.action === 'add_agents' && op.agents) {
|
|
2947
|
+
for (const agentConfig of op.agents) {
|
|
2948
|
+
const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
|
|
2949
|
+
if (existingIdx === -1) {
|
|
2950
|
+
proposedAgentConfigs.push(agentConfig);
|
|
2951
|
+
}
|
|
2952
|
+
}
|
|
2953
|
+
} else if (op.action === 'load_config' && op.config) {
|
|
2954
|
+
const loadedAgentConfigs = this._resolveLoadConfigAgents(op.config);
|
|
2955
|
+
for (const agentConfig of loadedAgentConfigs) {
|
|
2956
|
+
const existingIdx = proposedAgentConfigs.findIndex((a) => a.id === agentConfig.id);
|
|
2957
|
+
if (existingIdx === -1) {
|
|
2958
|
+
proposedAgentConfigs.push(agentConfig);
|
|
2959
|
+
}
|
|
2960
|
+
}
|
|
2961
|
+
} else if (op.action === 'remove_agents' && op.agentIds) {
|
|
2962
|
+
for (const agentId of op.agentIds) {
|
|
2963
|
+
const idx = proposedAgentConfigs.findIndex((a) => a.id === agentId);
|
|
2964
|
+
if (idx !== -1) {
|
|
2965
|
+
proposedAgentConfigs.splice(idx, 1);
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
} else if (op.action === 'update_agent' && op.agentId && op.updates) {
|
|
2969
|
+
const agentConfig = proposedAgentConfigs.find((a) => a.id === op.agentId);
|
|
2970
|
+
if (agentConfig) {
|
|
2971
|
+
Object.assign(agentConfig, op.updates);
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
return proposedAgentConfigs;
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
_resolveLoadConfigAgents(config) {
|
|
2980
|
+
if (!config) {
|
|
2981
|
+
throw new Error('load_config operation missing config');
|
|
2982
|
+
}
|
|
2983
|
+
|
|
2984
|
+
const templatesDir = path.join(__dirname, '..', 'cluster-templates');
|
|
2985
|
+
let loadedConfig;
|
|
2986
|
+
|
|
2987
|
+
// Parameterized template - resolve with TemplateResolver
|
|
2988
|
+
if (typeof config === 'object' && config.base) {
|
|
2989
|
+
const { base, params } = config;
|
|
2990
|
+
const resolver = new TemplateResolver(templatesDir);
|
|
2991
|
+
loadedConfig = resolver.resolve(base, params || {});
|
|
2992
|
+
} else if (typeof config === 'string') {
|
|
2993
|
+
// Static config - load directly from file
|
|
2994
|
+
const configPath = path.join(templatesDir, `${config}.json`);
|
|
2995
|
+
if (!fs.existsSync(configPath)) {
|
|
2996
|
+
throw new Error(`Config not found: ${config} (looked in ${configPath})`);
|
|
2997
|
+
}
|
|
2998
|
+
const configContent = fs.readFileSync(configPath, 'utf8');
|
|
2999
|
+
loadedConfig = JSON.parse(configContent);
|
|
3000
|
+
} else {
|
|
3001
|
+
throw new Error(
|
|
3002
|
+
`Invalid config format: expected string or {base, params}, got ${typeof config}`
|
|
3003
|
+
);
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
if (!loadedConfig.agents || !Array.isArray(loadedConfig.agents)) {
|
|
3007
|
+
throw new Error(`Config has no agents array`);
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
return loadedConfig.agents;
|
|
3011
|
+
}
|
|
3012
|
+
|
|
3013
|
+
_hasCompletionHandler(agentConfigs) {
|
|
3014
|
+
return (agentConfigs || []).some(
|
|
3015
|
+
(agent) =>
|
|
3016
|
+
agent.id === 'completion-detector' ||
|
|
3017
|
+
agent.id === 'git-pusher' ||
|
|
3018
|
+
agent.hooks?.onComplete?.config?.topic === 'CLUSTER_COMPLETE' ||
|
|
3019
|
+
agent.triggers?.some((trigger) => trigger.action === 'stop_cluster')
|
|
3020
|
+
);
|
|
3021
|
+
}
|
|
3022
|
+
|
|
3023
|
+
_clusterHasAgent(cluster, agentId) {
|
|
3024
|
+
return Boolean(
|
|
3025
|
+
cluster?.agents?.some((agent) => agent.id === agentId || agent.config?.id === agentId) ||
|
|
3026
|
+
cluster?.config?.agents?.some((agent) => agent.id === agentId)
|
|
3027
|
+
);
|
|
3028
|
+
}
|
|
3029
|
+
|
|
3030
|
+
_normalizeCompletionHandlersForPrMode(cluster, agentConfigs) {
|
|
3031
|
+
if (!Array.isArray(agentConfigs) || agentConfigs.length === 0) {
|
|
3032
|
+
return [];
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
const isPrMode = cluster?.autoPr ?? process.env.ZEROSHOT_PR === '1';
|
|
3036
|
+
const hasGitPusher =
|
|
3037
|
+
this._clusterHasAgent(cluster, 'git-pusher') ||
|
|
3038
|
+
agentConfigs.some((agent) => agent?.id === 'git-pusher');
|
|
3039
|
+
|
|
3040
|
+
if (!isPrMode || !hasGitPusher) {
|
|
3041
|
+
return agentConfigs;
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
return agentConfigs.filter((agent) => agent?.id !== 'completion-detector');
|
|
3045
|
+
}
|
|
3046
|
+
|
|
3047
|
+
_injectCriticalValidationResultProducer(agentConfigs) {
|
|
3048
|
+
const hasMetaCoordinator = agentConfigs.some((agent) => agent.id === 'meta-coordinator');
|
|
3049
|
+
const hasRuntimeValidator = agentConfigs.some((agent) => agent.role === 'validator');
|
|
3050
|
+
const hasSimulator = agentConfigs.some((agent) => agent.id === '__validation-result-simulator');
|
|
3051
|
+
|
|
3052
|
+
// CRITICAL flow loads quick/heavy validators dynamically; simulate stage-1 producer
|
|
3053
|
+
// for static validation so we don't reject legitimate staged topologies.
|
|
3054
|
+
if (hasMetaCoordinator && !hasRuntimeValidator && !hasSimulator) {
|
|
3055
|
+
agentConfigs.push({
|
|
3056
|
+
id: '__validation-result-simulator',
|
|
3057
|
+
role: 'validator',
|
|
3058
|
+
triggers: [{ topic: 'IMPLEMENTATION_READY', action: 'execute_task' }],
|
|
3059
|
+
hooks: {
|
|
3060
|
+
onComplete: {
|
|
3061
|
+
action: 'publish_message',
|
|
3062
|
+
config: {
|
|
3063
|
+
topic: 'VALIDATION_RESULT',
|
|
3064
|
+
content: {
|
|
3065
|
+
data: {
|
|
3066
|
+
approved: true,
|
|
3067
|
+
errors: [],
|
|
3068
|
+
criteriaResults: [],
|
|
3069
|
+
},
|
|
3070
|
+
},
|
|
3071
|
+
},
|
|
3072
|
+
},
|
|
3073
|
+
},
|
|
3074
|
+
});
|
|
3075
|
+
}
|
|
3076
|
+
}
|
|
3077
|
+
|
|
3078
|
+
_prepareValidationAgentConfigs(cluster, proposedAgentConfigs) {
|
|
3079
|
+
const validationAgents = this._normalizeCompletionHandlersForPrMode(
|
|
3080
|
+
cluster,
|
|
3081
|
+
JSON.parse(JSON.stringify(proposedAgentConfigs || []))
|
|
3082
|
+
);
|
|
3083
|
+
|
|
3084
|
+
this._injectCriticalValidationResultProducer(validationAgents);
|
|
3085
|
+
|
|
3086
|
+
// Operation-chain validation runs before _injectCompletionAgent executes.
|
|
3087
|
+
// Add a synthetic completion handler so transient missing completion detectors
|
|
3088
|
+
// don't invalidate otherwise-correct topology changes.
|
|
3089
|
+
if (!this._hasCompletionHandler(validationAgents)) {
|
|
3090
|
+
validationAgents.push({
|
|
3091
|
+
id: '__completion-validator',
|
|
3092
|
+
role: 'orchestrator',
|
|
3093
|
+
triggers: [{ topic: 'VALIDATION_RESULT', action: 'stop_cluster' }],
|
|
3094
|
+
});
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
return validationAgents;
|
|
3098
|
+
}
|
|
3099
|
+
|
|
3100
|
+
_validateProposedConfig(clusterId, cluster, proposedAgentConfigs, operations) {
|
|
3101
|
+
const validationAgents = this._prepareValidationAgentConfigs(cluster, proposedAgentConfigs);
|
|
3102
|
+
const mockConfig = { agents: validationAgents };
|
|
3103
|
+
const validation = configValidator.validateConfig(mockConfig);
|
|
3104
|
+
|
|
3105
|
+
if (!validation.valid) {
|
|
3106
|
+
const errorMsg = `Proposed cluster configuration is invalid:\n - ${validation.errors.join('\n - ')}`;
|
|
3107
|
+
this._log(`[Orchestrator] ❌ ${errorMsg}`);
|
|
3108
|
+
|
|
3109
|
+
// Publish validation failure for conductor to see and retry
|
|
3110
|
+
cluster.messageBus.publish({
|
|
3111
|
+
cluster_id: clusterId,
|
|
3112
|
+
topic: 'CLUSTER_OPERATIONS_VALIDATION_FAILED',
|
|
3113
|
+
sender: 'orchestrator',
|
|
3114
|
+
content: {
|
|
3115
|
+
text: 'Operation chain would create invalid cluster configuration',
|
|
3116
|
+
data: {
|
|
3117
|
+
errors: validation.errors,
|
|
3118
|
+
warnings: validation.warnings,
|
|
3119
|
+
operations: operations,
|
|
3120
|
+
},
|
|
3121
|
+
},
|
|
3122
|
+
});
|
|
3123
|
+
|
|
3124
|
+
throw new Error(errorMsg);
|
|
3125
|
+
}
|
|
3126
|
+
|
|
3127
|
+
return validation;
|
|
3128
|
+
}
|
|
3129
|
+
|
|
3130
|
+
async _executeOperations(cluster, operations, sender, context) {
|
|
3131
|
+
for (let i = 0; i < operations.length; i++) {
|
|
3132
|
+
const op = operations[i];
|
|
3133
|
+
this._log(` [${i + 1}/${operations.length}] ${op.action}`);
|
|
3134
|
+
await this._executeOperation(cluster, op, sender, context);
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
|
|
3138
|
+
async _executeOperation(cluster, op, sender, context) {
|
|
3139
|
+
switch (op.action) {
|
|
3140
|
+
case 'add_agents':
|
|
3141
|
+
await this._opAddAgents(cluster, op, context);
|
|
3142
|
+
break;
|
|
3143
|
+
case 'remove_agents':
|
|
3144
|
+
await this._opRemoveAgents(cluster, op);
|
|
3145
|
+
break;
|
|
3146
|
+
case 'update_agent':
|
|
3147
|
+
this._opUpdateAgent(cluster, op);
|
|
3148
|
+
break;
|
|
3149
|
+
case 'publish':
|
|
3150
|
+
this._opPublish(cluster, op, sender);
|
|
3151
|
+
break;
|
|
3152
|
+
case 'load_config':
|
|
3153
|
+
await this._opLoadConfig(cluster, op, context);
|
|
3154
|
+
break;
|
|
3155
|
+
}
|
|
3156
|
+
}
|
|
3157
|
+
|
|
3158
|
+
/**
|
|
3159
|
+
* Operation: add_agents - Spawn new agents dynamically
|
|
3160
|
+
* @private
|
|
3161
|
+
*/
|
|
3162
|
+
async _opAddAgents(cluster, op, context) {
|
|
3163
|
+
const agents = this._normalizeCompletionHandlersForPrMode(cluster, op.agents);
|
|
3164
|
+
if (!agents || !Array.isArray(agents)) {
|
|
3165
|
+
throw new Error('add_agents operation missing agents array');
|
|
3166
|
+
}
|
|
3167
|
+
if (agents.length === 0) {
|
|
3168
|
+
this._log(` Skipping add_agents: PR-mode completion detector replaced by git-pusher`);
|
|
3169
|
+
return;
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
// CRITICAL: Inject cluster's worktree cwd into dynamically added agents
|
|
3173
|
+
// Without this, template agents (planning, implementation, validator) run in
|
|
3174
|
+
// original directory instead of worktree, causing --ship mode to pollute main
|
|
3175
|
+
const agentCwd = cluster.worktree?.path || cluster.isolation?.workDir || process.cwd();
|
|
3176
|
+
|
|
3177
|
+
for (const agentConfig of agents) {
|
|
3178
|
+
// Inject cwd if not already set (same as startCluster does for initial agents)
|
|
3179
|
+
if (!agentConfig.cwd) {
|
|
3180
|
+
agentConfig.cwd = agentCwd;
|
|
3181
|
+
this._log(` [cwd] Injected worktree cwd for ${agentConfig.id}: ${agentCwd}`);
|
|
3182
|
+
}
|
|
3183
|
+
|
|
3184
|
+
// Apply model override if set (for consistency with initial agents)
|
|
3185
|
+
if (cluster.modelOverride) {
|
|
3186
|
+
applyModelOverride(agentConfig, cluster.modelOverride);
|
|
3187
|
+
this._log(` [model] Overridden model for ${agentConfig.id}: ${cluster.modelOverride}`);
|
|
3188
|
+
}
|
|
3189
|
+
|
|
3190
|
+
applyRequiredQualityGatesToAgent(agentConfig, cluster.requiredQualityGates);
|
|
3191
|
+
if (cluster.autoPr) {
|
|
3192
|
+
applyPushBlockedRepairTrigger(agentConfig);
|
|
3193
|
+
}
|
|
3194
|
+
|
|
3195
|
+
// Validate agent config has required fields
|
|
3196
|
+
if (!agentConfig.id) {
|
|
3197
|
+
throw new Error('Agent config missing required field: id');
|
|
3198
|
+
}
|
|
3199
|
+
|
|
3200
|
+
// Check for duplicate agent ID - REPLACE agent entirely
|
|
3201
|
+
// Previous behavior merged triggers but kept old hooks, causing bugs when
|
|
3202
|
+
// loading templates with same agent IDs but different hooks (e.g., quick-validation
|
|
3203
|
+
// and heavy-validation both have consensus-coordinator with different onComplete hooks)
|
|
3204
|
+
const existingAgentIndex = cluster.agents.findIndex((a) => a.id === agentConfig.id);
|
|
3205
|
+
if (existingAgentIndex !== -1) {
|
|
3206
|
+
const existingAgent = cluster.agents[existingAgentIndex];
|
|
3207
|
+
this._log(
|
|
3208
|
+
` 🔄 Replacing agent ${agentConfig.id} (old role: ${existingAgent.config.role})`
|
|
3209
|
+
);
|
|
3210
|
+
|
|
3211
|
+
// Stop the existing agent and wait for shutdown before replacement.
|
|
3212
|
+
// This prevents the old instance from draining buffered messages or
|
|
3213
|
+
// finishing in-flight work after the new agent has already started.
|
|
3214
|
+
if (existingAgent.stop) {
|
|
3215
|
+
await existingAgent.stop();
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
// Remove from cluster.agents array
|
|
3219
|
+
cluster.agents.splice(existingAgentIndex, 1);
|
|
3220
|
+
|
|
3221
|
+
// Remove from cluster.config.agents if present
|
|
3222
|
+
if (cluster.config.agents) {
|
|
3223
|
+
const configIndex = cluster.config.agents.findIndex((a) => a.id === agentConfig.id);
|
|
3224
|
+
if (configIndex !== -1) {
|
|
3225
|
+
cluster.config.agents.splice(configIndex, 1);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
// Continue to add the new agent below
|
|
3229
|
+
}
|
|
3230
|
+
|
|
3231
|
+
// Add to config agents array (for persistence)
|
|
3232
|
+
if (!cluster.config.agents) {
|
|
3233
|
+
cluster.config.agents = [];
|
|
3234
|
+
}
|
|
3235
|
+
cluster.config.agents.push(agentConfig);
|
|
3236
|
+
|
|
3237
|
+
// Build agent options
|
|
3238
|
+
const agentOptions = {
|
|
3239
|
+
testMode: !!this.taskRunner, // Enable testMode if taskRunner provided
|
|
3240
|
+
quiet: this.quiet,
|
|
3241
|
+
modelOverride: cluster.modelOverride || null,
|
|
3242
|
+
};
|
|
3243
|
+
|
|
3244
|
+
// TaskRunner DI - propagate to dynamically spawned agents
|
|
3245
|
+
if (this.taskRunner) {
|
|
3246
|
+
agentOptions.taskRunner = this.taskRunner;
|
|
3247
|
+
}
|
|
3248
|
+
|
|
3249
|
+
// Pass isolation context if cluster is running in isolation mode
|
|
3250
|
+
if (cluster.isolation?.enabled && context.isolationManager) {
|
|
3251
|
+
agentOptions.isolation = {
|
|
3252
|
+
enabled: true,
|
|
3253
|
+
manager: context.isolationManager,
|
|
3254
|
+
clusterId: cluster.id,
|
|
3255
|
+
};
|
|
3256
|
+
}
|
|
3257
|
+
|
|
3258
|
+
// Create and start agent
|
|
3259
|
+
const agent = new AgentWrapper(agentConfig, cluster.messageBus, cluster, agentOptions);
|
|
3260
|
+
cluster.agents.push(agent);
|
|
3261
|
+
await agent.start();
|
|
3262
|
+
|
|
3263
|
+
this._log(
|
|
3264
|
+
` ✓ Added agent: ${agentConfig.id} (role: ${agentConfig.role || 'unspecified'})`
|
|
3265
|
+
);
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
/**
|
|
3270
|
+
* Operation: remove_agents - Stop and remove agents by ID
|
|
3271
|
+
* @private
|
|
3272
|
+
*/
|
|
3273
|
+
async _opRemoveAgents(cluster, op) {
|
|
3274
|
+
const agentIds = op.agentIds;
|
|
3275
|
+
if (!agentIds || !Array.isArray(agentIds)) {
|
|
3276
|
+
throw new Error('remove_agents operation missing agentIds array');
|
|
3277
|
+
}
|
|
3278
|
+
|
|
3279
|
+
for (const agentId of agentIds) {
|
|
3280
|
+
const agentIndex = cluster.agents.findIndex((a) => a.id === agentId);
|
|
3281
|
+
if (agentIndex === -1) {
|
|
3282
|
+
this._log(` ⚠️ Agent ${agentId} not found, skipping removal`);
|
|
3283
|
+
continue;
|
|
3284
|
+
}
|
|
3285
|
+
|
|
3286
|
+
const agent = cluster.agents[agentIndex];
|
|
3287
|
+
await agent.stop();
|
|
3288
|
+
|
|
3289
|
+
// Remove from cluster.agents
|
|
3290
|
+
cluster.agents.splice(agentIndex, 1);
|
|
3291
|
+
|
|
3292
|
+
// Remove from config.agents
|
|
3293
|
+
if (cluster.config.agents) {
|
|
3294
|
+
const configIndex = cluster.config.agents.findIndex((a) => a.id === agentId);
|
|
3295
|
+
if (configIndex !== -1) {
|
|
3296
|
+
cluster.config.agents.splice(configIndex, 1);
|
|
3297
|
+
}
|
|
3298
|
+
}
|
|
3299
|
+
|
|
3300
|
+
this._log(` ✓ Removed agent: ${agentId}`);
|
|
3301
|
+
}
|
|
3302
|
+
}
|
|
3303
|
+
|
|
3304
|
+
/**
|
|
3305
|
+
* Operation: update_agent - Modify existing agent config at runtime
|
|
3306
|
+
* Note: Some updates may require agent restart to take effect
|
|
3307
|
+
* @private
|
|
3308
|
+
*/
|
|
3309
|
+
_opUpdateAgent(cluster, op) {
|
|
3310
|
+
const { agentId, updates } = op;
|
|
3311
|
+
if (!agentId) {
|
|
3312
|
+
throw new Error('update_agent operation missing agentId');
|
|
3313
|
+
}
|
|
3314
|
+
if (!updates || typeof updates !== 'object') {
|
|
3315
|
+
throw new Error('update_agent operation missing updates object');
|
|
3316
|
+
}
|
|
3317
|
+
|
|
3318
|
+
const agent = cluster.agents.find((a) => a.id === agentId);
|
|
3319
|
+
if (!agent) {
|
|
3320
|
+
throw new Error(`update_agent: Agent ${agentId} not found`);
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
// Apply updates to agent config
|
|
3324
|
+
Object.assign(agent.config, updates);
|
|
3325
|
+
|
|
3326
|
+
// Also update in cluster.config.agents for persistence
|
|
3327
|
+
if (cluster.config.agents) {
|
|
3328
|
+
const configAgent = cluster.config.agents.find((a) => a.id === agentId);
|
|
3329
|
+
if (configAgent) {
|
|
3330
|
+
Object.assign(configAgent, updates);
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
|
|
3334
|
+
this._log(` ✓ Updated agent: ${agentId} (fields: ${Object.keys(updates).join(', ')})`);
|
|
3335
|
+
}
|
|
3336
|
+
|
|
3337
|
+
/**
|
|
3338
|
+
* Operation: publish - Publish a message to the bus
|
|
3339
|
+
* @private
|
|
3340
|
+
*/
|
|
3341
|
+
_opPublish(cluster, op, sender) {
|
|
3342
|
+
const { topic, content, metadata } = op;
|
|
3343
|
+
if (!topic) {
|
|
3344
|
+
throw new Error('publish operation missing topic');
|
|
3345
|
+
}
|
|
3346
|
+
|
|
3347
|
+
cluster.messageBus.publish({
|
|
3348
|
+
cluster_id: cluster.id,
|
|
3349
|
+
topic,
|
|
3350
|
+
sender: op.sender || sender,
|
|
3351
|
+
receiver: op.receiver || 'broadcast',
|
|
3352
|
+
content: content || {},
|
|
3353
|
+
metadata: metadata || null,
|
|
3354
|
+
});
|
|
3355
|
+
|
|
3356
|
+
this._log(` ✓ Published to topic: ${topic}`);
|
|
3357
|
+
}
|
|
3358
|
+
|
|
3359
|
+
/**
|
|
3360
|
+
* Operation: load_config - Load agents from a cluster config
|
|
3361
|
+
*
|
|
3362
|
+
* Supports two formats:
|
|
3363
|
+
* 1. Static config: { config: 'config-name' } - loads from cluster-templates/config-name.json
|
|
3364
|
+
* 2. Parameterized: { config: { base: 'template-name', params: {...} } } - resolves base template with params
|
|
3365
|
+
*
|
|
3366
|
+
* @private
|
|
3367
|
+
*/
|
|
3368
|
+
async _opLoadConfig(cluster, op, context) {
|
|
3369
|
+
const { config } = op;
|
|
3370
|
+
if (!config) {
|
|
3371
|
+
throw new Error('load_config operation missing config');
|
|
3372
|
+
}
|
|
3373
|
+
|
|
3374
|
+
const templatesDir = path.join(__dirname, '..', 'cluster-templates');
|
|
3375
|
+
let loadedConfig;
|
|
3376
|
+
|
|
3377
|
+
// Check if config is parameterized ({ base, params }) or static (string)
|
|
3378
|
+
if (typeof config === 'object' && config.base) {
|
|
3379
|
+
// Parameterized template - resolve with TemplateResolver
|
|
3380
|
+
const { base, params } = config;
|
|
3381
|
+
this._log(` Loading parameterized template: ${base}`);
|
|
3382
|
+
this._log(` Params: ${JSON.stringify(params)}`);
|
|
3383
|
+
|
|
3384
|
+
const resolver = new TemplateResolver(templatesDir);
|
|
3385
|
+
loadedConfig = resolver.resolve(base, params);
|
|
3386
|
+
|
|
3387
|
+
this._log(` ✓ Resolved template: ${base} → ${loadedConfig.agents?.length || 0} agent(s)`);
|
|
3388
|
+
} else if (typeof config === 'string') {
|
|
3389
|
+
// Static config - load directly from file
|
|
3390
|
+
const configPath = path.join(templatesDir, `${config}.json`);
|
|
3391
|
+
|
|
3392
|
+
if (!fs.existsSync(configPath)) {
|
|
3393
|
+
throw new Error(`Config not found: ${config} (looked in ${configPath})`);
|
|
3394
|
+
}
|
|
3395
|
+
|
|
3396
|
+
this._log(` Loading static config: ${config}`);
|
|
3397
|
+
|
|
3398
|
+
const configContent = fs.readFileSync(configPath, 'utf8');
|
|
3399
|
+
loadedConfig = JSON.parse(configContent);
|
|
3400
|
+
} else {
|
|
3401
|
+
throw new Error(
|
|
3402
|
+
`Invalid config format: expected string or {base, params}, got ${typeof config}`
|
|
3403
|
+
);
|
|
3404
|
+
}
|
|
3405
|
+
|
|
3406
|
+
if (!loadedConfig.agents || !Array.isArray(loadedConfig.agents)) {
|
|
3407
|
+
throw new Error(`Config has no agents array`);
|
|
3408
|
+
}
|
|
3409
|
+
|
|
3410
|
+
this._log(` Found ${loadedConfig.agents.length} agent(s)`);
|
|
3411
|
+
|
|
3412
|
+
// Add agents from loaded config (reuse existing add_agents logic)
|
|
3413
|
+
await this._opAddAgents(cluster, { agents: loadedConfig.agents }, context);
|
|
3414
|
+
|
|
3415
|
+
this._log(` ✓ Config loaded (${loadedConfig.agents.length} agents)`);
|
|
3416
|
+
|
|
3417
|
+
// Inject completion agent (templates don't include one - orchestrator controls termination)
|
|
3418
|
+
await this._injectCompletionAgent(cluster, context);
|
|
3419
|
+
}
|
|
3420
|
+
|
|
3421
|
+
/**
|
|
3422
|
+
* Inject appropriate completion agent based on mode
|
|
3423
|
+
* Templates define work, orchestrator controls termination strategy
|
|
3424
|
+
* @private
|
|
3425
|
+
*/
|
|
3426
|
+
async _injectCompletionAgent(cluster, context) {
|
|
3427
|
+
// Skip if completion agent already exists
|
|
3428
|
+
const hasCompletionAgent = cluster.agents.some(
|
|
3429
|
+
(a) => a.config?.id === 'completion-detector' || a.config?.id === 'git-pusher'
|
|
3430
|
+
);
|
|
3431
|
+
if (hasCompletionAgent) {
|
|
3432
|
+
return;
|
|
3433
|
+
}
|
|
3434
|
+
|
|
3435
|
+
const isPrMode = cluster.autoPr ?? process.env.ZEROSHOT_PR === '1';
|
|
3436
|
+
|
|
3437
|
+
if (isPrMode) {
|
|
3438
|
+
// Detect platform from stored cluster metadata OR git context
|
|
3439
|
+
let platform = cluster.gitPlatform; // Use stored git platform (not issueProvider!)
|
|
3440
|
+
|
|
3441
|
+
// Fallback to git context detection if cluster metadata missing
|
|
3442
|
+
if (!platform) {
|
|
3443
|
+
const { getPlatformForPR } = require('./issue-providers');
|
|
3444
|
+
try {
|
|
3445
|
+
platform = getPlatformForPR(cluster.cwd || process.cwd());
|
|
3446
|
+
} catch (error) {
|
|
3447
|
+
throw new Error(`Cannot determine platform for PR creation on resume: ${error.message}`);
|
|
3448
|
+
}
|
|
3449
|
+
}
|
|
3450
|
+
|
|
3451
|
+
// Generate platform-specific git-pusher agent from template
|
|
3452
|
+
const {
|
|
3453
|
+
generateGitPusherAgent,
|
|
3454
|
+
isPlatformSupported,
|
|
3455
|
+
} = require('./agents/git-pusher-template');
|
|
3456
|
+
|
|
3457
|
+
if (!isPlatformSupported(platform)) {
|
|
3458
|
+
throw new Error(
|
|
3459
|
+
`Platform '${platform}' does not support --pr mode. Supported: github, gitlab, azure-devops`
|
|
3460
|
+
);
|
|
3461
|
+
}
|
|
3462
|
+
|
|
3463
|
+
// Use persisted PR options from cluster state (or empty for repo settings fallback)
|
|
3464
|
+
const gitPusherConfig = generateGitPusherAgent(platform, cluster.prOptions || {});
|
|
3465
|
+
|
|
3466
|
+
// Get issue context from ledger
|
|
3467
|
+
const issueMsg = cluster.messageBus.ledger.findLast({ topic: 'ISSUE_OPENED' });
|
|
3468
|
+
const issueNumber = issueMsg?.content?.data?.number || 'unknown';
|
|
3469
|
+
const issueTitle = issueMsg?.content?.data?.title || 'Implementation';
|
|
3470
|
+
|
|
3471
|
+
// Inject issue context into prompt
|
|
3472
|
+
gitPusherConfig.prompt = gitPusherConfig.prompt
|
|
3473
|
+
.replace(/\{\{issue_number\}\}/g, issueNumber)
|
|
3474
|
+
.replace(/\{\{issue_title\}\}/g, issueTitle);
|
|
3475
|
+
|
|
3476
|
+
await this._opAddAgents(cluster, { agents: [gitPusherConfig] }, context);
|
|
3477
|
+
this._log(` [--pr mode] Injected ${platform}-git-pusher agent`);
|
|
3478
|
+
} else {
|
|
3479
|
+
// Default completion-detector
|
|
3480
|
+
const { SHARED_TRIGGER_SCRIPT } = require('./agents/git-pusher-template');
|
|
3481
|
+
const completionDetector = {
|
|
3482
|
+
id: 'completion-detector',
|
|
3483
|
+
role: 'orchestrator',
|
|
3484
|
+
modelLevel: this._resolveCompletionDetectorLevel(cluster.config),
|
|
3485
|
+
timeout: 0,
|
|
3486
|
+
triggers: [
|
|
3487
|
+
{
|
|
3488
|
+
topic: 'VALIDATION_RESULT',
|
|
3489
|
+
logic: {
|
|
3490
|
+
engine: 'javascript',
|
|
3491
|
+
script: SHARED_TRIGGER_SCRIPT,
|
|
3492
|
+
},
|
|
3493
|
+
action: 'stop_cluster',
|
|
3494
|
+
},
|
|
3495
|
+
],
|
|
3496
|
+
};
|
|
3497
|
+
|
|
3498
|
+
await this._opAddAgents(cluster, { agents: [completionDetector] }, context);
|
|
3499
|
+
this._log(` Injected completion-detector agent`);
|
|
3500
|
+
}
|
|
3501
|
+
}
|
|
3502
|
+
|
|
3503
|
+
/**
|
|
3504
|
+
* Check if a process with given PID is running
|
|
3505
|
+
* @param {Number} pid - Process ID
|
|
3506
|
+
* @returns {Boolean} True if process exists
|
|
3507
|
+
* @private
|
|
3508
|
+
*/
|
|
3509
|
+
_isProcessRunning(pid) {
|
|
3510
|
+
if (!pid) return false;
|
|
3511
|
+
try {
|
|
3512
|
+
// Signal 0 doesn't kill, just checks if process exists
|
|
3513
|
+
process.kill(pid, 0);
|
|
3514
|
+
return true;
|
|
3515
|
+
} catch (e) {
|
|
3516
|
+
// ESRCH = No such process, EPERM = process exists but no permission
|
|
3517
|
+
return e.code === 'EPERM';
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
|
|
3521
|
+
/**
|
|
3522
|
+
* Get cluster status
|
|
3523
|
+
* @param {String} clusterId - Cluster ID
|
|
3524
|
+
* @returns {Object} Cluster status
|
|
3525
|
+
*/
|
|
3526
|
+
getStatus(clusterId) {
|
|
3527
|
+
const cluster = this.clusters.get(clusterId);
|
|
3528
|
+
if (!cluster) {
|
|
3529
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3532
|
+
// Detect zombie clusters: state=running but no backing process
|
|
3533
|
+
let state = cluster.state;
|
|
3534
|
+
let isZombie = false;
|
|
3535
|
+
if (state === 'running') {
|
|
3536
|
+
if (cluster.pid) {
|
|
3537
|
+
// PID recorded - check if process is running
|
|
3538
|
+
if (!this._isProcessRunning(cluster.pid)) {
|
|
3539
|
+
state = 'zombie';
|
|
3540
|
+
isZombie = true;
|
|
3541
|
+
this._log(
|
|
3542
|
+
`[Orchestrator] Detected zombie cluster ${clusterId} (PID ${cluster.pid} not running)`
|
|
3543
|
+
);
|
|
3544
|
+
}
|
|
3545
|
+
} else {
|
|
3546
|
+
// No PID recorded (legacy cluster or bug) - definitely a zombie
|
|
3547
|
+
// New code always records PID for running clusters
|
|
3548
|
+
state = 'zombie';
|
|
3549
|
+
isZombie = true;
|
|
3550
|
+
this._log(
|
|
3551
|
+
`[Orchestrator] Detected zombie cluster ${clusterId} (no PID recorded - legacy or killed cluster)`
|
|
3552
|
+
);
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
return {
|
|
3557
|
+
id: clusterId,
|
|
3558
|
+
state: state,
|
|
3559
|
+
isZombie: isZombie,
|
|
3560
|
+
pid: cluster.pid || null,
|
|
3561
|
+
createdAt: cluster.createdAt,
|
|
3562
|
+
agents: cluster.agents.map((a) => a.getState()),
|
|
3563
|
+
messageCount: (() => {
|
|
3564
|
+
try {
|
|
3565
|
+
return cluster.messageBus.count({ cluster_id: clusterId });
|
|
3566
|
+
} catch {
|
|
3567
|
+
// Cluster may have closed its ledger during startup failure cleanup.
|
|
3568
|
+
// Status/list should remain safe to call for visibility + supervisor cleanup.
|
|
3569
|
+
return 0;
|
|
3570
|
+
}
|
|
3571
|
+
})(),
|
|
3572
|
+
setupLogPath: cluster.setupLogPath || null,
|
|
3573
|
+
setupStage: cluster.setupStage || null,
|
|
3574
|
+
failureInfo: cluster.failureInfo || null,
|
|
3575
|
+
};
|
|
3576
|
+
}
|
|
3577
|
+
|
|
3578
|
+
/**
|
|
3579
|
+
* List all clusters
|
|
3580
|
+
* @returns {Array} List of cluster summaries
|
|
3581
|
+
*/
|
|
3582
|
+
listClusters() {
|
|
3583
|
+
return Array.from(this.clusters.values()).map((cluster) => {
|
|
3584
|
+
// Detect zombie clusters (state=running but no backing process)
|
|
3585
|
+
let state = cluster.state;
|
|
3586
|
+
if (state === 'running') {
|
|
3587
|
+
if (cluster.pid) {
|
|
3588
|
+
if (!this._isProcessRunning(cluster.pid)) {
|
|
3589
|
+
state = 'zombie';
|
|
3590
|
+
}
|
|
3591
|
+
} else {
|
|
3592
|
+
// No PID recorded - definitely a zombie
|
|
3593
|
+
state = 'zombie';
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
|
|
3597
|
+
return {
|
|
3598
|
+
id: cluster.id,
|
|
3599
|
+
state: state,
|
|
3600
|
+
createdAt: cluster.createdAt,
|
|
3601
|
+
issue: cluster.issue || null,
|
|
3602
|
+
agentCount: cluster.agents.length,
|
|
3603
|
+
messageCount: (() => {
|
|
3604
|
+
try {
|
|
3605
|
+
return cluster.messageBus.count({ cluster_id: cluster.id });
|
|
3606
|
+
} catch {
|
|
3607
|
+
// Cluster may have closed its ledger during startup failure cleanup.
|
|
3608
|
+
// List should remain safe to call for cleanup routines.
|
|
3609
|
+
return 0;
|
|
3610
|
+
}
|
|
3611
|
+
})(),
|
|
3612
|
+
pid: cluster.pid || null,
|
|
3613
|
+
setupLogPath: cluster.setupLogPath || null,
|
|
3614
|
+
setupStage: cluster.setupStage || null,
|
|
3615
|
+
failureInfo: cluster.failureInfo || null,
|
|
3616
|
+
};
|
|
3617
|
+
});
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
/**
|
|
3621
|
+
* Get cluster object (for internal use)
|
|
3622
|
+
* @param {String} clusterId - Cluster ID
|
|
3623
|
+
* @returns {Object} Full cluster object
|
|
3624
|
+
*/
|
|
3625
|
+
getCluster(clusterId) {
|
|
3626
|
+
return this.clusters.get(clusterId);
|
|
3627
|
+
}
|
|
3628
|
+
|
|
3629
|
+
/**
|
|
3630
|
+
* Export cluster conversation
|
|
3631
|
+
* @param {String} clusterId - Cluster ID
|
|
3632
|
+
* @param {String} format - Export format ('json' or 'markdown')
|
|
3633
|
+
* @returns {String} Exported data
|
|
3634
|
+
*/
|
|
3635
|
+
export(clusterId, format = 'json') {
|
|
3636
|
+
const cluster = this.clusters.get(clusterId);
|
|
3637
|
+
if (!cluster) {
|
|
3638
|
+
throw new Error(`Cluster ${clusterId} not found`);
|
|
3639
|
+
}
|
|
3640
|
+
|
|
3641
|
+
const messages = cluster.messageBus.getAll(clusterId);
|
|
3642
|
+
|
|
3643
|
+
if (format === 'json') {
|
|
3644
|
+
return JSON.stringify(
|
|
3645
|
+
{
|
|
3646
|
+
cluster_id: clusterId,
|
|
3647
|
+
state: cluster.state,
|
|
3648
|
+
created_at: cluster.createdAt,
|
|
3649
|
+
agents: cluster.agents.map((a) => a.getState()),
|
|
3650
|
+
messages,
|
|
3651
|
+
},
|
|
3652
|
+
null,
|
|
3653
|
+
2
|
|
3654
|
+
);
|
|
3655
|
+
} else if (format === 'markdown') {
|
|
3656
|
+
return this._exportMarkdown(cluster, clusterId, messages);
|
|
3657
|
+
} else {
|
|
3658
|
+
throw new Error(`Unknown export format: ${format}`);
|
|
3659
|
+
}
|
|
3660
|
+
}
|
|
3661
|
+
|
|
3662
|
+
/**
|
|
3663
|
+
* Export cluster as nicely formatted markdown
|
|
3664
|
+
* @private
|
|
3665
|
+
*/
|
|
3666
|
+
_exportMarkdown(cluster, clusterId, messages) {
|
|
3667
|
+
const { parseProviderChunk } = require('./providers');
|
|
3668
|
+
|
|
3669
|
+
// Find task info
|
|
3670
|
+
const issueOpened = messages.find((m) => m.topic === 'ISSUE_OPENED');
|
|
3671
|
+
const taskText = issueOpened?.content?.text || 'Unknown task';
|
|
3672
|
+
|
|
3673
|
+
// Calculate duration
|
|
3674
|
+
const firstMsg = messages[0];
|
|
3675
|
+
const lastMsg = messages[messages.length - 1];
|
|
3676
|
+
const durationMs = lastMsg ? lastMsg.timestamp - firstMsg.timestamp : 0;
|
|
3677
|
+
const durationMin = Math.round(durationMs / 60000);
|
|
3678
|
+
|
|
3679
|
+
// Header
|
|
3680
|
+
let md = `# Cluster: ${clusterId}\n\n`;
|
|
3681
|
+
md += `| Property | Value |\n|----------|-------|\n`;
|
|
3682
|
+
md += `| State | ${cluster.state} |\n`;
|
|
3683
|
+
md += `| Created | ${new Date(cluster.createdAt).toLocaleString()} |\n`;
|
|
3684
|
+
md += `| Duration | ${durationMin} minutes |\n`;
|
|
3685
|
+
md += `| Agents | ${cluster.agents.map((a) => a.id).join(', ')} |\n\n`;
|
|
3686
|
+
|
|
3687
|
+
// Task
|
|
3688
|
+
md += `## Task\n\n${taskText}\n\n`;
|
|
3689
|
+
|
|
3690
|
+
// Group messages by agent for cleaner output
|
|
3691
|
+
const agentOutputs = this._collectAgentOutputs(messages);
|
|
3692
|
+
|
|
3693
|
+
for (const [agentId, agentMsgs] of agentOutputs) {
|
|
3694
|
+
md += this._renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk);
|
|
3695
|
+
}
|
|
3696
|
+
|
|
3697
|
+
md += this._renderValidationMarkdown(messages);
|
|
3698
|
+
|
|
3699
|
+
// Final status
|
|
3700
|
+
const clusterComplete = messages.find((m) => m.topic === 'CLUSTER_COMPLETE');
|
|
3701
|
+
if (clusterComplete) {
|
|
3702
|
+
md += `## Result\n\n✅ **Cluster completed successfully**\n`;
|
|
3703
|
+
}
|
|
3704
|
+
|
|
3705
|
+
return md;
|
|
3706
|
+
}
|
|
3707
|
+
|
|
3708
|
+
_collectAgentOutputs(messages) {
|
|
3709
|
+
const agentOutputs = new Map();
|
|
3710
|
+
|
|
3711
|
+
for (const msg of messages) {
|
|
3712
|
+
if (msg.topic !== 'AGENT_OUTPUT') continue;
|
|
3713
|
+
if (!agentOutputs.has(msg.sender)) {
|
|
3714
|
+
agentOutputs.set(msg.sender, []);
|
|
3715
|
+
}
|
|
3716
|
+
agentOutputs.get(msg.sender).push(msg);
|
|
3717
|
+
}
|
|
3718
|
+
|
|
3719
|
+
return agentOutputs;
|
|
3720
|
+
}
|
|
3721
|
+
|
|
3722
|
+
_extractAgentOutput(agentMsgs, parseProviderChunk) {
|
|
3723
|
+
let text = '';
|
|
3724
|
+
const tools = [];
|
|
3725
|
+
|
|
3726
|
+
for (const msg of agentMsgs) {
|
|
3727
|
+
const content = msg.content?.data?.line || msg.content?.data?.chunk || msg.content?.text;
|
|
3728
|
+
if (!content) continue;
|
|
3729
|
+
|
|
3730
|
+
const provider = normalizeProviderName(
|
|
3731
|
+
msg.content?.data?.provider || msg.sender_provider || 'claude'
|
|
3732
|
+
);
|
|
3733
|
+
const events = parseProviderChunk(provider, content);
|
|
3734
|
+
for (const event of events) {
|
|
3735
|
+
switch (event.type) {
|
|
3736
|
+
case 'text':
|
|
3737
|
+
text += event.text;
|
|
3738
|
+
break;
|
|
3739
|
+
case 'tool_call':
|
|
3740
|
+
tools.push({ name: event.toolName, input: event.input });
|
|
3741
|
+
break;
|
|
3742
|
+
case 'tool_result':
|
|
3743
|
+
if (tools.length > 0) {
|
|
3744
|
+
const lastTool = tools[tools.length - 1];
|
|
3745
|
+
lastTool.result = event.content;
|
|
3746
|
+
lastTool.isError = event.isError;
|
|
3747
|
+
}
|
|
3748
|
+
break;
|
|
3749
|
+
}
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
|
|
3753
|
+
return { text, tools };
|
|
3754
|
+
}
|
|
3755
|
+
|
|
3756
|
+
_renderToolMarkdown(tools) {
|
|
3757
|
+
let md = `### Tools Used\n\n`;
|
|
3758
|
+
|
|
3759
|
+
for (const tool of tools) {
|
|
3760
|
+
const status = tool.isError ? '❌' : '✓';
|
|
3761
|
+
md += `- **${tool.name}** ${status}\n`;
|
|
3762
|
+
if (tool.input) {
|
|
3763
|
+
const inputStr = typeof tool.input === 'string' ? tool.input : JSON.stringify(tool.input);
|
|
3764
|
+
if (inputStr.length < 100) {
|
|
3765
|
+
md += ` - Input: \`${inputStr}\`\n`;
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
}
|
|
3769
|
+
|
|
3770
|
+
return `${md}\n`;
|
|
3771
|
+
}
|
|
3772
|
+
|
|
3773
|
+
_renderAgentMarkdown(agentId, agentMsgs, parseProviderChunk) {
|
|
3774
|
+
let md = `## Agent: ${agentId}\n\n`;
|
|
3775
|
+
const { text, tools } = this._extractAgentOutput(agentMsgs, parseProviderChunk);
|
|
3776
|
+
|
|
3777
|
+
if (text.trim()) {
|
|
3778
|
+
md += `### Output\n\n${text.trim()}\n\n`;
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
if (tools.length > 0) {
|
|
3782
|
+
md += this._renderToolMarkdown(tools);
|
|
3783
|
+
}
|
|
3784
|
+
|
|
3785
|
+
return md;
|
|
3786
|
+
}
|
|
3787
|
+
|
|
3788
|
+
_renderCriteriaMarkdown(criteriaResults) {
|
|
3789
|
+
if (!Array.isArray(criteriaResults)) {
|
|
3790
|
+
return '';
|
|
3791
|
+
}
|
|
3792
|
+
|
|
3793
|
+
let md = '';
|
|
3794
|
+
const cannotValidateYet = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE_YET');
|
|
3795
|
+
if (cannotValidateYet.length > 0) {
|
|
3796
|
+
md += `**❌ Cannot Validate Yet (${cannotValidateYet.length} criteria - work incomplete):**\n`;
|
|
3797
|
+
for (const cv of cannotValidateYet) {
|
|
3798
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
3799
|
+
}
|
|
3800
|
+
md += '\n';
|
|
3801
|
+
}
|
|
3802
|
+
|
|
3803
|
+
const cannotValidate = criteriaResults.filter((c) => c.status === 'CANNOT_VALIDATE');
|
|
3804
|
+
if (cannotValidate.length > 0) {
|
|
3805
|
+
md += `**⚠️ Could Not Validate (${cannotValidate.length} criteria - permanent):**\n`;
|
|
3806
|
+
for (const cv of cannotValidate) {
|
|
3807
|
+
md += `- ${cv.id}: ${cv.reason || 'No reason provided'}\n`;
|
|
3808
|
+
}
|
|
3809
|
+
md += '\n';
|
|
3810
|
+
}
|
|
3811
|
+
|
|
3812
|
+
return md;
|
|
3813
|
+
}
|
|
3814
|
+
|
|
3815
|
+
_renderValidationEntryMarkdown(validation) {
|
|
3816
|
+
const data = validation.content?.data || {};
|
|
3817
|
+
const approved = data.approved === true || data.approved === 'true';
|
|
3818
|
+
const icon = approved ? '✅' : '❌';
|
|
3819
|
+
let md = `### ${validation.sender} ${icon}\n\n`;
|
|
3820
|
+
|
|
3821
|
+
if (data.summary) {
|
|
3822
|
+
md += `${data.summary}\n\n`;
|
|
3823
|
+
}
|
|
3824
|
+
|
|
3825
|
+
if (!approved && data.issues) {
|
|
3826
|
+
const issues = typeof data.issues === 'string' ? JSON.parse(data.issues) : data.issues;
|
|
3827
|
+
if (Array.isArray(issues) && issues.length > 0) {
|
|
3828
|
+
md += `**Issues:**\n`;
|
|
3829
|
+
for (const issue of issues) {
|
|
3830
|
+
md += `- ${issue}\n`;
|
|
3831
|
+
}
|
|
3832
|
+
md += '\n';
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
|
|
3836
|
+
md += this._renderCriteriaMarkdown(data.criteriaResults);
|
|
3837
|
+
return md;
|
|
3838
|
+
}
|
|
3839
|
+
|
|
3840
|
+
_renderValidationMarkdown(messages) {
|
|
3841
|
+
const validations = messages.filter((m) => m.topic === 'VALIDATION_RESULT');
|
|
3842
|
+
if (validations.length === 0) {
|
|
3843
|
+
return '';
|
|
3844
|
+
}
|
|
3845
|
+
|
|
3846
|
+
let md = `## Validation Results\n\n`;
|
|
3847
|
+
for (const validation of validations) {
|
|
3848
|
+
md += this._renderValidationEntryMarkdown(validation);
|
|
3849
|
+
}
|
|
3850
|
+
|
|
3851
|
+
return md;
|
|
3852
|
+
}
|
|
3853
|
+
|
|
3854
|
+
/**
|
|
3855
|
+
* Validate cluster configuration (delegates to config-validator module)
|
|
3856
|
+
* @param {Object} config - Cluster configuration
|
|
3857
|
+
* @param {Object} options - Validation options
|
|
3858
|
+
* @param {boolean} options.strict - Treat warnings as errors (default: false)
|
|
3859
|
+
* @returns {Object} { valid: Boolean, errors: Array, warnings: Array }
|
|
3860
|
+
*/
|
|
3861
|
+
validateConfig(config, options = {}) {
|
|
3862
|
+
const result = configValidator.validateConfig(config);
|
|
3863
|
+
|
|
3864
|
+
// In strict mode, warnings become errors
|
|
3865
|
+
if (options.strict && result.warnings.length > 0) {
|
|
3866
|
+
result.errors.push(...result.warnings.map((w) => `[strict] ${w}`));
|
|
3867
|
+
result.valid = false;
|
|
3868
|
+
}
|
|
3869
|
+
|
|
3870
|
+
return result;
|
|
3871
|
+
}
|
|
3872
|
+
|
|
3873
|
+
/**
|
|
3874
|
+
* Load cluster configuration from file
|
|
3875
|
+
* @param {String} configPath - Path to config JSON file
|
|
3876
|
+
* @param {Object} options - Load options
|
|
3877
|
+
* @param {boolean} options.strict - Treat warnings as errors
|
|
3878
|
+
* @returns {Object} Parsed configuration
|
|
3879
|
+
*/
|
|
3880
|
+
loadConfig(configPath, options = {}) {
|
|
3881
|
+
const fullPath = path.resolve(configPath);
|
|
3882
|
+
const content = fs.readFileSync(fullPath, 'utf8');
|
|
3883
|
+
const config = JSON.parse(content);
|
|
3884
|
+
|
|
3885
|
+
const validation = this.validateConfig(config, options);
|
|
3886
|
+
|
|
3887
|
+
// Show warnings (but don't fail unless strict mode)
|
|
3888
|
+
if (validation.warnings && validation.warnings.length > 0 && !this.quiet) {
|
|
3889
|
+
console.warn('\n⚠️ Configuration warnings:');
|
|
3890
|
+
for (const warning of validation.warnings) {
|
|
3891
|
+
console.warn(` ${warning}`);
|
|
3892
|
+
}
|
|
3893
|
+
console.warn('');
|
|
3894
|
+
}
|
|
3895
|
+
|
|
3896
|
+
if (!validation.valid) {
|
|
3897
|
+
const errorMsg = validation.errors.join('\n ');
|
|
3898
|
+
throw new Error(`Invalid config:\n ${errorMsg}`);
|
|
3899
|
+
}
|
|
3900
|
+
|
|
3901
|
+
return config;
|
|
3902
|
+
}
|
|
3903
|
+
|
|
3904
|
+
/**
|
|
3905
|
+
* Garbage-collect orphaned worktree directories and database files.
|
|
3906
|
+
*
|
|
3907
|
+
* Delegates to the standalone gc module, passing in-memory cluster IDs
|
|
3908
|
+
* as additional known IDs (clusters.json may not reflect recently spawned clusters).
|
|
3909
|
+
*
|
|
3910
|
+
* @param {object} [options]
|
|
3911
|
+
* @param {boolean} [options.dryRun=false] - If true, report but don't delete
|
|
3912
|
+
* @returns {{ orphanedWorktrees: string[], orphanedDbs: string[], errors: string[] }}
|
|
3913
|
+
*/
|
|
3914
|
+
gcWorktrees(options = {}) {
|
|
3915
|
+
const { gcOrphanedWorktrees } = require('./lib/gc');
|
|
3916
|
+
return gcOrphanedWorktrees({
|
|
3917
|
+
storageDir: this.storageDir,
|
|
3918
|
+
extraKnownIds: new Set(this.clusters.keys()),
|
|
3919
|
+
dryRun: options.dryRun || false,
|
|
3920
|
+
});
|
|
3921
|
+
}
|
|
3922
|
+
}
|
|
3923
|
+
|
|
3924
|
+
module.exports = Orchestrator;
|