@triflux/remote 10.50.2 → 10.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/cto/brief.mjs +0 -12
  2. package/cto/collect.mjs +0 -73
  3. package/cto/current.schema.json +0 -4
  4. package/cto/status.mjs +2 -50
  5. package/hub/index.mjs +0 -8
  6. package/hub/team/agent-map.json +0 -1
  7. package/hub/team/ansi.mjs +1 -35
  8. package/hub/team/backend.mjs +9 -145
  9. package/hub/team/cli/commands/attach.mjs +1 -17
  10. package/hub/team/cli/commands/control.mjs +12 -38
  11. package/hub/team/cli/commands/debug.mjs +1 -37
  12. package/hub/team/cli/commands/focus.mjs +5 -31
  13. package/hub/team/cli/commands/interrupt.mjs +6 -25
  14. package/hub/team/cli/commands/kill.mjs +1 -30
  15. package/hub/team/cli/commands/list.mjs +0 -16
  16. package/hub/team/cli/commands/send.mjs +6 -26
  17. package/hub/team/cli/commands/start/index.mjs +41 -133
  18. package/hub/team/cli/commands/start/parse-args.mjs +15 -40
  19. package/hub/team/cli/commands/start/start-headless.mjs +138 -105
  20. package/hub/team/cli/commands/start/start-mux.mjs +0 -3
  21. package/hub/team/cli/commands/status.mjs +2 -57
  22. package/hub/team/cli/commands/stop.mjs +14 -19
  23. package/hub/team/cli/help.mjs +4 -5
  24. package/hub/team/cli/manifest.mjs +5 -1
  25. package/hub/team/cli/render.mjs +0 -12
  26. package/hub/team/cli/services/hub-client.mjs +2 -88
  27. package/hub/team/cli/services/runtime-mode.mjs +27 -98
  28. package/hub/team/handoff.mjs +3 -34
  29. package/hub/team/headless.mjs +13 -754
  30. package/hub/team/orchestrator.mjs +4 -30
  31. package/hub/team/pane.mjs +6 -28
  32. package/hub/team/psmux.mjs +0 -1
  33. package/hub/team/remote-session.mjs +1 -2
  34. package/hub/team/session.mjs +7 -91
  35. package/hub/team/swarm-intent.mjs +0 -198
  36. package/hub/team/tui-core.mjs +1 -1
  37. package/hub/team/tui-viewer.mjs +2 -2
  38. package/hub/team/tui.mjs +8 -269
  39. package/hub/team/wt-manager.mjs +2 -2
  40. package/hub/workers/factory.mjs +4 -37
  41. package/hub/workers/lib/jsonrpc-ws-uds.mjs +0 -2
  42. package/package.json +1 -1
  43. package/scripts/lib/agent-route-policy.mjs +162 -14
  44. package/scripts/lib/claudemd-scanner.mjs +2 -199
  45. package/scripts/lib/cli-claude.mjs +0 -2
  46. package/scripts/lib/context.mjs +3 -19
  47. package/scripts/lib/env-probe.mjs +5 -64
  48. package/scripts/lib/gemini-profiles.mjs +2 -69
  49. package/scripts/lib/machine-profile.mjs +3 -17
  50. package/scripts/lib/mcp-filter.mjs +109 -53
  51. package/scripts/lib/mcp-health.mjs +2 -2
  52. package/scripts/lib/mcp-server-catalog.mjs +2 -2
  53. package/scripts/lib/psmux-info.mjs +11 -59
  54. package/scripts/lib/skill-template.mjs +0 -245
  55. package/hub/team/build-worker-prompt.mjs +0 -92
  56. package/hub/team/check-mcp-hub.mjs +0 -49
  57. package/hub/team/claude-native-bridge.mjs +0 -987
  58. package/hub/team/cli/commands/start/start-in-process.mjs +0 -50
  59. package/hub/team/cli/commands/start/start-wt.mjs +0 -86
  60. package/hub/team/cli/services/native-control.mjs +0 -145
  61. package/hub/team/codex-review.mjs +0 -520
  62. package/hub/team/conductor-mesh-bridge.mjs +0 -122
  63. package/hub/team/conductor.mjs +0 -1529
  64. package/hub/team/consensus-meta.mjs +0 -176
  65. package/hub/team/cto-pull.mjs +0 -31
  66. package/hub/team/daemon-pty-tmux-bridge.mjs +0 -146
  67. package/hub/team/dashboard.mjs +0 -322
  68. package/hub/team/event-log.mjs +0 -84
  69. package/hub/team/execution-mode.mjs +0 -324
  70. package/hub/team/extract-completion-payload.mjs +0 -110
  71. package/hub/team/headless-bridge-session.mjs +0 -17
  72. package/hub/team/health-probe.mjs +0 -435
  73. package/hub/team/interactive-native-launcher.mjs +0 -119
  74. package/hub/team/interactive-native-worker.mjs +0 -80
  75. package/hub/team/interactive-tui-transport.mjs +0 -479
  76. package/hub/team/launcher-template.mjs +0 -144
  77. package/hub/team/mcp-selector.mjs +0 -145
  78. package/hub/team/native-supervisor.mjs +0 -460
  79. package/hub/team/native.mjs +0 -727
  80. package/hub/team/recovery-store.mjs +0 -149
  81. package/hub/team/remote-probe.mjs +0 -292
  82. package/hub/team/remote-watcher.mjs +0 -506
  83. package/hub/team/runtime-strategy.mjs +0 -200
  84. package/hub/team/schemas/worker-completion.json +0 -39
  85. package/hub/team/sentinel-capture.mjs +0 -127
  86. package/hub/team/swarm-cli.mjs +0 -390
  87. package/hub/team/swarm-hypervisor.mjs +0 -2206
  88. package/hub/team/swarm-locks.mjs +0 -335
  89. package/hub/team/swarm-planner.mjs +0 -371
  90. package/hub/team/swarm-preflight.mjs +0 -366
  91. package/hub/team/swarm-reconciler.mjs +0 -242
  92. package/hub/team/tui-remote-adapter.mjs +0 -377
  93. package/hub/team/worker-completion-validator.mjs +0 -79
  94. package/hub/team/worktree-lifecycle.mjs +0 -639
  95. package/hub/team/wt-templates.json +0 -43
  96. package/hub/workers/antigravity-route-worker.mjs +0 -179
  97. package/scripts/lib/dynamic-route-cli.mjs +0 -107
@@ -1,2206 +0,0 @@
1
- // hub/team/swarm-hypervisor.mjs — Multi-model swarm orchestration hypervisor
2
- // Consumes a SwarmPlan (from swarm-planner.mjs) and orchestrates parallel
3
- // conductor sessions with file-lease enforcement, result validation,
4
- // and ordered integration.
5
- //
6
- // Failure modes handled:
7
- // F1: Worker crash → conductor auto-restart (maxRestarts)
8
- // F2: Rate limit → account-broker cooldown + fallback agent
9
- // F3: Stall → health probe L1 detection + kill + restart
10
- // F4: File lease violation → revert worker changes, flag shard as failed
11
- // F5: Merge conflict → retry integration with conflict resolution
12
-
13
- import { execFile } from "node:child_process";
14
- import crypto from "node:crypto";
15
- import { EventEmitter } from "node:events";
16
- import { existsSync, mkdirSync, readFileSync } from "node:fs";
17
- import { join } from "node:path";
18
- import {
19
- createDynamicRouter,
20
- isDynamicRoutingEnabled,
21
- } from "@triflux/core/hub/dynamic-routing-engine.mjs";
22
- import { cleanupShardProcesses } from "@triflux/core/hub/lib/process-utils.mjs";
23
- import { getHostConfig } from "@triflux/core/hub/lib/ssh-command.mjs";
24
- import { DEFAULT_SWARM_LOCK_TTL_MS } from "@triflux/core/hub/lib/timeout-defaults.mjs";
25
- import { buildWorkerPrompt } from "./build-worker-prompt.mjs";
26
- import { registerSwarmShard } from "./claude-native-bridge.mjs";
27
- import { createConductor, STATES } from "./conductor.mjs";
28
- import { ensureConductorRegistry } from "./conductor-registry.mjs";
29
- import { createEventLog } from "./event-log.mjs";
30
- import { preserveWorktreePatch } from "./recovery-store.mjs";
31
- import { probeRemoteEnv, resolveRemoteDir } from "./remote-session.mjs";
32
- import { createSwarmLocks } from "./swarm-locks.mjs";
33
- import { validateWorkerCompletion } from "./worker-completion-validator.mjs";
34
- import {
35
- cleanupWorktree,
36
- EXPECTED_WORKTREE_DELETIONS,
37
- ensureWorktree,
38
- extractDirtyFiles,
39
- fetchRemoteShard,
40
- prepareIntegrationBranch,
41
- pruneWorktree,
42
- rebaseShardOntoIntegration,
43
- } from "./worktree-lifecycle.mjs";
44
-
45
- let ensureHubAliveFn = null;
46
- let getHubInfoFn = null;
47
- try {
48
- ({ ensureHubAlive: ensureHubAliveFn, getHubInfo: getHubInfoFn } =
49
- await import("./cli/services/hub-client.mjs"));
50
- } catch {
51
- // hub-client 미설치 환경 — Hub 수동 관리 필요
52
- }
53
-
54
- let importedCreateRegistry = null;
55
- let meshRegistryImportError = null;
56
- try {
57
- ({ createRegistry: importedCreateRegistry } = await import(
58
- "@triflux/core/mesh/mesh-registry.mjs"
59
- ));
60
- } catch (err) {
61
- meshRegistryImportError = err;
62
- }
63
-
64
- // ── Swarm states ──────────────────────────────────────────────
65
-
66
- export const SWARM_STATES = Object.freeze({
67
- PLANNING: "planning",
68
- LAUNCHING: "launching",
69
- RUNNING: "running",
70
- INTEGRATING: "integrating",
71
- VALIDATING: "validating",
72
- COMPLETED: "completed",
73
- FAILED: "failed",
74
- });
75
-
76
- // ── Failure mode classification ───────────────────────────────
77
-
78
- const FAILURE_MODES = Object.freeze({
79
- F1_CRASH: "F1_crash",
80
- F2_RATE_LIMIT: "F2_rate_limit",
81
- F3_STALL: "F3_stall",
82
- F4_LEASE_VIOLATION: "F4_lease_violation",
83
- F5_MERGE_CONFLICT: "F5_merge_conflict",
84
- F6_NO_COMMIT: "F6_no_commit",
85
- F7_WORKER_DID_NOT_COMMIT: "F7_worker_did_not_commit",
86
- });
87
-
88
- const FALLBACK_AGENTS = Object.freeze({
89
- codex: "gemini",
90
- gemini: "codex",
91
- claude: "codex",
92
- });
93
-
94
- function resolveRedundantAgent(shard) {
95
- const mode = shard?.redundancy;
96
- if (!mode) return null;
97
- if (mode === "same-cli") return shard.agent;
98
- if (mode === true || mode === "fallback") {
99
- return FALLBACK_AGENTS[shard.agent] || shard.agent;
100
- }
101
- return null;
102
- }
103
-
104
- function createNoopRegistry() {
105
- return Object.freeze({
106
- register() {},
107
- unregister() {},
108
- discover() {
109
- return [];
110
- },
111
- getAgent() {
112
- return null;
113
- },
114
- listAll() {
115
- return [];
116
- },
117
- clear() {},
118
- });
119
- }
120
-
121
- function createSharedRegistry(factory) {
122
- if (typeof factory !== "function") {
123
- return {
124
- registry: createNoopRegistry(),
125
- fallbackReason: meshRegistryImportError
126
- ? `mesh_import_failed:${meshRegistryImportError.message}`
127
- : "mesh_registry_unavailable",
128
- };
129
- }
130
-
131
- try {
132
- return { registry: factory(), fallbackReason: null };
133
- } catch (err) {
134
- return {
135
- registry: createNoopRegistry(),
136
- fallbackReason: `mesh_registry_init_failed:${err.message}`,
137
- };
138
- }
139
- }
140
-
141
- function hasOwnDep(deps, key) {
142
- return Object.hasOwn(deps, key);
143
- }
144
-
145
- function formatHubHostForUrl(host) {
146
- return host.includes(":") ? `[${host}]` : host;
147
- }
148
-
149
- function resolveHubStatusUrl(hub) {
150
- if (!hub) return null;
151
-
152
- if (typeof hub.url === "string" && hub.url.trim()) {
153
- try {
154
- const url = new URL(hub.url);
155
- url.pathname = "/status";
156
- url.search = "";
157
- url.hash = "";
158
- return url.toString();
159
- } catch {
160
- return null;
161
- }
162
- }
163
-
164
- const host = typeof hub.host === "string" ? hub.host.trim() : "";
165
- const port = Number(hub.port);
166
- if (!host || !Number.isFinite(port) || port <= 0) return null;
167
- return `http://${formatHubHostForUrl(host)}:${port}/status`;
168
- }
169
-
170
- function isTruthyConfigValue(value) {
171
- if (typeof value === "boolean") return value;
172
- return /^(true|yes|1|interactive)$/i.test(String(value || "").trim());
173
- }
174
-
175
- function isInteractiveNativeBridgeShard(shard) {
176
- return (
177
- isTruthyConfigValue(shard?.interactive) ||
178
- isTruthyConfigValue(shard?.nativeBridgeInteractive) ||
179
- String(shard?.workerType || "").toLowerCase() === "interactive"
180
- );
181
- }
182
-
183
- /**
184
- * Create a swarm hypervisor.
185
- * @param {object} opts
186
- * @param {string} opts.workdir — repository root / working directory
187
- * @param {string} opts.logsDir — base directory for all logs
188
- * @param {string} [opts.baseBranch='main'] — base branch for shard worktrees
189
- * @param {string} [opts.runId=`swarm-${Date.now()}`] — logical swarm run id
190
- * @param {number} [opts.maxRestarts=2] — per-shard max restarts
191
- * @param {boolean} [opts.nativeBridge=false] — expose local shards in Claude agents
192
- * @param {number} [opts.graceMs=10000] — conductor shutdown grace period
193
- * @param {boolean} [opts.keepFailedWorktrees=false] — preserve failed shard worktrees for debugging
194
- * @param {object} [opts.probeOpts] — health probe overrides
195
- * @param {object} [opts.deps] — dependency injection for testing
196
- * @returns {SwarmHypervisor}
197
- */
198
- export function createSwarmHypervisor(opts) {
199
- const {
200
- workdir,
201
- logsDir,
202
- baseBranch = "main",
203
- runId = `swarm-${Date.now()}`,
204
- maxRestarts = 2,
205
- nativeBridge = false,
206
- graceMs = 10_000,
207
- keepFailedWorktrees = false,
208
- probeOpts = {},
209
- _deps = {},
210
- } = opts;
211
-
212
- if (!workdir) throw new Error("workdir is required");
213
- if (!logsDir) throw new Error("logsDir is required");
214
-
215
- mkdirSync(logsDir, { recursive: true });
216
- ensureConductorRegistry();
217
-
218
- const createConductorImpl = _deps.createConductor || createConductor;
219
- const createRegistryImpl = _deps.createRegistry || importedCreateRegistry;
220
- const ensureWorktreeImpl = _deps.ensureWorktree || ensureWorktree;
221
- const prepareIntegrationBranchImpl =
222
- _deps.prepareIntegrationBranch || prepareIntegrationBranch;
223
- const rebaseShardOntoIntegrationImpl =
224
- _deps.rebaseShardOntoIntegration || rebaseShardOntoIntegration;
225
- const cleanupWorktreeImpl =
226
- _deps.cleanupWorktree || cleanupWorktree || pruneWorktree;
227
- const cleanupShardProcessesImpl =
228
- _deps.cleanupShardProcesses || cleanupShardProcesses;
229
- const registerSwarmShardImpl = _deps.registerSwarmShard || registerSwarmShard;
230
- const preserveWorktreePatchImpl =
231
- _deps.preserveWorktreePatch || preserveWorktreePatch;
232
- const ensureHubAliveImpl = hasOwnDep(_deps, "ensureHubAlive")
233
- ? _deps.ensureHubAlive
234
- : ensureHubAliveFn;
235
- const getHubInfoImpl = hasOwnDep(_deps, "getHubInfo")
236
- ? _deps.getHubInfo
237
- : getHubInfoFn;
238
- const emitter = new EventEmitter();
239
- const eventLog = createEventLog(join(logsDir, "swarm-events.jsonl"));
240
- const { registry: sharedRegistry, fallbackReason: meshRegistryFallback } =
241
- createSharedRegistry(createRegistryImpl);
242
-
243
- let state = SWARM_STATES.PLANNING;
244
- let plan = null;
245
- let lockManager = null;
246
-
247
- /** @type {Map<string, { conductor, shardConfig, result, status }>} */
248
- const workers = new Map();
249
-
250
- /** @type {Map<string, { conductor, shardConfig }>} redundant workers for critical shards */
251
- const redundantWorkers = new Map();
252
-
253
- /** @type {Set<string>} shards that have fully completed (not just launched) */
254
- const completedShards = new Set();
255
- const cleanedWorktreePaths = new Set();
256
- const processCleanupTotals = {
257
- totalKilled: 0,
258
- byCategory: {},
259
- };
260
-
261
- const results = new Map(); // shardName → validated result
262
- const failures = new Map(); // shardName → failure info
263
- let integrationResult = null;
264
- let resolveIntegrationPromise = null;
265
- let integrationPromiseState = {
266
- state: "idle",
267
- startedAt: null,
268
- settledAt: null,
269
- partial: false,
270
- integrated: [],
271
- failed: [],
272
- integrationFailures: [],
273
- skipped: [],
274
- integrationBranch: null,
275
- report: [],
276
- processCleanup: {
277
- totalKilled: 0,
278
- byCategory: {},
279
- },
280
- error: null,
281
- };
282
- const integrationPromise = new Promise((resolve) => {
283
- resolveIntegrationPromise = resolve;
284
- });
285
-
286
- if (meshRegistryFallback) {
287
- eventLog.append("mesh_registry_fallback", { reason: meshRegistryFallback });
288
- }
289
-
290
- // ── State machine ───────────────────────────────────────────
291
-
292
- function setState(next, reason = "") {
293
- const prev = state;
294
- state = next;
295
- eventLog.append("swarm_state", { from: prev, to: next, reason });
296
- emitter.emit("stateChange", { from: prev, to: next, reason });
297
- // Best-effort phase tracking via phase-manager
298
- void writeSwarmPhaseSafely(
299
- plan?.runId || plan?.id || `swarm-${Date.now()}`,
300
- next,
301
- );
302
- }
303
-
304
- async function writeSwarmPhaseSafely(runId, swarmState) {
305
- if (!runId) return;
306
- const mapping = {
307
- [SWARM_STATES.LAUNCHING]: ["Execution", "active"],
308
- [SWARM_STATES.RUNNING]: ["Execution", "active"],
309
- [SWARM_STATES.INTEGRATING]: ["Validation", "active"],
310
- [SWARM_STATES.VALIDATING]: ["Validation", "active"],
311
- [SWARM_STATES.COMPLETED]: ["Validation", "complete"],
312
- [SWARM_STATES.FAILED]: ["Validation", "failed"],
313
- };
314
- const target = mapping[swarmState];
315
- if (!target) return;
316
- try {
317
- const mod = await import("@triflux/core/hub/lib/phase-manager.mjs");
318
- if (typeof mod?.writePhase === "function") {
319
- await mod.writePhase(runId, target[0], target[1]);
320
- }
321
- if (typeof mod?.syncToGstack === "function") {
322
- const slug = process.cwd().split(/[\\/]/).pop();
323
- await mod.syncToGstack(runId, slug);
324
- }
325
- } catch {
326
- /* silent — phase tracking is best-effort */
327
- }
328
- }
329
-
330
- function markIntegrationPromisePending() {
331
- if (integrationPromiseState.state !== "idle") return;
332
- integrationPromiseState = {
333
- ...integrationPromiseState,
334
- state: "pending",
335
- startedAt: Date.now(),
336
- };
337
- }
338
-
339
- function settleIntegrationPromise(payload) {
340
- if (integrationPromiseState.state === "fulfilled" && integrationResult) {
341
- return integrationResult;
342
- }
343
-
344
- const processCleanup = payload.processCleanup || getProcessCleanupSummary();
345
-
346
- integrationResult = Object.freeze({
347
- integrated: Object.freeze([...(payload.integrated || [])]),
348
- failed: Object.freeze([...(payload.failed || [])]),
349
- integrationFailures: Object.freeze([
350
- ...(payload.integrationFailures || []),
351
- ]),
352
- skipped: Object.freeze([...(payload.skipped || [])]),
353
- integrationBranch: payload.integrationBranch || null,
354
- report: Object.freeze(
355
- (payload.report || []).map((row) => Object.freeze({ ...row })),
356
- ),
357
- results: Object.freeze([...(payload.results || [])]),
358
- processCleanup: Object.freeze({
359
- totalKilled: processCleanup.totalKilled || 0,
360
- byCategory: Object.freeze({ ...(processCleanup.byCategory || {}) }),
361
- }),
362
- partial:
363
- payload.partial ??
364
- (Array.isArray(payload.failed) && payload.failed.length > 0),
365
- error: payload.error || null,
366
- });
367
-
368
- integrationPromiseState = {
369
- state: "fulfilled",
370
- startedAt: integrationPromiseState.startedAt ?? Date.now(),
371
- settledAt: Date.now(),
372
- partial: integrationResult.partial,
373
- integrated: [...integrationResult.integrated],
374
- failed: [...integrationResult.failed],
375
- integrationFailures: [...integrationResult.integrationFailures],
376
- skipped: [...integrationResult.skipped],
377
- integrationBranch: integrationResult.integrationBranch,
378
- report: integrationResult.report.map((row) => ({ ...row })),
379
- processCleanup: integrationResult.processCleanup,
380
- error: integrationResult.error,
381
- };
382
-
383
- resolveIntegrationPromise?.(integrationResult);
384
- return integrationResult;
385
- }
386
-
387
- function getProcessCleanupSummary() {
388
- return {
389
- totalKilled: processCleanupTotals.totalKilled,
390
- byCategory: { ...processCleanupTotals.byCategory },
391
- };
392
- }
393
-
394
- function recordProcessCleanup(result) {
395
- if (!result) return getProcessCleanupSummary();
396
-
397
- const killed = Number(result.killed ?? result.totalKilled ?? 0);
398
- if (Number.isFinite(killed) && killed > 0) {
399
- processCleanupTotals.totalKilled += killed;
400
- }
401
-
402
- for (const [category, count] of Object.entries(result.byCategory || {})) {
403
- const numeric = Number(count);
404
- if (!Number.isFinite(numeric) || numeric <= 0) continue;
405
- processCleanupTotals.byCategory[category] =
406
- (processCleanupTotals.byCategory[category] || 0) + numeric;
407
- }
408
-
409
- return getProcessCleanupSummary();
410
- }
411
-
412
- function normalizeIntegrationReportRow(row) {
413
- return {
414
- shard: row.shard,
415
- strategy: row.strategy || "failed",
416
- commits:
417
- Number.isFinite(Number(row.commits)) && row.commits !== null
418
- ? Number(row.commits)
419
- : null,
420
- finalSha: row.finalSha || null,
421
- notes: row.notes || "",
422
- recoveryPatch: row.recoveryPatch || null,
423
- };
424
- }
425
-
426
- function git(args, cwd = workdir) {
427
- return new Promise((resolve, reject) => {
428
- execFile(
429
- "git",
430
- args,
431
- { cwd, windowsHide: true, timeout: 30_000 },
432
- (err, stdout, stderr) => {
433
- if (err) {
434
- reject(
435
- new Error(
436
- `git ${args[0]} failed: ${stderr?.trim() || err.message}`,
437
- ),
438
- );
439
- return;
440
- }
441
- resolve(stdout.trim());
442
- },
443
- );
444
- });
445
- }
446
-
447
- // Injectable git seam (#394). Defaults to the execFile closure above; tests
448
- // override via `_deps.git` to drive the redundant-win commit-evidence gate
449
- // deterministically without provisioning a real worktree.
450
- const gitRun = _deps.git || git;
451
-
452
- function computeAuthoritativeStatus(shardName, workerEntry, sessions) {
453
- const failureInfo = failures.get(shardName) || null;
454
- if (failureInfo) {
455
- return {
456
- status: "failed",
457
- reason: failureInfo.mode || failureInfo.reason || "failed",
458
- };
459
- }
460
-
461
- if (results.has(shardName)) {
462
- return { status: "done", reason: "integrated" };
463
- }
464
-
465
- if (!sessions.length) {
466
- if (completedShards.has(shardName)) {
467
- return { status: "done", reason: "awaiting_integration" };
468
- }
469
- return { status: "running", reason: "no_sessions" };
470
- }
471
-
472
- const states = sessions.map((session) => session.state);
473
- if (states.every((stateValue) => stateValue === STATES.COMPLETED)) {
474
- return { status: "done", reason: "awaiting_integration" };
475
- }
476
- if (states.some((stateValue) => stateValue === STATES.INPUT_WAIT)) {
477
- return { status: "blocked", reason: "user_input" };
478
- }
479
- if (states.some((stateValue) => stateValue === STATES.STALLED)) {
480
- return { status: "stalled", reason: "health_probe_stall" };
481
- }
482
- if (
483
- states.some(
484
- (stateValue) =>
485
- stateValue === STATES.FAILED || stateValue === STATES.DEAD,
486
- )
487
- ) {
488
- return { status: "failed", reason: "session_terminal" };
489
- }
490
- if (
491
- states.some(
492
- (stateValue) =>
493
- stateValue === STATES.STARTING ||
494
- stateValue === STATES.RESTARTING ||
495
- stateValue === STATES.INIT,
496
- )
497
- ) {
498
- return { status: "running", reason: "starting" };
499
- }
500
-
501
- return { status: "running", reason: "healthy" };
502
- }
503
-
504
- async function collectCommitEvidence(worker, integrationBranch) {
505
- const branchName = worker?.branchName || null;
506
- const evidence = {
507
- branchName,
508
- integrationBranch,
509
- commitsAhead: 0,
510
- dirty: false,
511
- dirtyFiles: [],
512
- headCommit: null,
513
- ok: false,
514
- error: null,
515
- };
516
-
517
- if (!branchName) {
518
- evidence.error = "missing_branch_name";
519
- return evidence;
520
- }
521
-
522
- try {
523
- evidence.commitsAhead =
524
- Number.parseInt(
525
- await gitRun([
526
- "rev-list",
527
- "--count",
528
- `${integrationBranch}..${branchName}`,
529
- ]),
530
- 10,
531
- ) || 0;
532
- } catch (err) {
533
- evidence.error = err.message;
534
- return evidence;
535
- }
536
-
537
- try {
538
- evidence.headCommit = await gitRun(["rev-parse", branchName]);
539
- } catch {
540
- /* best-effort */
541
- }
542
-
543
- if (worker?.worktreePath && !worker?.shardConfig?.host) {
544
- try {
545
- const rawStatus = await gitRun(
546
- ["status", "--short"],
547
- worker.worktreePath,
548
- );
549
- // Only explicitly expected lifecycle deletions are filtered. Plugin
550
- // metadata deletions remain dirty because Codex workers need them.
551
- evidence.dirtyFiles = extractDirtyFiles(
552
- rawStatus,
553
- EXPECTED_WORKTREE_DELETIONS,
554
- );
555
- evidence.dirty = evidence.dirtyFiles.length > 0;
556
- } catch (err) {
557
- evidence.error = evidence.error || err.message;
558
- return evidence;
559
- }
560
- }
561
-
562
- evidence.ok = evidence.commitsAhead > 0 && evidence.dirty === false;
563
- return evidence;
564
- }
565
-
566
- // ── Worker lifecycle ────────────────────────────────────────
567
-
568
- function safeSessionPart(value) {
569
- return String(value || "unknown")
570
- .replace(/[^a-zA-Z0-9._-]+/g, "-")
571
- .replace(/^-+|-+$/g, "")
572
- .slice(0, 80);
573
- }
574
-
575
- function buildSwarmSessionId(shard) {
576
- const short8 = crypto.randomUUID().replace(/-/g, "").slice(0, 8);
577
- return `swarm-${safeSessionPart(runId)}-${safeSessionPart(shard.name)}-${short8}`;
578
- }
579
-
580
- function getShardDisplayPosition(shard) {
581
- const shardIndex = plan?.shards?.findIndex(
582
- (item) => item.name === shard.name,
583
- );
584
- if (!plan?.shards?.length || shardIndex < 0) return {};
585
- return {
586
- shardIndex: shardIndex + 1,
587
- shardCount: plan.shards.length,
588
- };
589
- }
590
-
591
- async function closeNativeBridgeRegistration(worker, shardName, reason) {
592
- if (worker?.nativeBridgeClosePromise) {
593
- await worker.nativeBridgeClosePromise;
594
- return;
595
- }
596
- const registration = worker?.nativeBridgeRegistration;
597
- if (!registration || typeof registration.close !== "function") return;
598
- worker.nativeBridgeClosePromise = (async () => {
599
- let closed = false;
600
- try {
601
- await registration.close();
602
- closed = true;
603
- eventLog.append("native_bridge_registration_closed", {
604
- shard: shardName,
605
- reason,
606
- });
607
- } catch (err) {
608
- eventLog.append("native_bridge_registration_close_failed", {
609
- shard: shardName,
610
- reason,
611
- error: err.message,
612
- });
613
- } finally {
614
- worker.nativeBridgeClosePromise = null;
615
- if (closed) worker.nativeBridgeRegistration = null;
616
- }
617
- })();
618
- await worker.nativeBridgeClosePromise;
619
- }
620
-
621
- async function maybeRegisterNativeBridgeShard(shard, sessionConfig) {
622
- if (!nativeBridge) return null;
623
- try {
624
- const interactive = !shard.host && isInteractiveNativeBridgeShard(shard);
625
- const registration = await registerSwarmShardImpl({
626
- sessionId: sessionConfig.id,
627
- cli: shard.agent,
628
- role: shard.role || "worker",
629
- swarmName: runId,
630
- ...getShardDisplayPosition(shard),
631
- shardName: shard.name,
632
- cwd: interactive
633
- ? shard.worktreePath || sessionConfig.workdir || workdir
634
- : workdir,
635
- host: shard.host || "local",
636
- interactive,
637
- _deps: {
638
- warn(message) {
639
- emitter.emit("warning", {
640
- type: "native_bridge_registration_skipped",
641
- shard: shard.name,
642
- host: shard.host || "local",
643
- message,
644
- });
645
- },
646
- },
647
- });
648
- eventLog.append(
649
- registration?.skipped
650
- ? "native_bridge_registration_skipped"
651
- : "native_bridge_registration",
652
- {
653
- shard: shard.name,
654
- sessionId: sessionConfig.id,
655
- host: shard.host || "local",
656
- displayName: registration?.displayName || null,
657
- },
658
- );
659
- return registration;
660
- } catch (err) {
661
- eventLog.append("native_bridge_registration_failed", {
662
- shard: shard.name,
663
- sessionId: sessionConfig.id,
664
- host: shard.host || "local",
665
- error: err.message,
666
- });
667
- emitter.emit("warning", {
668
- type: "native_bridge_registration_failed",
669
- shard: shard.name,
670
- host: shard.host || "local",
671
- error: err.message,
672
- });
673
- return null;
674
- }
675
- }
676
-
677
- function buildSessionConfig(shard) {
678
- const config = {
679
- id: buildSwarmSessionId(shard),
680
- agent: shard.agent,
681
- role: shard.role,
682
- // #125: append Completion Protocol appendix so workers emit a
683
- // sentinel-framed JSON payload that conductor can reliably capture.
684
- prompt: buildWorkerPrompt(shard.prompt, {
685
- leaseFiles: shard.files,
686
- worktreePath: shard.worktreePath,
687
- }),
688
- workdir: shard.worktreePath || workdir,
689
- mcpServers: shard.mcp,
690
- worktreePath: shard.worktreePath || null,
691
- branchName: shard.branchName || null,
692
- // #90: shard는 worktree 격리가 본질. main 직접 커밋 방지 가드 env 주입.
693
- branchGuard: true,
694
- };
695
-
696
- if (shard.worktreePath) {
697
- return shard.host
698
- ? {
699
- ...config,
700
- remote: true,
701
- host: shard.host,
702
- }
703
- : config;
704
- }
705
-
706
- // Remote shard: use host's default_dir from hosts.json
707
- if (shard.host && shard._remoteEnv) {
708
- const hostConfig = getHostConfig(shard.host);
709
- const remoteSrc = hostConfig?.default_dir || workdir;
710
- const remoteDir = resolveRemoteDir(remoteSrc, shard._remoteEnv);
711
- return {
712
- ...config,
713
- remote: true,
714
- host: shard.host,
715
- workdir: remoteDir,
716
- };
717
- }
718
-
719
- return config;
720
- }
721
-
722
- async function launchShard(shard, isRedundant = false) {
723
- // Clone frozen shard so we can attach mutable runtime state (_remoteEnv)
724
- shard = { ...shard };
725
- let nativeBridgeRegistration = null;
726
-
727
- const shardLogsDir = join(
728
- logsDir,
729
- isRedundant ? `${shard.name}-redundant` : shard.name,
730
- );
731
- mkdirSync(shardLogsDir, { recursive: true });
732
-
733
- try {
734
- // Remote shard: probe environment before conductor creation
735
- if (shard.host && !shard._remoteEnv) {
736
- try {
737
- shard._remoteEnv = probeRemoteEnv(shard.host);
738
- if (!shard._remoteEnv.claudePath) {
739
- eventLog.append("remote_probe_no_claude", {
740
- shard: shard.name,
741
- host: shard.host,
742
- });
743
- if (!isRedundant) {
744
- failures.set(shard.name, {
745
- mode: FAILURE_MODES.F1_CRASH,
746
- reason: `claude not found on ${shard.host}`,
747
- });
748
- }
749
- return null;
750
- }
751
- eventLog.append("remote_probe_ok", {
752
- shard: shard.name,
753
- host: shard.host,
754
- env: shard._remoteEnv,
755
- });
756
- } catch (err) {
757
- eventLog.append("remote_probe_failed", {
758
- shard: shard.name,
759
- host: shard.host,
760
- error: err.message,
761
- });
762
- if (!isRedundant) {
763
- failures.set(shard.name, {
764
- mode: FAILURE_MODES.F1_CRASH,
765
- reason: `remote probe failed: ${err.message}`,
766
- });
767
- }
768
- return null;
769
- }
770
- }
771
-
772
- // Acquire file leases
773
- if (!isRedundant) {
774
- const leaseResult = lockManager.acquire(shard.name, shard.files);
775
- if (!leaseResult.ok) {
776
- eventLog.append("lease_denied", {
777
- shard: shard.name,
778
- conflicts: leaseResult.conflicts,
779
- });
780
- failures.set(shard.name, {
781
- mode: FAILURE_MODES.F4_LEASE_VIOLATION,
782
- conflicts: leaseResult.conflicts,
783
- });
784
- return null;
785
- }
786
- }
787
-
788
- const worktreeMeta = await ensureWorktreeImpl({
789
- slug: isRedundant ? `${shard.name}-redundant` : shard.name,
790
- runId,
791
- rootDir: workdir,
792
- baseBranch,
793
- host: shard.host,
794
- remoteEnv: shard._remoteEnv,
795
- });
796
- shard.worktreePath = worktreeMeta.worktreePath;
797
- shard.branchName = worktreeMeta.branchName;
798
-
799
- const conductor = createConductorImpl({
800
- logsDir: shardLogsDir,
801
- maxRestarts,
802
- graceMs,
803
- probeOpts,
804
- meshRegistry: sharedRegistry,
805
- enableMesh: true,
806
- });
807
-
808
- // NOTE: conductor 는 `session.config.onCompleted({ sessionId })` 형태로
809
- // 세션별 콜백만 호출한다 (factory 옵션은 무시). 따라서 sessionConfig 에
810
- // 직접 주입해야 shardCompleted 이벤트가 발화된다. 2026-04-17 세션에서
811
- // follow-up swarm 이 여기서 hang 했던 원인.
812
- const sessionConfig = buildSessionConfig(shard);
813
- nativeBridgeRegistration = await maybeRegisterNativeBridgeShard(
814
- shard,
815
- sessionConfig,
816
- );
817
- sessionConfig.onCompleted = ({ sessionId, completionPayload } = {}) =>
818
- handleShardCompleted(
819
- shard.name,
820
- sessionId,
821
- isRedundant,
822
- completionPayload,
823
- );
824
- conductor.spawnSession(sessionConfig);
825
-
826
- eventLog.append("shard_launched", {
827
- shard: shard.name,
828
- agent: shard.agent,
829
- sessionId: sessionConfig.id,
830
- isRedundant,
831
- files: shard.files,
832
- remote: Boolean(shard.host),
833
- host: shard.host || null,
834
- worktreePath: sessionConfig.worktreePath,
835
- branchName: sessionConfig.branchName,
836
- });
837
-
838
- emitter.emit("shardLaunched", {
839
- shardName: shard.name,
840
- sessionId: sessionConfig.id,
841
- agent: shard.agent,
842
- isRedundant,
843
- remote: Boolean(shard.host),
844
- host: shard.host || null,
845
- });
846
-
847
- const entry = {
848
- conductor,
849
- shardConfig: shard,
850
- sessionConfig,
851
- startedAt: Date.now(),
852
- worktreePath: sessionConfig.worktreePath,
853
- branchName: sessionConfig.branchName,
854
- nativeBridgeRegistration,
855
- };
856
-
857
- if (isRedundant) {
858
- redundantWorkers.set(shard.name, entry);
859
- } else {
860
- workers.set(shard.name, entry);
861
- }
862
-
863
- // Listen for dead events (F1/F2/F3)
864
- conductor.on("dead", ({ sessionId, reason }) => {
865
- handleShardFailed(shard.name, sessionId, reason, isRedundant);
866
- });
867
-
868
- return entry;
869
- } catch (err) {
870
- if (nativeBridgeRegistration) {
871
- await closeNativeBridgeRegistration(
872
- { nativeBridgeRegistration },
873
- shard.name,
874
- "launch_failed",
875
- );
876
- }
877
- eventLog.append("shard_launch_failed", {
878
- shard: shard.name,
879
- isRedundant,
880
- error: err.message,
881
- });
882
- if (!isRedundant) {
883
- failures.set(shard.name, {
884
- mode: FAILURE_MODES.F1_CRASH,
885
- reason: err.message,
886
- });
887
- }
888
- if (!isRedundant) {
889
- lockManager.release(shard.name);
890
- }
891
- if (shard.worktreePath) {
892
- await maybeCleanupWorktree(
893
- shard.name,
894
- {
895
- worktreePath: shard.worktreePath,
896
- branchName: shard.branchName,
897
- },
898
- shard,
899
- {
900
- branchName: shard.branchName,
901
- force: true,
902
- failureReason: `launch_failed:${err.message}`,
903
- },
904
- );
905
- }
906
- return null;
907
- }
908
- }
909
-
910
- // ── Completion handling ─────────────────────────────────────
911
-
912
- function handleShardCompleted(
913
- shardName,
914
- sessionId,
915
- isRedundant,
916
- completionPayload,
917
- ) {
918
- eventLog.append("shard_completed", {
919
- shard: shardName,
920
- sessionId,
921
- isRedundant,
922
- hasPayload: completionPayload !== undefined,
923
- });
924
- const completedWorker = isRedundant
925
- ? redundantWorkers.get(shardName)
926
- : workers.get(shardName);
927
-
928
- // F7 — worker self-reported completion must include commits_made.
929
- // Only enforce when payload is actually provided; legacy workers that
930
- // don't emit a structured payload keep the F6 integration-time guard.
931
- if (completionPayload !== undefined) {
932
- const verdict = validateWorkerCompletion(completionPayload);
933
- if (!verdict.ok) {
934
- const reason = verdict.reason || "invalid_completion_payload";
935
-
936
- if (isRedundant) {
937
- eventLog.append("redundant_completion_rejected", {
938
- shard: shardName,
939
- sessionId,
940
- reason,
941
- });
942
- redundantWorkers.delete(shardName);
943
- if (completedWorker) {
944
- void closeNativeBridgeRegistration(
945
- completedWorker,
946
- shardName,
947
- "failed",
948
- );
949
- }
950
- checkAllShardsCompleted();
951
- return;
952
- }
953
-
954
- eventLog.append("no_worker_commit_report", {
955
- shard: shardName,
956
- sessionId,
957
- reason,
958
- });
959
- if (completedWorker) {
960
- void closeNativeBridgeRegistration(
961
- completedWorker,
962
- shardName,
963
- "failed",
964
- );
965
- }
966
- failures.set(shardName, {
967
- mode: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
968
- reason,
969
- sessionId,
970
- completionPayload,
971
- });
972
- lockManager.release(shardName);
973
-
974
- const worker = workers.get(shardName);
975
- const shard = plan?.shards.find((item) => item.name === shardName);
976
- if (worker && shard) {
977
- const cleanupPromise = maybeCleanupWorktree(
978
- shardName,
979
- worker,
980
- shard,
981
- {
982
- force: true,
983
- failureReason: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
984
- },
985
- );
986
- worker.cleanupPromise = cleanupPromise;
987
- void cleanupPromise
988
- .finally(() => {
989
- if (worker.cleanupPromise === cleanupPromise) {
990
- worker.cleanupPromise = null;
991
- }
992
- })
993
- .catch(() => {});
994
- }
995
-
996
- const redundant = redundantWorkers.get(shardName);
997
- if (redundant) {
998
- void redundant.conductor.shutdown("primary_failed_f7");
999
- }
1000
-
1001
- emitter.emit("shardFailed", {
1002
- shardName,
1003
- failureMode: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
1004
- reason,
1005
- });
1006
-
1007
- cascadeDependencyFailure(shardName);
1008
- checkAllShardsCompleted();
1009
- return;
1010
- }
1011
-
1012
- if (
1013
- completionPayload?.status === "failed" ||
1014
- completionPayload?.status === "blocked"
1015
- ) {
1016
- const detail =
1017
- typeof completionPayload.reason === "string" &&
1018
- completionPayload.reason.length > 0
1019
- ? completionPayload.reason
1020
- : "unspecified";
1021
- const reason = `worker_reported_${completionPayload.status}:${detail}`;
1022
-
1023
- if (isRedundant) {
1024
- eventLog.append("redundant_completion_rejected", {
1025
- shard: shardName,
1026
- sessionId,
1027
- reason,
1028
- });
1029
- redundantWorkers.delete(shardName);
1030
- if (completedWorker) {
1031
- void closeNativeBridgeRegistration(
1032
- completedWorker,
1033
- shardName,
1034
- "failed",
1035
- );
1036
- }
1037
- checkAllShardsCompleted();
1038
- return;
1039
- }
1040
-
1041
- eventLog.append("worker_reported_non_ok_completion", {
1042
- shard: shardName,
1043
- sessionId,
1044
- status: completionPayload.status,
1045
- reason,
1046
- });
1047
- if (completedWorker) {
1048
- void closeNativeBridgeRegistration(
1049
- completedWorker,
1050
- shardName,
1051
- "failed",
1052
- );
1053
- }
1054
- failures.set(shardName, {
1055
- mode: classifyFailure(reason),
1056
- reason,
1057
- sessionId,
1058
- completionPayload,
1059
- });
1060
- lockManager.release(shardName);
1061
-
1062
- const redundant = redundantWorkers.get(shardName);
1063
- if (redundant) {
1064
- void redundant.conductor.shutdown("primary_reported_non_ok");
1065
- }
1066
-
1067
- emitter.emit("shardFailed", {
1068
- shardName,
1069
- failureMode: classifyFailure(reason),
1070
- reason,
1071
- });
1072
-
1073
- cascadeDependencyFailure(shardName);
1074
- checkAllShardsCompleted();
1075
- return;
1076
- }
1077
- }
1078
-
1079
- if (isRedundant) {
1080
- // #394: a redundant worker may only "win" (and SIGKILL the still-running
1081
- // primary) once it has produced real, integratable commits. The earlier
1082
- // F7 payload check trusts a self-report; agy/gemini redundant workers are
1083
- // known to signal completion with zero commits (and legacy workers emit
1084
- // no payload at all, bypassing F7). Gate the win on authoritative git
1085
- // evidence collected now — before any kill — so a phantom completion can
1086
- // never kill the worker that is doing the actual work.
1087
- void finalizeRedundantWin(shardName, sessionId, completedWorker);
1088
- return;
1089
- }
1090
-
1091
- if (completedWorker) {
1092
- void closeNativeBridgeRegistration(
1093
- completedWorker,
1094
- shardName,
1095
- "completed",
1096
- );
1097
- }
1098
-
1099
- completedShards.add(shardName);
1100
-
1101
- // Primary completed — kill redundant if exists
1102
- const redundant = redundantWorkers.get(shardName);
1103
- if (redundant) {
1104
- void redundant.conductor.shutdown("primary_completed_first");
1105
- }
1106
-
1107
- emitter.emit("shardCompleted", { shardName, sessionId, isRedundant });
1108
- checkAllShardsCompleted();
1109
- }
1110
-
1111
- /**
1112
- * Gate a redundant worker's completion on authoritative git commit evidence
1113
- * before it is allowed to win the shard and SIGKILL the primary (#394).
1114
- *
1115
- * A redundant worker with no commits ahead of the base branch is NOT a valid
1116
- * winner: it is rejected, the primary is left running, and the shard is not
1117
- * marked complete. Only a worker with `commitsAhead > 0` promotes itself and
1118
- * terminates the primary.
1119
- *
1120
- * @param {string} shardName
1121
- * @param {string} sessionId
1122
- * @param {object} redundantWorker — entry fetched before this async hop
1123
- */
1124
- async function finalizeRedundantWin(shardName, sessionId, redundantWorker) {
1125
- const redundant = redundantWorker || redundantWorkers.get(shardName);
1126
- if (!redundant) {
1127
- // The primary already completed/failed and cleared the redundant entry
1128
- // (primary_completed_first / primary_failed_f7). Nothing to promote.
1129
- checkAllShardsCompleted();
1130
- return;
1131
- }
1132
-
1133
- // A redundant worker may win only while the shard is still undecided. This
1134
- // is checked TWICE — once up front (a cheap early-out before the git
1135
- // round-trip) and again AFTER the async evidence hop. The second check is
1136
- // the load-bearing one: the primary can complete/fail while
1137
- // collectCommitEvidence() is in flight, and without re-checking, a slow git
1138
- // probe would resume and flip an already-decided winner via
1139
- // `workers.set(shardName, redundant)` (#394 TOCTOU race).
1140
- const retireSupersededRedundant = (phase) => {
1141
- eventLog.append("redundant_win_superseded", {
1142
- shard: shardName,
1143
- sessionId,
1144
- phase,
1145
- });
1146
- redundantWorkers.delete(shardName);
1147
- void closeNativeBridgeRegistration(redundant, shardName, "failed");
1148
- checkAllShardsCompleted();
1149
- };
1150
-
1151
- if (completedShards.has(shardName) || failures.has(shardName)) {
1152
- retireSupersededRedundant("before_evidence");
1153
- return;
1154
- }
1155
-
1156
- let evidence;
1157
- try {
1158
- evidence = await collectCommitEvidence(redundant, baseBranch);
1159
- } catch (err) {
1160
- evidence = { commitsAhead: 0, dirty: false, error: err.message };
1161
- }
1162
-
1163
- // Re-check after the await: the primary may have won while we waited.
1164
- if (completedShards.has(shardName) || failures.has(shardName)) {
1165
- retireSupersededRedundant("after_evidence");
1166
- return;
1167
- }
1168
-
1169
- // A redundant worker may win only with CLEAN, integratable commits — the
1170
- // exact bar integration applies (collectCommitEvidence.ok =
1171
- // commitsAhead > 0 && !dirty). A weaker gate (commits-only) would promote a
1172
- // dirty-but-committed redundant, kill the primary, then fail integration on
1173
- // that same dirty evidence (F6) — losing the shard the primary could have
1174
- // delivered (#394 re-review). Reject anything integration would not accept.
1175
- if (!evidence?.ok) {
1176
- const reason = evidence?.error
1177
- ? "evidence_error"
1178
- : evidence?.dirty
1179
- ? "dirty_worktree"
1180
- : "no_commit";
1181
- eventLog.append("redundant_win_rejected", {
1182
- shard: shardName,
1183
- sessionId,
1184
- reason,
1185
- commitsAhead: evidence?.commitsAhead || 0,
1186
- dirty: Boolean(evidence?.dirty),
1187
- error: evidence?.error || null,
1188
- });
1189
- redundantWorkers.delete(shardName);
1190
- void closeNativeBridgeRegistration(redundant, shardName, "failed");
1191
- // Re-check in case the primary already reached a terminal state while we
1192
- // were collecting evidence.
1193
- checkAllShardsCompleted();
1194
- return;
1195
- }
1196
-
1197
- // Clean, integratable commit evidence — promote the redundant worker and
1198
- // kill the primary if it is still running.
1199
- const primary = workers.get(shardName);
1200
- workers.set(shardName, redundant);
1201
- redundantWorkers.delete(shardName);
1202
- completedShards.add(shardName);
1203
- void closeNativeBridgeRegistration(redundant, shardName, "completed");
1204
- if (primary && !isTerminal(primary)) {
1205
- eventLog.append("redundant_wins", {
1206
- shard: shardName,
1207
- commitsAhead: evidence.commitsAhead,
1208
- });
1209
- void primary.conductor.shutdown("redundant_completed_first");
1210
- }
1211
-
1212
- emitter.emit("shardCompleted", {
1213
- shardName,
1214
- sessionId,
1215
- isRedundant: true,
1216
- });
1217
- checkAllShardsCompleted();
1218
- }
1219
-
1220
- function handleShardFailed(shardName, sessionId, reason, isRedundant) {
1221
- const failureMode = classifyFailure(reason);
1222
-
1223
- eventLog.append("shard_failed", {
1224
- shard: shardName,
1225
- sessionId,
1226
- reason,
1227
- failureMode,
1228
- isRedundant,
1229
- });
1230
-
1231
- if (isRedundant) return; // redundant failure is non-critical
1232
- const failedWorker = workers.get(shardName);
1233
- if (failedWorker) {
1234
- void closeNativeBridgeRegistration(failedWorker, shardName, "failed");
1235
- }
1236
-
1237
- // F2: Rate limit — try fallback agent
1238
- if (failureMode === FAILURE_MODES.F2_RATE_LIMIT) {
1239
- const shard = plan.shards.find((s) => s.name === shardName);
1240
- if (shard) {
1241
- const fallbackAgent = FALLBACK_AGENTS[shard.agent];
1242
- if (fallbackAgent) {
1243
- eventLog.append("fallback_agent", {
1244
- shard: shardName,
1245
- from: shard.agent,
1246
- to: fallbackAgent,
1247
- });
1248
- const fallbackShard = { ...shard, agent: fallbackAgent };
1249
- lockManager.release(shardName);
1250
- void launchShard(fallbackShard);
1251
- return;
1252
- }
1253
- }
1254
- }
1255
-
1256
- failures.set(shardName, { mode: failureMode, reason, sessionId });
1257
- lockManager.release(shardName);
1258
-
1259
- emitter.emit("shardFailed", { shardName, failureMode, reason });
1260
-
1261
- // Cascade failure to dependents so they do not hold integration hostage
1262
- // (BFS over plan.shards). Without this, a dead shard keeps dependents in
1263
- // `pending` forever and integrateResults is never reachable.
1264
- cascadeDependencyFailure(shardName);
1265
-
1266
- checkAllShardsCompleted();
1267
- }
1268
-
1269
- function cascadeDependencyFailure(rootFailedShard) {
1270
- const queue = [rootFailedShard];
1271
- const visited = new Set([rootFailedShard]);
1272
- while (queue.length > 0) {
1273
- const failedName = queue.shift();
1274
- for (const candidate of plan.shards) {
1275
- if (visited.has(candidate.name)) continue;
1276
- if (failures.has(candidate.name)) continue;
1277
- if (completedShards.has(candidate.name)) continue;
1278
- if (!candidate.depends?.includes(failedName)) continue;
1279
-
1280
- visited.add(candidate.name);
1281
- failures.set(candidate.name, {
1282
- mode: FAILURE_MODES.F1_CRASH,
1283
- reason: `dep_failed:${failedName}`,
1284
- sessionId: null,
1285
- });
1286
- lockManager.release(candidate.name);
1287
- eventLog.append("shard_blocked", {
1288
- shard: candidate.name,
1289
- failedDep: failedName,
1290
- });
1291
- emitter.emit("shardFailed", {
1292
- shardName: candidate.name,
1293
- failureMode: FAILURE_MODES.F1_CRASH,
1294
- reason: `dep_failed:${failedName}`,
1295
- });
1296
- queue.push(candidate.name);
1297
- }
1298
- }
1299
- }
1300
-
1301
- function classifyFailure(reason) {
1302
- if (!reason) return FAILURE_MODES.F1_CRASH;
1303
- const r = String(reason).toLowerCase();
1304
- if (/rate.?limit|cooldown/u.test(r)) return FAILURE_MODES.F2_RATE_LIMIT;
1305
- if (/stall|l1_stall|timeout/u.test(r)) return FAILURE_MODES.F3_STALL;
1306
- if (/lease|violation/u.test(r)) return FAILURE_MODES.F4_LEASE_VIOLATION;
1307
- if (/merge|conflict/u.test(r)) return FAILURE_MODES.F5_MERGE_CONFLICT;
1308
- if (/no.?commit|dirty_worktree/u.test(r)) return FAILURE_MODES.F6_NO_COMMIT;
1309
- return FAILURE_MODES.F1_CRASH;
1310
- }
1311
-
1312
- function isTerminal(entry) {
1313
- const snap = entry.conductor.getSnapshot();
1314
- return snap.every(
1315
- (s) => s.state === STATES.COMPLETED || s.state === STATES.DEAD,
1316
- );
1317
- }
1318
-
1319
- // ── Integration ─────────────────────────────────────────────
1320
-
1321
- function checkAllShardsCompleted() {
1322
- if (state !== SWARM_STATES.RUNNING) return;
1323
-
1324
- // `completedShards` + `failures` are set synchronously inside
1325
- // handleShardCompleted / handleShardFailed, so they are the authoritative
1326
- // terminal signal. `isTerminal(w)` queries conductor.getSnapshot() which
1327
- // lags because the snapshot state transition is applied asynchronously
1328
- // after the onCompleted callback chain that invoked this check. Using the
1329
- // snapshot here races with its own producer and leaves integrateResults
1330
- // unreachable (2026-04-17 swarm hang — required manual cherry-pick).
1331
- const allDone = plan.mergeOrder.every(
1332
- (name) => completedShards.has(name) || failures.has(name),
1333
- );
1334
-
1335
- if (allDone) {
1336
- void integrateResults();
1337
- }
1338
- }
1339
-
1340
- /**
1341
- * Validate a shard's output — check for file lease violations.
1342
- * @param {string} shardName
1343
- * @param {string[]} changedFiles — files the shard actually modified
1344
- * @returns {{ ok: boolean, violations: Array }}
1345
- */
1346
- function validateResult(shardName, changedFiles) {
1347
- // Pass the shard's own lease so validateChanges can flag out-of-lease
1348
- // writes to distribution-critical paths (regression of #115/#34 —
1349
- // recovery patches were carrying `+++ /dev/null` deletions of
1350
- // .claude-plugin/marketplace.json, undetected by lease-only checks).
1351
- const ownLease = plan.leaseMap.get(shardName) || [];
1352
- const violations = lockManager.validateChanges(shardName, changedFiles, {
1353
- ownLease,
1354
- });
1355
-
1356
- eventLog.append("validate_result", {
1357
- shard: shardName,
1358
- changedFiles,
1359
- violations,
1360
- ok: violations.length === 0,
1361
- });
1362
-
1363
- return {
1364
- ok: violations.length === 0,
1365
- violations,
1366
- };
1367
- }
1368
-
1369
- /**
1370
- * Integrate results from all completed shards in merge order.
1371
- * Uses git operations for conflict detection.
1372
- */
1373
- async function integrateResults() {
1374
- setState(SWARM_STATES.INTEGRATING, "all_shards_done");
1375
-
1376
- const integrated = [];
1377
- const integrationFailures = [];
1378
- const integrationReport = [];
1379
- const preIntegrationFailures = [...failures.keys()];
1380
- let integrationBranch = null;
1381
-
1382
- const appendIntegrationReport = (row) => {
1383
- const normalized = normalizeIntegrationReportRow(row);
1384
- integrationReport.push(normalized);
1385
- eventLog.append("integration_shard_report", normalized);
1386
- return normalized;
1387
- };
1388
-
1389
- const failureNotes = (shardName, fallback = "failed") => {
1390
- const info = failures.get(shardName);
1391
- if (!info) return fallback;
1392
- return info.reason || info.mode || fallback;
1393
- };
1394
-
1395
- try {
1396
- ({ integrationBranch } = await prepareIntegrationBranchImpl({
1397
- runId,
1398
- baseBranch,
1399
- rootDir: workdir,
1400
- }));
1401
-
1402
- for (const shardName of plan.mergeOrder) {
1403
- if (failures.has(shardName)) {
1404
- const worker = workers.get(shardName);
1405
- if (worker?.cleanupPromise) await worker.cleanupPromise;
1406
- eventLog.append("skip_failed_shard", { shard: shardName });
1407
- appendIntegrationReport({
1408
- shard: shardName,
1409
- strategy: "failed",
1410
- notes: `skipped: ${failureNotes(shardName)}`,
1411
- });
1412
- continue;
1413
- }
1414
-
1415
- const worker = workers.get(shardName);
1416
- if (!worker) continue;
1417
-
1418
- // Fetch remote shard branch to local (push-blocked hosts like Ultra4)
1419
- const shard = plan.shards.find((s) => s.name === shardName);
1420
- if (shard?.host && shard._remoteEnv) {
1421
- const hostConfig = getHostConfig(shard.host, workdir);
1422
- const sshUser = hostConfig?.ssh_user || shard.host;
1423
- const remoteRepoPath = resolveRemoteDir(workdir, shard._remoteEnv);
1424
- const fetchResult = await fetchRemoteShard({
1425
- host: shard.host,
1426
- sshUser,
1427
- remoteRepoPath,
1428
- branchName: worker.branchName || `swarm/${runId}/${shardName}`,
1429
- rootDir: workdir,
1430
- });
1431
-
1432
- if (!fetchResult.ok) {
1433
- eventLog.append("remote_fetch_failed", {
1434
- shard: shardName,
1435
- error: fetchResult.error,
1436
- });
1437
- const cleanup = await maybeCleanupWorktree(
1438
- shardName,
1439
- worker,
1440
- shard,
1441
- {
1442
- force: true,
1443
- failureReason: "remote_fetch_failed",
1444
- },
1445
- );
1446
- integrationFailures.push(shardName);
1447
- appendIntegrationReport({
1448
- shard: shardName,
1449
- strategy: "failed",
1450
- notes: `remote_fetch_failed: ${fetchResult.error || "unknown"}`,
1451
- recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
1452
- });
1453
- continue;
1454
- }
1455
- eventLog.append("remote_fetch_ok", {
1456
- shard: shardName,
1457
- headCommit: fetchResult.headCommit,
1458
- });
1459
- }
1460
-
1461
- // Read shard output log for changed files
1462
- const commitEvidence = await collectCommitEvidence(
1463
- worker,
1464
- integrationBranch,
1465
- );
1466
- worker.commitEvidence = commitEvidence;
1467
- eventLog.append("commit_evidence", {
1468
- shard: shardName,
1469
- ...commitEvidence,
1470
- });
1471
-
1472
- const expectsCommitEvidence =
1473
- Array.isArray(shard?.files) && shard.files.length > 0;
1474
- if (expectsCommitEvidence && !commitEvidence.ok) {
1475
- failures.set(shardName, {
1476
- mode: FAILURE_MODES.F6_NO_COMMIT,
1477
- reason: commitEvidence.error
1478
- ? `no_commit_evidence:${commitEvidence.error}`
1479
- : commitEvidence.dirty
1480
- ? "dirty_worktree_without_commit"
1481
- : "no_commit_evidence",
1482
- commitEvidence,
1483
- });
1484
- eventLog.append("no_commit_guard_failed", {
1485
- shard: shardName,
1486
- ...commitEvidence,
1487
- });
1488
- const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
1489
- force: true,
1490
- failureReason:
1491
- failures.get(shardName)?.mode || "no_commit_evidence",
1492
- });
1493
- integrationFailures.push(shardName);
1494
- appendIntegrationReport({
1495
- shard: shardName,
1496
- strategy: "failed",
1497
- commits: commitEvidence.commitsAhead,
1498
- finalSha: commitEvidence.headCommit,
1499
- notes: failureNotes(shardName, "no_commit_evidence"),
1500
- recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
1501
- });
1502
- continue;
1503
- }
1504
-
1505
- // Read shard output log for changed files
1506
- const changedFiles = detectChangedFiles(shardName, worker);
1507
-
1508
- // Validate against lease map
1509
- const validation = validateResult(shardName, changedFiles);
1510
- if (!validation.ok) {
1511
- failures.set(shardName, {
1512
- mode: FAILURE_MODES.F4_LEASE_VIOLATION,
1513
- violations: validation.violations,
1514
- });
1515
- eventLog.append("lease_violation_revert", {
1516
- shard: shardName,
1517
- violations: validation.violations,
1518
- });
1519
- const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
1520
- force: true,
1521
- failureReason:
1522
- failures.get(shardName)?.mode || "lease_violation_revert",
1523
- });
1524
- integrationFailures.push(shardName);
1525
- appendIntegrationReport({
1526
- shard: shardName,
1527
- strategy: "failed",
1528
- notes: failureNotes(shardName, "lease_violation_revert"),
1529
- recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
1530
- });
1531
- continue;
1532
- }
1533
-
1534
- const shardBranch = worker.branchName || `swarm/${runId}/${shardName}`;
1535
- const rebaseResult = await rebaseShardOntoIntegrationImpl({
1536
- shardBranch,
1537
- integrationBranch,
1538
- rootDir: workdir,
1539
- });
1540
- if (!rebaseResult.ok) {
1541
- eventLog.append("integration_rebase_failed", {
1542
- shard: shardName,
1543
- shardBranch,
1544
- integrationBranch,
1545
- error: rebaseResult.error,
1546
- });
1547
- const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
1548
- force: true,
1549
- failureReason:
1550
- rebaseResult.error || FAILURE_MODES.F5_MERGE_CONFLICT,
1551
- });
1552
- integrationFailures.push(shardName);
1553
- appendIntegrationReport({
1554
- shard: shardName,
1555
- strategy: "failed",
1556
- commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
1557
- notes: rebaseResult.error || FAILURE_MODES.F5_MERGE_CONFLICT,
1558
- recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
1559
- });
1560
- continue;
1561
- }
1562
-
1563
- results.set(shardName, {
1564
- shard: shardName,
1565
- changedFiles,
1566
- branchName: shardBranch,
1567
- worktreePath: worker.worktreePath || null,
1568
- integrationBranch,
1569
- headCommit: rebaseResult.headCommit,
1570
- integrationStrategy: rebaseResult.strategy || "cherry-pick",
1571
- commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
1572
- notes: rebaseResult.notes || "",
1573
- completedAt: Date.now(),
1574
- });
1575
- integrated.push(shardName);
1576
- appendIntegrationReport({
1577
- shard: shardName,
1578
- strategy: rebaseResult.strategy || "cherry-pick",
1579
- commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
1580
- finalSha: rebaseResult.headCommit,
1581
- notes: rebaseResult.notes || "",
1582
- });
1583
-
1584
- await maybeCleanupWorktree(shardName, worker, shard, {
1585
- branchName: shardBranch,
1586
- });
1587
- }
1588
- } catch (err) {
1589
- const unresolved = plan.mergeOrder.filter(
1590
- (name) =>
1591
- !integrated.includes(name) &&
1592
- !preIntegrationFailures.includes(name) &&
1593
- !integrationFailures.includes(name),
1594
- );
1595
- const failed = [
1596
- ...new Set([
1597
- ...preIntegrationFailures,
1598
- ...integrationFailures,
1599
- ...unresolved,
1600
- ]),
1601
- ];
1602
- const skipped = preIntegrationFailures.filter(
1603
- (name) => !integrationFailures.includes(name),
1604
- );
1605
- const reported = new Set(integrationReport.map((row) => row.shard));
1606
- for (const shardName of failed) {
1607
- if (reported.has(shardName)) continue;
1608
- appendIntegrationReport({
1609
- shard: shardName,
1610
- strategy: "failed",
1611
- notes: `integration_error: ${err.message}`,
1612
- });
1613
- }
1614
-
1615
- eventLog.append("integration_complete", {
1616
- integrated,
1617
- failed,
1618
- integrationFailures,
1619
- integrationBranch,
1620
- skipped,
1621
- report: integrationReport,
1622
- error: err.message,
1623
- });
1624
-
1625
- setState(SWARM_STATES.FAILED, "integration_error");
1626
- const payload = settleIntegrationPromise({
1627
- integrated,
1628
- failed,
1629
- integrationFailures,
1630
- integrationBranch,
1631
- skipped,
1632
- report: integrationReport,
1633
- results: [...results.values()],
1634
- partial: true,
1635
- error: err.message,
1636
- });
1637
- emitter.emit("integrationComplete", payload);
1638
- return payload;
1639
- }
1640
-
1641
- const skipped = preIntegrationFailures.filter(
1642
- (name) => !integrationFailures.includes(name),
1643
- );
1644
- const failed = [...new Set([...skipped, ...integrationFailures])];
1645
-
1646
- eventLog.append("integration_complete", {
1647
- integrated,
1648
- failed,
1649
- integrationFailures,
1650
- integrationBranch,
1651
- skipped,
1652
- report: integrationReport,
1653
- });
1654
-
1655
- if (failed.length > 0 && integrated.length === 0) {
1656
- setState(SWARM_STATES.FAILED, "all_shards_failed_integration");
1657
- } else {
1658
- setState(
1659
- SWARM_STATES.COMPLETED,
1660
- `${integrated.length}/${plan.shards.length} integrated`,
1661
- );
1662
- }
1663
-
1664
- const payload = settleIntegrationPromise({
1665
- integrated,
1666
- failed,
1667
- integrationFailures,
1668
- integrationBranch,
1669
- skipped,
1670
- report: integrationReport,
1671
- results: [...results.values()],
1672
- partial: failed.length > 0,
1673
- });
1674
- emitter.emit("integrationComplete", payload);
1675
- return payload;
1676
- }
1677
-
1678
- async function maybeCleanupWorktree(
1679
- shardName,
1680
- worker,
1681
- shard,
1682
- {
1683
- branchName = worker?.branchName,
1684
- force = false,
1685
- failureReason = null,
1686
- } = {},
1687
- ) {
1688
- const outcome = { recoveryPatch: null, processCleanup: null };
1689
- if (!worker?.worktreePath || shard?.host) return outcome;
1690
- if (failureReason && keepFailedWorktrees) return outcome;
1691
- if (cleanedWorktreePaths.has(worker.worktreePath)) return outcome;
1692
-
1693
- await closeNativeBridgeRegistration(
1694
- worker,
1695
- shardName,
1696
- "before_worktree_cleanup",
1697
- );
1698
-
1699
- // Best-effort: preserve any uncommitted worker changes before removing the
1700
- // worktree, on BOTH the failure and success teardown paths (#394). A worker
1701
- // can exit 0 yet leave dirty, uncommitted work (the hub-observability case:
1702
- // exit 0, commits_made []), and the success cleanup carries no
1703
- // failureReason — without this it would silently delete that work.
1704
- // preserveWorktreePatchImpl self-skips when the worktree is clean, so a
1705
- // properly-committed shard saves nothing here.
1706
- try {
1707
- const preservation = await preserveWorktreePatchImpl({
1708
- worktreePath: worker.worktreePath,
1709
- shardId: shardName,
1710
- recoveryDir: join(workdir, ".codex-swarm", "recovery"),
1711
- });
1712
- if (preservation?.ok && !preservation.skipped) {
1713
- outcome.recoveryPatch = {
1714
- patchPath: preservation.patchPath,
1715
- manifestPath: preservation.manifestPath || null,
1716
- };
1717
- eventLog.append("recovery_patch_saved", {
1718
- shard: shardName,
1719
- worktreePath: worker.worktreePath,
1720
- patchPath: preservation.patchPath,
1721
- reason: failureReason || "dirty_on_cleanup",
1722
- });
1723
- }
1724
- } catch {
1725
- /* silent: preservation failures must not block cleanup */
1726
- }
1727
-
1728
- try {
1729
- const snapshot = worker.conductor?.getSnapshot?.();
1730
- const processCleanup = await cleanupShardProcessesImpl({
1731
- worktreePath: worker.worktreePath,
1732
- sessionIds: [worker.sessionId || worker.sessionConfig?.id].filter(
1733
- Boolean,
1734
- ),
1735
- topPids: snapshot?.topPids ?? [],
1736
- runId: plan?.runId || runId,
1737
- shardName,
1738
- });
1739
- outcome.processCleanup = processCleanup;
1740
- worker.processCleanup = processCleanup;
1741
- recordProcessCleanup(processCleanup);
1742
- eventLog.append("process_cleanup", {
1743
- shard: shardName,
1744
- worktreePath: worker.worktreePath,
1745
- killed: processCleanup?.killed ?? 0,
1746
- byCategory: processCleanup?.byCategory || {},
1747
- });
1748
- } catch (err) {
1749
- eventLog.append("process_cleanup_failed", {
1750
- shard: shardName,
1751
- worktreePath: worker.worktreePath,
1752
- error: err.message,
1753
- });
1754
- }
1755
-
1756
- try {
1757
- await cleanupWorktreeImpl({
1758
- worktreePath: worker.worktreePath,
1759
- branchName,
1760
- rootDir: workdir,
1761
- force,
1762
- });
1763
- cleanedWorktreePaths.add(worker.worktreePath);
1764
- if (failureReason) {
1765
- eventLog.append("worktree_auto_cleanup", {
1766
- shard: shardName,
1767
- worktreePath: worker.worktreePath,
1768
- reason: failureReason,
1769
- });
1770
- }
1771
- } catch (err) {
1772
- eventLog.append("worktree_cleanup_failed", {
1773
- shard: shardName,
1774
- worktreePath: worker.worktreePath,
1775
- reason: failureReason || null,
1776
- error: err.message,
1777
- });
1778
- }
1779
-
1780
- return outcome;
1781
- }
1782
-
1783
- /**
1784
- * Detect which files a shard modified by reading its output logs.
1785
- * Falls back to an empty list if detection fails.
1786
- * @param {string} shardName
1787
- * @param {object} worker
1788
- * @returns {string[]}
1789
- */
1790
- function detectChangedFiles(shardName, worker) {
1791
- // Best-effort: parse output log for file paths
1792
- const _outPath = join(logsDir, shardName);
1793
- try {
1794
- const snap = worker.conductor.getSnapshot();
1795
- for (const session of snap) {
1796
- if (session.outPath && existsSync(session.outPath)) {
1797
- const output = readFileSync(session.outPath, "utf8");
1798
- return extractFilePathsFromOutput(
1799
- output,
1800
- plan.leaseMap.get(shardName) || [],
1801
- );
1802
- }
1803
- }
1804
- } catch {
1805
- /* best-effort */
1806
- }
1807
-
1808
- // Fallback: trust the lease map (shard was allowed these files)
1809
- return plan.leaseMap.get(shardName) || [];
1810
- }
1811
-
1812
- /**
1813
- * Extract modified file paths from worker output text.
1814
- * Looks for common patterns: "wrote file.mjs", "modified file.mjs", diff headers.
1815
- * @param {string} output
1816
- * @param {string[]} allowedFiles — lease map files to match against
1817
- * @returns {string[]}
1818
- */
1819
- function extractFilePathsFromOutput(output, allowedFiles) {
1820
- if (!output) return allowedFiles;
1821
-
1822
- const found = new Set();
1823
- const lines = output.split(/\r?\n/);
1824
-
1825
- for (const line of lines) {
1826
- // Match common patterns
1827
- const patterns = [
1828
- /(?:wrote|created|modified|updated|edited)\s+['"]?([^\s'"]+\.\w+)/i,
1829
- /^[+-]{3}\s+[ab]\/(.+)/, // diff headers
1830
- /^diff --git a\/(.+)\s+b\//, // git diff headers
1831
- ];
1832
-
1833
- for (const re of patterns) {
1834
- const match = line.match(re);
1835
- if (match) found.add(match[1]);
1836
- }
1837
- }
1838
-
1839
- // Intersect with allowed files if we found anything
1840
- if (found.size > 0) {
1841
- return [...found].filter((f) =>
1842
- allowedFiles.some((a) => f.endsWith(a) || a.endsWith(f) || f === a),
1843
- );
1844
- }
1845
-
1846
- return allowedFiles;
1847
- }
1848
-
1849
- // ── Status monitor ──────────────────────────────────────────
1850
-
1851
- /**
1852
- * Get current swarm status snapshot.
1853
- * @returns {SwarmStatus}
1854
- */
1855
- function getStatus() {
1856
- const workerStatuses = [];
1857
-
1858
- for (const [name, w] of workers) {
1859
- const snap = w.conductor.getSnapshot();
1860
- const authoritative = computeAuthoritativeStatus(name, w, snap);
1861
- workerStatuses.push({
1862
- shard: name,
1863
- agent: w.shardConfig.agent,
1864
- sessions: snap,
1865
- failed: failures.has(name),
1866
- failureInfo: failures.get(name) || null,
1867
- integrated: results.has(name),
1868
- authoritativeStatus: authoritative.status,
1869
- authoritativeReason: authoritative.reason,
1870
- commitEvidence: w.commitEvidence || null,
1871
- });
1872
- }
1873
-
1874
- return Object.freeze({
1875
- state,
1876
- totalShards: plan?.shards.length || 0,
1877
- completedShards: completedShards.size,
1878
- failedShards: failures.size,
1879
- workers: workerStatuses,
1880
- mergeOrder: plan?.mergeOrder || [],
1881
- criticalShards: plan?.criticalShards || [],
1882
- locks: lockManager?.snapshot() || [],
1883
- integrationPromise: Object.freeze({
1884
- state: integrationPromiseState.state,
1885
- startedAt: integrationPromiseState.startedAt,
1886
- settledAt: integrationPromiseState.settledAt,
1887
- partial: integrationPromiseState.partial,
1888
- integrated: [...integrationPromiseState.integrated],
1889
- failed: [...integrationPromiseState.failed],
1890
- integrationFailures: [...integrationPromiseState.integrationFailures],
1891
- skipped: [...integrationPromiseState.skipped],
1892
- integrationBranch: integrationPromiseState.integrationBranch,
1893
- report: integrationPromiseState.report.map((row) => ({ ...row })),
1894
- processCleanup: Object.freeze({
1895
- totalKilled: integrationPromiseState.processCleanup.totalKilled,
1896
- byCategory: Object.freeze({
1897
- ...integrationPromiseState.processCleanup.byCategory,
1898
- }),
1899
- }),
1900
- error: integrationPromiseState.error,
1901
- }),
1902
- });
1903
- }
1904
-
1905
- // ── Public API ──────────────────────────────────────────────
1906
-
1907
- /**
1908
- * Launch the swarm from a pre-built plan.
1909
- * @param {SwarmPlan} swarmPlan — from planSwarm()
1910
- * @returns {Promise<SwarmStatus>}
1911
- */
1912
- /** Hub keepalive — Hub crash recovery (idle timeout은 기본 비활성이므로 crash 복구 용도) */
1913
- let hubKeepaliveTimer = null;
1914
- let lockRenewTimer = null;
1915
- function startHubKeepalive() {
1916
- // 5분마다 Hub /status 핑 — crash 감지 시 ensureHubAlive로 재시작
1917
- hubKeepaliveTimer = setInterval(
1918
- async () => {
1919
- try {
1920
- const hub = getHubInfoImpl ? await getHubInfoImpl() : null;
1921
- const statusUrl = resolveHubStatusUrl(hub);
1922
- if (!statusUrl) {
1923
- if (ensureHubAliveImpl) {
1924
- eventLog.append("hub_keepalive_restart", {
1925
- reason: "hub_url_unresolved",
1926
- });
1927
- await ensureHubAliveImpl();
1928
- }
1929
- return;
1930
- }
1931
-
1932
- const resp = await fetch(statusUrl, {
1933
- signal: AbortSignal.timeout(5000),
1934
- });
1935
- if (!resp.ok && ensureHubAliveImpl) {
1936
- eventLog.append("hub_keepalive_restart", {});
1937
- await ensureHubAliveImpl();
1938
- }
1939
- } catch {
1940
- // Hub 다운 — 재시작 시도
1941
- if (ensureHubAliveImpl) {
1942
- eventLog.append("hub_keepalive_restart", {
1943
- reason: "fetch_failed",
1944
- });
1945
- try {
1946
- await ensureHubAliveImpl();
1947
- } catch {
1948
- eventLog.append("hub_restart_failed", {});
1949
- }
1950
- }
1951
- }
1952
- },
1953
- 5 * 60 * 1000,
1954
- );
1955
- }
1956
-
1957
- function stopHubKeepalive() {
1958
- if (hubKeepaliveTimer) {
1959
- clearInterval(hubKeepaliveTimer);
1960
- hubKeepaliveTimer = null;
1961
- }
1962
- }
1963
-
1964
- function startLockRenewal() {
1965
- lockRenewTimer = setInterval(() => {
1966
- if (!lockManager) return;
1967
- for (const shardName of workers.keys()) {
1968
- if (!completedShards.has(shardName) && !failures.has(shardName))
1969
- lockManager.renew(shardName);
1970
- }
1971
- }, DEFAULT_SWARM_LOCK_TTL_MS / 2);
1972
- lockRenewTimer.unref?.();
1973
- }
1974
-
1975
- function stopLockRenewal() {
1976
- if (lockRenewTimer) clearInterval(lockRenewTimer);
1977
- lockRenewTimer = null;
1978
- }
1979
-
1980
- async function launch(swarmPlan) {
1981
- if (state !== SWARM_STATES.PLANNING) {
1982
- throw new Error(`Cannot launch in state "${state}"`);
1983
- }
1984
-
1985
- plan = swarmPlan;
1986
- markIntegrationPromisePending();
1987
-
1988
- // Phase 1 dynamic routing wire-up (opt-in env TRIFLUX_DYNAMIC_ROUTING).
1989
- // launch() 는 plan-time, 이미 async 라 D-1 facade 의 await routeRequest 를
1990
- // 그대로 사용한다 (conductor.spawnSession 의 sync invariant 충돌 없음).
1991
- // team_size = plan.shards.length 매핑 → S5 leaseMany 시나리오 자연 매핑.
1992
- // decision.shards[i].cli 가 plan.shards[i].agent 와 다르면 1:1 override.
1993
- if (
1994
- isDynamicRoutingEnabled(process.env) &&
1995
- Array.isArray(plan?.shards) &&
1996
- plan.shards.length > 0
1997
- ) {
1998
- try {
1999
- const router = createDynamicRouter();
2000
- const decision = await router.routeRequest({
2001
- task_id: runId,
2002
- team_size: plan.shards.length,
2003
- agent_hint: plan.shards[0]?.agent || "codex",
2004
- });
2005
- eventLog.append("dynamic_route_plan_decision", {
2006
- decisionId: decision.decisionId ?? null,
2007
- scenario: decision.scenario ?? "unknown",
2008
- lane: decision.lane ?? null,
2009
- mode: decision.mode ?? null,
2010
- shardCount: decision.shards?.length ?? 0,
2011
- planShardCount: plan.shards.length,
2012
- snapshotAgeMs: decision.snapshotAgeMs ?? null,
2013
- });
2014
- const overrides = [];
2015
- const decShards = Array.isArray(decision.shards) ? decision.shards : [];
2016
- for (let i = 0; i < plan.shards.length; i++) {
2017
- const shard = plan.shards[i];
2018
- const decShard = decShards[i];
2019
- if (decShard?.cli && decShard.cli !== shard.agent) {
2020
- overrides.push({
2021
- shard: shard.name,
2022
- original: shard.agent,
2023
- override: decShard.cli,
2024
- accountId: decShard.accountId ?? null,
2025
- });
2026
- shard.agent = decShard.cli;
2027
- }
2028
- }
2029
- if (overrides.length > 0) {
2030
- eventLog.append("dynamic_route_plan_overrides", {
2031
- decisionId: decision.decisionId ?? null,
2032
- count: overrides.length,
2033
- overrides,
2034
- });
2035
- }
2036
- } catch (err) {
2037
- eventLog.append("dynamic_route_plan_error", {
2038
- error: err?.message || String(err),
2039
- });
2040
- }
2041
- }
2042
-
2043
- // Hub alive 확인 — 죽어있으면 재시작
2044
- if (ensureHubAliveImpl) {
2045
- ensureHubAliveImpl()
2046
- .then((hub) => {
2047
- eventLog.append("hub_ensured", { port: hub?.port });
2048
- })
2049
- .catch((err) => {
2050
- eventLog.append("hub_ensure_failed", { error: err.message });
2051
- emitter.emit("warning", {
2052
- type: "hub_unavailable",
2053
- error: err.message,
2054
- });
2055
- });
2056
- }
2057
-
2058
- // Warn about file conflicts but don't block
2059
- if (plan.conflicts.length > 0) {
2060
- eventLog.append("file_conflicts_warning", { conflicts: plan.conflicts });
2061
- emitter.emit("warning", {
2062
- type: "file_conflicts",
2063
- conflicts: plan.conflicts,
2064
- });
2065
- }
2066
-
2067
- // Initialize lock manager
2068
- lockManager = createSwarmLocks({
2069
- repoRoot: workdir,
2070
- persistPath: join(workdir, ".triflux", "swarm-locks.json"),
2071
- });
2072
-
2073
- startLockRenewal();
2074
- // Hub keepalive 시작
2075
- startHubKeepalive();
2076
-
2077
- setState(SWARM_STATES.LAUNCHING, `${plan.shards.length} shards`);
2078
-
2079
- // Launch shards respecting dependency order
2080
- const launched = new Set();
2081
- const pending = new Set(plan.mergeOrder);
2082
-
2083
- async function launchReady() {
2084
- for (const name of pending) {
2085
- const shard = plan.shards.find((s) => s.name === name);
2086
- if (!shard) continue;
2087
-
2088
- // Check all dependencies are completed (not just launched)
2089
- const depsReady = shard.depends.every((d) => completedShards.has(d));
2090
- if (!depsReady) continue;
2091
-
2092
- pending.delete(name);
2093
- launched.add(name);
2094
- await launchShard(shard);
2095
-
2096
- // Launch redundant worker only when explicitly requested.
2097
- const redundantAgent = resolveRedundantAgent(shard);
2098
- if (redundantAgent) {
2099
- const redundantShard = {
2100
- ...shard,
2101
- agent: redundantAgent,
2102
- };
2103
- await launchShard(redundantShard, true);
2104
- }
2105
- }
2106
- }
2107
-
2108
- await launchReady();
2109
-
2110
- // Re-check pending on each shard completion (dependency chains)
2111
- emitter.on("shardCompleted", () => {
2112
- if (pending.size > 0) void launchReady();
2113
- });
2114
-
2115
- setState(
2116
- SWARM_STATES.RUNNING,
2117
- `${launched.size} launched, ${pending.size} pending deps`,
2118
- );
2119
-
2120
- return { ...getStatus(), done: integrationPromise };
2121
- }
2122
-
2123
- /**
2124
- * Graceful shutdown — kill all workers and release locks.
2125
- * @param {string} [reason]
2126
- */
2127
- async function shutdown(reason = "shutdown") {
2128
- stopHubKeepalive();
2129
- stopLockRenewal();
2130
- eventLog.append("swarm_shutdown", { reason, state });
2131
-
2132
- const shutdowns = [];
2133
- for (const [, w] of workers) {
2134
- shutdowns.push(w.conductor.shutdown(reason));
2135
- shutdowns.push(
2136
- closeNativeBridgeRegistration(w, w.shardConfig?.name, reason),
2137
- );
2138
- }
2139
- for (const [, w] of redundantWorkers) {
2140
- shutdowns.push(w.conductor.shutdown(reason));
2141
- shutdowns.push(
2142
- closeNativeBridgeRegistration(w, w.shardConfig?.name, reason),
2143
- );
2144
- }
2145
-
2146
- await Promise.allSettled(shutdowns);
2147
-
2148
- if (!keepFailedWorktrees) {
2149
- const cleanupCandidates = new Map();
2150
- for (const [shardName, worker] of workers) {
2151
- cleanupCandidates.set(shardName, worker);
2152
- }
2153
- for (const [shardName, worker] of redundantWorkers) {
2154
- cleanupCandidates.set(`${shardName}:redundant`, worker);
2155
- }
2156
-
2157
- for (const [candidateName, worker] of cleanupCandidates) {
2158
- const shardName = worker?.shardConfig?.name || candidateName;
2159
- const failureInfo = failures.get(shardName);
2160
- const shard =
2161
- worker?.shardConfig ||
2162
- plan?.shards.find((item) => item.name === shardName);
2163
- if (!worker?.worktreePath) continue;
2164
- await maybeCleanupWorktree(shardName, worker, shard, {
2165
- force: true,
2166
- failureReason: failureInfo?.mode || failureInfo?.reason || reason,
2167
- });
2168
- }
2169
- }
2170
-
2171
- sharedRegistry.clear();
2172
- lockManager?.releaseAll();
2173
- await eventLog.flush();
2174
- await eventLog.close();
2175
-
2176
- if (state !== SWARM_STATES.COMPLETED && state !== SWARM_STATES.FAILED) {
2177
- setState(SWARM_STATES.FAILED, reason);
2178
- }
2179
-
2180
- emitter.emit("shutdown", { reason });
2181
- }
2182
-
2183
- return Object.freeze({
2184
- launch,
2185
- shutdown,
2186
- getStatus,
2187
- integrationComplete() {
2188
- return integrationPromise;
2189
- },
2190
- getMeshRegistry() {
2191
- return sharedRegistry;
2192
- },
2193
- validateResult,
2194
- on: emitter.on.bind(emitter),
2195
- off: emitter.off.bind(emitter),
2196
- get state() {
2197
- return state;
2198
- },
2199
- get plan() {
2200
- return plan;
2201
- },
2202
- get eventLogPath() {
2203
- return eventLog.filePath;
2204
- },
2205
- });
2206
- }