@triflux/remote 10.50.2 → 10.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cto/brief.mjs +0 -12
- package/cto/collect.mjs +0 -73
- package/cto/current.schema.json +0 -4
- package/cto/status.mjs +2 -50
- package/hub/index.mjs +0 -8
- package/hub/team/agent-map.json +0 -1
- package/hub/team/ansi.mjs +1 -35
- package/hub/team/backend.mjs +9 -145
- package/hub/team/cli/commands/attach.mjs +1 -17
- package/hub/team/cli/commands/control.mjs +12 -38
- package/hub/team/cli/commands/debug.mjs +1 -37
- package/hub/team/cli/commands/focus.mjs +5 -31
- package/hub/team/cli/commands/interrupt.mjs +6 -25
- package/hub/team/cli/commands/kill.mjs +1 -30
- package/hub/team/cli/commands/list.mjs +0 -16
- package/hub/team/cli/commands/send.mjs +6 -26
- package/hub/team/cli/commands/start/index.mjs +41 -133
- package/hub/team/cli/commands/start/parse-args.mjs +15 -40
- package/hub/team/cli/commands/start/start-headless.mjs +138 -105
- package/hub/team/cli/commands/start/start-mux.mjs +0 -3
- package/hub/team/cli/commands/status.mjs +2 -57
- package/hub/team/cli/commands/stop.mjs +14 -19
- package/hub/team/cli/help.mjs +4 -5
- package/hub/team/cli/manifest.mjs +5 -1
- package/hub/team/cli/render.mjs +0 -12
- package/hub/team/cli/services/hub-client.mjs +2 -88
- package/hub/team/cli/services/runtime-mode.mjs +27 -98
- package/hub/team/handoff.mjs +3 -34
- package/hub/team/headless.mjs +13 -754
- package/hub/team/orchestrator.mjs +4 -30
- package/hub/team/pane.mjs +6 -28
- package/hub/team/psmux.mjs +0 -1
- package/hub/team/remote-session.mjs +1 -2
- package/hub/team/session.mjs +7 -91
- package/hub/team/swarm-intent.mjs +0 -198
- package/hub/team/tui-core.mjs +1 -1
- package/hub/team/tui-viewer.mjs +2 -2
- package/hub/team/tui.mjs +8 -269
- package/hub/team/wt-manager.mjs +2 -2
- package/hub/workers/factory.mjs +4 -37
- package/hub/workers/lib/jsonrpc-ws-uds.mjs +0 -2
- package/package.json +1 -1
- package/scripts/lib/agent-route-policy.mjs +162 -14
- package/scripts/lib/claudemd-scanner.mjs +2 -199
- package/scripts/lib/cli-claude.mjs +0 -2
- package/scripts/lib/context.mjs +3 -19
- package/scripts/lib/env-probe.mjs +5 -64
- package/scripts/lib/gemini-profiles.mjs +2 -69
- package/scripts/lib/machine-profile.mjs +3 -17
- package/scripts/lib/mcp-filter.mjs +109 -53
- package/scripts/lib/mcp-health.mjs +2 -2
- package/scripts/lib/mcp-server-catalog.mjs +2 -2
- package/scripts/lib/psmux-info.mjs +11 -59
- package/scripts/lib/skill-template.mjs +0 -245
- package/hub/team/build-worker-prompt.mjs +0 -92
- package/hub/team/check-mcp-hub.mjs +0 -49
- package/hub/team/claude-native-bridge.mjs +0 -987
- package/hub/team/cli/commands/start/start-in-process.mjs +0 -50
- package/hub/team/cli/commands/start/start-wt.mjs +0 -86
- package/hub/team/cli/services/native-control.mjs +0 -145
- package/hub/team/codex-review.mjs +0 -520
- package/hub/team/conductor-mesh-bridge.mjs +0 -122
- package/hub/team/conductor.mjs +0 -1529
- package/hub/team/consensus-meta.mjs +0 -176
- package/hub/team/cto-pull.mjs +0 -31
- package/hub/team/daemon-pty-tmux-bridge.mjs +0 -146
- package/hub/team/dashboard.mjs +0 -322
- package/hub/team/event-log.mjs +0 -84
- package/hub/team/execution-mode.mjs +0 -324
- package/hub/team/extract-completion-payload.mjs +0 -110
- package/hub/team/headless-bridge-session.mjs +0 -17
- package/hub/team/health-probe.mjs +0 -435
- package/hub/team/interactive-native-launcher.mjs +0 -119
- package/hub/team/interactive-native-worker.mjs +0 -80
- package/hub/team/interactive-tui-transport.mjs +0 -479
- package/hub/team/launcher-template.mjs +0 -144
- package/hub/team/mcp-selector.mjs +0 -145
- package/hub/team/native-supervisor.mjs +0 -460
- package/hub/team/native.mjs +0 -727
- package/hub/team/recovery-store.mjs +0 -149
- package/hub/team/remote-probe.mjs +0 -292
- package/hub/team/remote-watcher.mjs +0 -506
- package/hub/team/runtime-strategy.mjs +0 -200
- package/hub/team/schemas/worker-completion.json +0 -39
- package/hub/team/sentinel-capture.mjs +0 -127
- package/hub/team/swarm-cli.mjs +0 -390
- package/hub/team/swarm-hypervisor.mjs +0 -2206
- package/hub/team/swarm-locks.mjs +0 -335
- package/hub/team/swarm-planner.mjs +0 -371
- package/hub/team/swarm-preflight.mjs +0 -366
- package/hub/team/swarm-reconciler.mjs +0 -242
- package/hub/team/tui-remote-adapter.mjs +0 -377
- package/hub/team/worker-completion-validator.mjs +0 -79
- package/hub/team/worktree-lifecycle.mjs +0 -639
- package/hub/team/wt-templates.json +0 -43
- package/hub/workers/antigravity-route-worker.mjs +0 -179
- package/scripts/lib/dynamic-route-cli.mjs +0 -107
|
@@ -1,2206 +0,0 @@
|
|
|
1
|
-
// hub/team/swarm-hypervisor.mjs — Multi-model swarm orchestration hypervisor
|
|
2
|
-
// Consumes a SwarmPlan (from swarm-planner.mjs) and orchestrates parallel
|
|
3
|
-
// conductor sessions with file-lease enforcement, result validation,
|
|
4
|
-
// and ordered integration.
|
|
5
|
-
//
|
|
6
|
-
// Failure modes handled:
|
|
7
|
-
// F1: Worker crash → conductor auto-restart (maxRestarts)
|
|
8
|
-
// F2: Rate limit → account-broker cooldown + fallback agent
|
|
9
|
-
// F3: Stall → health probe L1 detection + kill + restart
|
|
10
|
-
// F4: File lease violation → revert worker changes, flag shard as failed
|
|
11
|
-
// F5: Merge conflict → retry integration with conflict resolution
|
|
12
|
-
|
|
13
|
-
import { execFile } from "node:child_process";
|
|
14
|
-
import crypto from "node:crypto";
|
|
15
|
-
import { EventEmitter } from "node:events";
|
|
16
|
-
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
17
|
-
import { join } from "node:path";
|
|
18
|
-
import {
|
|
19
|
-
createDynamicRouter,
|
|
20
|
-
isDynamicRoutingEnabled,
|
|
21
|
-
} from "@triflux/core/hub/dynamic-routing-engine.mjs";
|
|
22
|
-
import { cleanupShardProcesses } from "@triflux/core/hub/lib/process-utils.mjs";
|
|
23
|
-
import { getHostConfig } from "@triflux/core/hub/lib/ssh-command.mjs";
|
|
24
|
-
import { DEFAULT_SWARM_LOCK_TTL_MS } from "@triflux/core/hub/lib/timeout-defaults.mjs";
|
|
25
|
-
import { buildWorkerPrompt } from "./build-worker-prompt.mjs";
|
|
26
|
-
import { registerSwarmShard } from "./claude-native-bridge.mjs";
|
|
27
|
-
import { createConductor, STATES } from "./conductor.mjs";
|
|
28
|
-
import { ensureConductorRegistry } from "./conductor-registry.mjs";
|
|
29
|
-
import { createEventLog } from "./event-log.mjs";
|
|
30
|
-
import { preserveWorktreePatch } from "./recovery-store.mjs";
|
|
31
|
-
import { probeRemoteEnv, resolveRemoteDir } from "./remote-session.mjs";
|
|
32
|
-
import { createSwarmLocks } from "./swarm-locks.mjs";
|
|
33
|
-
import { validateWorkerCompletion } from "./worker-completion-validator.mjs";
|
|
34
|
-
import {
|
|
35
|
-
cleanupWorktree,
|
|
36
|
-
EXPECTED_WORKTREE_DELETIONS,
|
|
37
|
-
ensureWorktree,
|
|
38
|
-
extractDirtyFiles,
|
|
39
|
-
fetchRemoteShard,
|
|
40
|
-
prepareIntegrationBranch,
|
|
41
|
-
pruneWorktree,
|
|
42
|
-
rebaseShardOntoIntegration,
|
|
43
|
-
} from "./worktree-lifecycle.mjs";
|
|
44
|
-
|
|
45
|
-
let ensureHubAliveFn = null;
|
|
46
|
-
let getHubInfoFn = null;
|
|
47
|
-
try {
|
|
48
|
-
({ ensureHubAlive: ensureHubAliveFn, getHubInfo: getHubInfoFn } =
|
|
49
|
-
await import("./cli/services/hub-client.mjs"));
|
|
50
|
-
} catch {
|
|
51
|
-
// hub-client 미설치 환경 — Hub 수동 관리 필요
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
let importedCreateRegistry = null;
|
|
55
|
-
let meshRegistryImportError = null;
|
|
56
|
-
try {
|
|
57
|
-
({ createRegistry: importedCreateRegistry } = await import(
|
|
58
|
-
"@triflux/core/mesh/mesh-registry.mjs"
|
|
59
|
-
));
|
|
60
|
-
} catch (err) {
|
|
61
|
-
meshRegistryImportError = err;
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// ── Swarm states ──────────────────────────────────────────────
|
|
65
|
-
|
|
66
|
-
export const SWARM_STATES = Object.freeze({
|
|
67
|
-
PLANNING: "planning",
|
|
68
|
-
LAUNCHING: "launching",
|
|
69
|
-
RUNNING: "running",
|
|
70
|
-
INTEGRATING: "integrating",
|
|
71
|
-
VALIDATING: "validating",
|
|
72
|
-
COMPLETED: "completed",
|
|
73
|
-
FAILED: "failed",
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
// ── Failure mode classification ───────────────────────────────
|
|
77
|
-
|
|
78
|
-
const FAILURE_MODES = Object.freeze({
|
|
79
|
-
F1_CRASH: "F1_crash",
|
|
80
|
-
F2_RATE_LIMIT: "F2_rate_limit",
|
|
81
|
-
F3_STALL: "F3_stall",
|
|
82
|
-
F4_LEASE_VIOLATION: "F4_lease_violation",
|
|
83
|
-
F5_MERGE_CONFLICT: "F5_merge_conflict",
|
|
84
|
-
F6_NO_COMMIT: "F6_no_commit",
|
|
85
|
-
F7_WORKER_DID_NOT_COMMIT: "F7_worker_did_not_commit",
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
const FALLBACK_AGENTS = Object.freeze({
|
|
89
|
-
codex: "gemini",
|
|
90
|
-
gemini: "codex",
|
|
91
|
-
claude: "codex",
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
function resolveRedundantAgent(shard) {
|
|
95
|
-
const mode = shard?.redundancy;
|
|
96
|
-
if (!mode) return null;
|
|
97
|
-
if (mode === "same-cli") return shard.agent;
|
|
98
|
-
if (mode === true || mode === "fallback") {
|
|
99
|
-
return FALLBACK_AGENTS[shard.agent] || shard.agent;
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
function createNoopRegistry() {
|
|
105
|
-
return Object.freeze({
|
|
106
|
-
register() {},
|
|
107
|
-
unregister() {},
|
|
108
|
-
discover() {
|
|
109
|
-
return [];
|
|
110
|
-
},
|
|
111
|
-
getAgent() {
|
|
112
|
-
return null;
|
|
113
|
-
},
|
|
114
|
-
listAll() {
|
|
115
|
-
return [];
|
|
116
|
-
},
|
|
117
|
-
clear() {},
|
|
118
|
-
});
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
function createSharedRegistry(factory) {
|
|
122
|
-
if (typeof factory !== "function") {
|
|
123
|
-
return {
|
|
124
|
-
registry: createNoopRegistry(),
|
|
125
|
-
fallbackReason: meshRegistryImportError
|
|
126
|
-
? `mesh_import_failed:${meshRegistryImportError.message}`
|
|
127
|
-
: "mesh_registry_unavailable",
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
try {
|
|
132
|
-
return { registry: factory(), fallbackReason: null };
|
|
133
|
-
} catch (err) {
|
|
134
|
-
return {
|
|
135
|
-
registry: createNoopRegistry(),
|
|
136
|
-
fallbackReason: `mesh_registry_init_failed:${err.message}`,
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
function hasOwnDep(deps, key) {
|
|
142
|
-
return Object.hasOwn(deps, key);
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function formatHubHostForUrl(host) {
|
|
146
|
-
return host.includes(":") ? `[${host}]` : host;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
function resolveHubStatusUrl(hub) {
|
|
150
|
-
if (!hub) return null;
|
|
151
|
-
|
|
152
|
-
if (typeof hub.url === "string" && hub.url.trim()) {
|
|
153
|
-
try {
|
|
154
|
-
const url = new URL(hub.url);
|
|
155
|
-
url.pathname = "/status";
|
|
156
|
-
url.search = "";
|
|
157
|
-
url.hash = "";
|
|
158
|
-
return url.toString();
|
|
159
|
-
} catch {
|
|
160
|
-
return null;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
const host = typeof hub.host === "string" ? hub.host.trim() : "";
|
|
165
|
-
const port = Number(hub.port);
|
|
166
|
-
if (!host || !Number.isFinite(port) || port <= 0) return null;
|
|
167
|
-
return `http://${formatHubHostForUrl(host)}:${port}/status`;
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function isTruthyConfigValue(value) {
|
|
171
|
-
if (typeof value === "boolean") return value;
|
|
172
|
-
return /^(true|yes|1|interactive)$/i.test(String(value || "").trim());
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function isInteractiveNativeBridgeShard(shard) {
|
|
176
|
-
return (
|
|
177
|
-
isTruthyConfigValue(shard?.interactive) ||
|
|
178
|
-
isTruthyConfigValue(shard?.nativeBridgeInteractive) ||
|
|
179
|
-
String(shard?.workerType || "").toLowerCase() === "interactive"
|
|
180
|
-
);
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Create a swarm hypervisor.
|
|
185
|
-
* @param {object} opts
|
|
186
|
-
* @param {string} opts.workdir — repository root / working directory
|
|
187
|
-
* @param {string} opts.logsDir — base directory for all logs
|
|
188
|
-
* @param {string} [opts.baseBranch='main'] — base branch for shard worktrees
|
|
189
|
-
* @param {string} [opts.runId=`swarm-${Date.now()}`] — logical swarm run id
|
|
190
|
-
* @param {number} [opts.maxRestarts=2] — per-shard max restarts
|
|
191
|
-
* @param {boolean} [opts.nativeBridge=false] — expose local shards in Claude agents
|
|
192
|
-
* @param {number} [opts.graceMs=10000] — conductor shutdown grace period
|
|
193
|
-
* @param {boolean} [opts.keepFailedWorktrees=false] — preserve failed shard worktrees for debugging
|
|
194
|
-
* @param {object} [opts.probeOpts] — health probe overrides
|
|
195
|
-
* @param {object} [opts.deps] — dependency injection for testing
|
|
196
|
-
* @returns {SwarmHypervisor}
|
|
197
|
-
*/
|
|
198
|
-
export function createSwarmHypervisor(opts) {
|
|
199
|
-
const {
|
|
200
|
-
workdir,
|
|
201
|
-
logsDir,
|
|
202
|
-
baseBranch = "main",
|
|
203
|
-
runId = `swarm-${Date.now()}`,
|
|
204
|
-
maxRestarts = 2,
|
|
205
|
-
nativeBridge = false,
|
|
206
|
-
graceMs = 10_000,
|
|
207
|
-
keepFailedWorktrees = false,
|
|
208
|
-
probeOpts = {},
|
|
209
|
-
_deps = {},
|
|
210
|
-
} = opts;
|
|
211
|
-
|
|
212
|
-
if (!workdir) throw new Error("workdir is required");
|
|
213
|
-
if (!logsDir) throw new Error("logsDir is required");
|
|
214
|
-
|
|
215
|
-
mkdirSync(logsDir, { recursive: true });
|
|
216
|
-
ensureConductorRegistry();
|
|
217
|
-
|
|
218
|
-
const createConductorImpl = _deps.createConductor || createConductor;
|
|
219
|
-
const createRegistryImpl = _deps.createRegistry || importedCreateRegistry;
|
|
220
|
-
const ensureWorktreeImpl = _deps.ensureWorktree || ensureWorktree;
|
|
221
|
-
const prepareIntegrationBranchImpl =
|
|
222
|
-
_deps.prepareIntegrationBranch || prepareIntegrationBranch;
|
|
223
|
-
const rebaseShardOntoIntegrationImpl =
|
|
224
|
-
_deps.rebaseShardOntoIntegration || rebaseShardOntoIntegration;
|
|
225
|
-
const cleanupWorktreeImpl =
|
|
226
|
-
_deps.cleanupWorktree || cleanupWorktree || pruneWorktree;
|
|
227
|
-
const cleanupShardProcessesImpl =
|
|
228
|
-
_deps.cleanupShardProcesses || cleanupShardProcesses;
|
|
229
|
-
const registerSwarmShardImpl = _deps.registerSwarmShard || registerSwarmShard;
|
|
230
|
-
const preserveWorktreePatchImpl =
|
|
231
|
-
_deps.preserveWorktreePatch || preserveWorktreePatch;
|
|
232
|
-
const ensureHubAliveImpl = hasOwnDep(_deps, "ensureHubAlive")
|
|
233
|
-
? _deps.ensureHubAlive
|
|
234
|
-
: ensureHubAliveFn;
|
|
235
|
-
const getHubInfoImpl = hasOwnDep(_deps, "getHubInfo")
|
|
236
|
-
? _deps.getHubInfo
|
|
237
|
-
: getHubInfoFn;
|
|
238
|
-
const emitter = new EventEmitter();
|
|
239
|
-
const eventLog = createEventLog(join(logsDir, "swarm-events.jsonl"));
|
|
240
|
-
const { registry: sharedRegistry, fallbackReason: meshRegistryFallback } =
|
|
241
|
-
createSharedRegistry(createRegistryImpl);
|
|
242
|
-
|
|
243
|
-
let state = SWARM_STATES.PLANNING;
|
|
244
|
-
let plan = null;
|
|
245
|
-
let lockManager = null;
|
|
246
|
-
|
|
247
|
-
/** @type {Map<string, { conductor, shardConfig, result, status }>} */
|
|
248
|
-
const workers = new Map();
|
|
249
|
-
|
|
250
|
-
/** @type {Map<string, { conductor, shardConfig }>} redundant workers for critical shards */
|
|
251
|
-
const redundantWorkers = new Map();
|
|
252
|
-
|
|
253
|
-
/** @type {Set<string>} shards that have fully completed (not just launched) */
|
|
254
|
-
const completedShards = new Set();
|
|
255
|
-
const cleanedWorktreePaths = new Set();
|
|
256
|
-
const processCleanupTotals = {
|
|
257
|
-
totalKilled: 0,
|
|
258
|
-
byCategory: {},
|
|
259
|
-
};
|
|
260
|
-
|
|
261
|
-
const results = new Map(); // shardName → validated result
|
|
262
|
-
const failures = new Map(); // shardName → failure info
|
|
263
|
-
let integrationResult = null;
|
|
264
|
-
let resolveIntegrationPromise = null;
|
|
265
|
-
let integrationPromiseState = {
|
|
266
|
-
state: "idle",
|
|
267
|
-
startedAt: null,
|
|
268
|
-
settledAt: null,
|
|
269
|
-
partial: false,
|
|
270
|
-
integrated: [],
|
|
271
|
-
failed: [],
|
|
272
|
-
integrationFailures: [],
|
|
273
|
-
skipped: [],
|
|
274
|
-
integrationBranch: null,
|
|
275
|
-
report: [],
|
|
276
|
-
processCleanup: {
|
|
277
|
-
totalKilled: 0,
|
|
278
|
-
byCategory: {},
|
|
279
|
-
},
|
|
280
|
-
error: null,
|
|
281
|
-
};
|
|
282
|
-
const integrationPromise = new Promise((resolve) => {
|
|
283
|
-
resolveIntegrationPromise = resolve;
|
|
284
|
-
});
|
|
285
|
-
|
|
286
|
-
if (meshRegistryFallback) {
|
|
287
|
-
eventLog.append("mesh_registry_fallback", { reason: meshRegistryFallback });
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
// ── State machine ───────────────────────────────────────────
|
|
291
|
-
|
|
292
|
-
function setState(next, reason = "") {
|
|
293
|
-
const prev = state;
|
|
294
|
-
state = next;
|
|
295
|
-
eventLog.append("swarm_state", { from: prev, to: next, reason });
|
|
296
|
-
emitter.emit("stateChange", { from: prev, to: next, reason });
|
|
297
|
-
// Best-effort phase tracking via phase-manager
|
|
298
|
-
void writeSwarmPhaseSafely(
|
|
299
|
-
plan?.runId || plan?.id || `swarm-${Date.now()}`,
|
|
300
|
-
next,
|
|
301
|
-
);
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
async function writeSwarmPhaseSafely(runId, swarmState) {
|
|
305
|
-
if (!runId) return;
|
|
306
|
-
const mapping = {
|
|
307
|
-
[SWARM_STATES.LAUNCHING]: ["Execution", "active"],
|
|
308
|
-
[SWARM_STATES.RUNNING]: ["Execution", "active"],
|
|
309
|
-
[SWARM_STATES.INTEGRATING]: ["Validation", "active"],
|
|
310
|
-
[SWARM_STATES.VALIDATING]: ["Validation", "active"],
|
|
311
|
-
[SWARM_STATES.COMPLETED]: ["Validation", "complete"],
|
|
312
|
-
[SWARM_STATES.FAILED]: ["Validation", "failed"],
|
|
313
|
-
};
|
|
314
|
-
const target = mapping[swarmState];
|
|
315
|
-
if (!target) return;
|
|
316
|
-
try {
|
|
317
|
-
const mod = await import("@triflux/core/hub/lib/phase-manager.mjs");
|
|
318
|
-
if (typeof mod?.writePhase === "function") {
|
|
319
|
-
await mod.writePhase(runId, target[0], target[1]);
|
|
320
|
-
}
|
|
321
|
-
if (typeof mod?.syncToGstack === "function") {
|
|
322
|
-
const slug = process.cwd().split(/[\\/]/).pop();
|
|
323
|
-
await mod.syncToGstack(runId, slug);
|
|
324
|
-
}
|
|
325
|
-
} catch {
|
|
326
|
-
/* silent — phase tracking is best-effort */
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
function markIntegrationPromisePending() {
|
|
331
|
-
if (integrationPromiseState.state !== "idle") return;
|
|
332
|
-
integrationPromiseState = {
|
|
333
|
-
...integrationPromiseState,
|
|
334
|
-
state: "pending",
|
|
335
|
-
startedAt: Date.now(),
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
function settleIntegrationPromise(payload) {
|
|
340
|
-
if (integrationPromiseState.state === "fulfilled" && integrationResult) {
|
|
341
|
-
return integrationResult;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
const processCleanup = payload.processCleanup || getProcessCleanupSummary();
|
|
345
|
-
|
|
346
|
-
integrationResult = Object.freeze({
|
|
347
|
-
integrated: Object.freeze([...(payload.integrated || [])]),
|
|
348
|
-
failed: Object.freeze([...(payload.failed || [])]),
|
|
349
|
-
integrationFailures: Object.freeze([
|
|
350
|
-
...(payload.integrationFailures || []),
|
|
351
|
-
]),
|
|
352
|
-
skipped: Object.freeze([...(payload.skipped || [])]),
|
|
353
|
-
integrationBranch: payload.integrationBranch || null,
|
|
354
|
-
report: Object.freeze(
|
|
355
|
-
(payload.report || []).map((row) => Object.freeze({ ...row })),
|
|
356
|
-
),
|
|
357
|
-
results: Object.freeze([...(payload.results || [])]),
|
|
358
|
-
processCleanup: Object.freeze({
|
|
359
|
-
totalKilled: processCleanup.totalKilled || 0,
|
|
360
|
-
byCategory: Object.freeze({ ...(processCleanup.byCategory || {}) }),
|
|
361
|
-
}),
|
|
362
|
-
partial:
|
|
363
|
-
payload.partial ??
|
|
364
|
-
(Array.isArray(payload.failed) && payload.failed.length > 0),
|
|
365
|
-
error: payload.error || null,
|
|
366
|
-
});
|
|
367
|
-
|
|
368
|
-
integrationPromiseState = {
|
|
369
|
-
state: "fulfilled",
|
|
370
|
-
startedAt: integrationPromiseState.startedAt ?? Date.now(),
|
|
371
|
-
settledAt: Date.now(),
|
|
372
|
-
partial: integrationResult.partial,
|
|
373
|
-
integrated: [...integrationResult.integrated],
|
|
374
|
-
failed: [...integrationResult.failed],
|
|
375
|
-
integrationFailures: [...integrationResult.integrationFailures],
|
|
376
|
-
skipped: [...integrationResult.skipped],
|
|
377
|
-
integrationBranch: integrationResult.integrationBranch,
|
|
378
|
-
report: integrationResult.report.map((row) => ({ ...row })),
|
|
379
|
-
processCleanup: integrationResult.processCleanup,
|
|
380
|
-
error: integrationResult.error,
|
|
381
|
-
};
|
|
382
|
-
|
|
383
|
-
resolveIntegrationPromise?.(integrationResult);
|
|
384
|
-
return integrationResult;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
function getProcessCleanupSummary() {
|
|
388
|
-
return {
|
|
389
|
-
totalKilled: processCleanupTotals.totalKilled,
|
|
390
|
-
byCategory: { ...processCleanupTotals.byCategory },
|
|
391
|
-
};
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
function recordProcessCleanup(result) {
|
|
395
|
-
if (!result) return getProcessCleanupSummary();
|
|
396
|
-
|
|
397
|
-
const killed = Number(result.killed ?? result.totalKilled ?? 0);
|
|
398
|
-
if (Number.isFinite(killed) && killed > 0) {
|
|
399
|
-
processCleanupTotals.totalKilled += killed;
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
for (const [category, count] of Object.entries(result.byCategory || {})) {
|
|
403
|
-
const numeric = Number(count);
|
|
404
|
-
if (!Number.isFinite(numeric) || numeric <= 0) continue;
|
|
405
|
-
processCleanupTotals.byCategory[category] =
|
|
406
|
-
(processCleanupTotals.byCategory[category] || 0) + numeric;
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
return getProcessCleanupSummary();
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
function normalizeIntegrationReportRow(row) {
|
|
413
|
-
return {
|
|
414
|
-
shard: row.shard,
|
|
415
|
-
strategy: row.strategy || "failed",
|
|
416
|
-
commits:
|
|
417
|
-
Number.isFinite(Number(row.commits)) && row.commits !== null
|
|
418
|
-
? Number(row.commits)
|
|
419
|
-
: null,
|
|
420
|
-
finalSha: row.finalSha || null,
|
|
421
|
-
notes: row.notes || "",
|
|
422
|
-
recoveryPatch: row.recoveryPatch || null,
|
|
423
|
-
};
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
function git(args, cwd = workdir) {
|
|
427
|
-
return new Promise((resolve, reject) => {
|
|
428
|
-
execFile(
|
|
429
|
-
"git",
|
|
430
|
-
args,
|
|
431
|
-
{ cwd, windowsHide: true, timeout: 30_000 },
|
|
432
|
-
(err, stdout, stderr) => {
|
|
433
|
-
if (err) {
|
|
434
|
-
reject(
|
|
435
|
-
new Error(
|
|
436
|
-
`git ${args[0]} failed: ${stderr?.trim() || err.message}`,
|
|
437
|
-
),
|
|
438
|
-
);
|
|
439
|
-
return;
|
|
440
|
-
}
|
|
441
|
-
resolve(stdout.trim());
|
|
442
|
-
},
|
|
443
|
-
);
|
|
444
|
-
});
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
// Injectable git seam (#394). Defaults to the execFile closure above; tests
|
|
448
|
-
// override via `_deps.git` to drive the redundant-win commit-evidence gate
|
|
449
|
-
// deterministically without provisioning a real worktree.
|
|
450
|
-
const gitRun = _deps.git || git;
|
|
451
|
-
|
|
452
|
-
function computeAuthoritativeStatus(shardName, workerEntry, sessions) {
|
|
453
|
-
const failureInfo = failures.get(shardName) || null;
|
|
454
|
-
if (failureInfo) {
|
|
455
|
-
return {
|
|
456
|
-
status: "failed",
|
|
457
|
-
reason: failureInfo.mode || failureInfo.reason || "failed",
|
|
458
|
-
};
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
if (results.has(shardName)) {
|
|
462
|
-
return { status: "done", reason: "integrated" };
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
if (!sessions.length) {
|
|
466
|
-
if (completedShards.has(shardName)) {
|
|
467
|
-
return { status: "done", reason: "awaiting_integration" };
|
|
468
|
-
}
|
|
469
|
-
return { status: "running", reason: "no_sessions" };
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
const states = sessions.map((session) => session.state);
|
|
473
|
-
if (states.every((stateValue) => stateValue === STATES.COMPLETED)) {
|
|
474
|
-
return { status: "done", reason: "awaiting_integration" };
|
|
475
|
-
}
|
|
476
|
-
if (states.some((stateValue) => stateValue === STATES.INPUT_WAIT)) {
|
|
477
|
-
return { status: "blocked", reason: "user_input" };
|
|
478
|
-
}
|
|
479
|
-
if (states.some((stateValue) => stateValue === STATES.STALLED)) {
|
|
480
|
-
return { status: "stalled", reason: "health_probe_stall" };
|
|
481
|
-
}
|
|
482
|
-
if (
|
|
483
|
-
states.some(
|
|
484
|
-
(stateValue) =>
|
|
485
|
-
stateValue === STATES.FAILED || stateValue === STATES.DEAD,
|
|
486
|
-
)
|
|
487
|
-
) {
|
|
488
|
-
return { status: "failed", reason: "session_terminal" };
|
|
489
|
-
}
|
|
490
|
-
if (
|
|
491
|
-
states.some(
|
|
492
|
-
(stateValue) =>
|
|
493
|
-
stateValue === STATES.STARTING ||
|
|
494
|
-
stateValue === STATES.RESTARTING ||
|
|
495
|
-
stateValue === STATES.INIT,
|
|
496
|
-
)
|
|
497
|
-
) {
|
|
498
|
-
return { status: "running", reason: "starting" };
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
return { status: "running", reason: "healthy" };
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
async function collectCommitEvidence(worker, integrationBranch) {
|
|
505
|
-
const branchName = worker?.branchName || null;
|
|
506
|
-
const evidence = {
|
|
507
|
-
branchName,
|
|
508
|
-
integrationBranch,
|
|
509
|
-
commitsAhead: 0,
|
|
510
|
-
dirty: false,
|
|
511
|
-
dirtyFiles: [],
|
|
512
|
-
headCommit: null,
|
|
513
|
-
ok: false,
|
|
514
|
-
error: null,
|
|
515
|
-
};
|
|
516
|
-
|
|
517
|
-
if (!branchName) {
|
|
518
|
-
evidence.error = "missing_branch_name";
|
|
519
|
-
return evidence;
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
try {
|
|
523
|
-
evidence.commitsAhead =
|
|
524
|
-
Number.parseInt(
|
|
525
|
-
await gitRun([
|
|
526
|
-
"rev-list",
|
|
527
|
-
"--count",
|
|
528
|
-
`${integrationBranch}..${branchName}`,
|
|
529
|
-
]),
|
|
530
|
-
10,
|
|
531
|
-
) || 0;
|
|
532
|
-
} catch (err) {
|
|
533
|
-
evidence.error = err.message;
|
|
534
|
-
return evidence;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
try {
|
|
538
|
-
evidence.headCommit = await gitRun(["rev-parse", branchName]);
|
|
539
|
-
} catch {
|
|
540
|
-
/* best-effort */
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
if (worker?.worktreePath && !worker?.shardConfig?.host) {
|
|
544
|
-
try {
|
|
545
|
-
const rawStatus = await gitRun(
|
|
546
|
-
["status", "--short"],
|
|
547
|
-
worker.worktreePath,
|
|
548
|
-
);
|
|
549
|
-
// Only explicitly expected lifecycle deletions are filtered. Plugin
|
|
550
|
-
// metadata deletions remain dirty because Codex workers need them.
|
|
551
|
-
evidence.dirtyFiles = extractDirtyFiles(
|
|
552
|
-
rawStatus,
|
|
553
|
-
EXPECTED_WORKTREE_DELETIONS,
|
|
554
|
-
);
|
|
555
|
-
evidence.dirty = evidence.dirtyFiles.length > 0;
|
|
556
|
-
} catch (err) {
|
|
557
|
-
evidence.error = evidence.error || err.message;
|
|
558
|
-
return evidence;
|
|
559
|
-
}
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
evidence.ok = evidence.commitsAhead > 0 && evidence.dirty === false;
|
|
563
|
-
return evidence;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
// ── Worker lifecycle ────────────────────────────────────────
|
|
567
|
-
|
|
568
|
-
function safeSessionPart(value) {
|
|
569
|
-
return String(value || "unknown")
|
|
570
|
-
.replace(/[^a-zA-Z0-9._-]+/g, "-")
|
|
571
|
-
.replace(/^-+|-+$/g, "")
|
|
572
|
-
.slice(0, 80);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
function buildSwarmSessionId(shard) {
|
|
576
|
-
const short8 = crypto.randomUUID().replace(/-/g, "").slice(0, 8);
|
|
577
|
-
return `swarm-${safeSessionPart(runId)}-${safeSessionPart(shard.name)}-${short8}`;
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
function getShardDisplayPosition(shard) {
|
|
581
|
-
const shardIndex = plan?.shards?.findIndex(
|
|
582
|
-
(item) => item.name === shard.name,
|
|
583
|
-
);
|
|
584
|
-
if (!plan?.shards?.length || shardIndex < 0) return {};
|
|
585
|
-
return {
|
|
586
|
-
shardIndex: shardIndex + 1,
|
|
587
|
-
shardCount: plan.shards.length,
|
|
588
|
-
};
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
async function closeNativeBridgeRegistration(worker, shardName, reason) {
|
|
592
|
-
if (worker?.nativeBridgeClosePromise) {
|
|
593
|
-
await worker.nativeBridgeClosePromise;
|
|
594
|
-
return;
|
|
595
|
-
}
|
|
596
|
-
const registration = worker?.nativeBridgeRegistration;
|
|
597
|
-
if (!registration || typeof registration.close !== "function") return;
|
|
598
|
-
worker.nativeBridgeClosePromise = (async () => {
|
|
599
|
-
let closed = false;
|
|
600
|
-
try {
|
|
601
|
-
await registration.close();
|
|
602
|
-
closed = true;
|
|
603
|
-
eventLog.append("native_bridge_registration_closed", {
|
|
604
|
-
shard: shardName,
|
|
605
|
-
reason,
|
|
606
|
-
});
|
|
607
|
-
} catch (err) {
|
|
608
|
-
eventLog.append("native_bridge_registration_close_failed", {
|
|
609
|
-
shard: shardName,
|
|
610
|
-
reason,
|
|
611
|
-
error: err.message,
|
|
612
|
-
});
|
|
613
|
-
} finally {
|
|
614
|
-
worker.nativeBridgeClosePromise = null;
|
|
615
|
-
if (closed) worker.nativeBridgeRegistration = null;
|
|
616
|
-
}
|
|
617
|
-
})();
|
|
618
|
-
await worker.nativeBridgeClosePromise;
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
async function maybeRegisterNativeBridgeShard(shard, sessionConfig) {
|
|
622
|
-
if (!nativeBridge) return null;
|
|
623
|
-
try {
|
|
624
|
-
const interactive = !shard.host && isInteractiveNativeBridgeShard(shard);
|
|
625
|
-
const registration = await registerSwarmShardImpl({
|
|
626
|
-
sessionId: sessionConfig.id,
|
|
627
|
-
cli: shard.agent,
|
|
628
|
-
role: shard.role || "worker",
|
|
629
|
-
swarmName: runId,
|
|
630
|
-
...getShardDisplayPosition(shard),
|
|
631
|
-
shardName: shard.name,
|
|
632
|
-
cwd: interactive
|
|
633
|
-
? shard.worktreePath || sessionConfig.workdir || workdir
|
|
634
|
-
: workdir,
|
|
635
|
-
host: shard.host || "local",
|
|
636
|
-
interactive,
|
|
637
|
-
_deps: {
|
|
638
|
-
warn(message) {
|
|
639
|
-
emitter.emit("warning", {
|
|
640
|
-
type: "native_bridge_registration_skipped",
|
|
641
|
-
shard: shard.name,
|
|
642
|
-
host: shard.host || "local",
|
|
643
|
-
message,
|
|
644
|
-
});
|
|
645
|
-
},
|
|
646
|
-
},
|
|
647
|
-
});
|
|
648
|
-
eventLog.append(
|
|
649
|
-
registration?.skipped
|
|
650
|
-
? "native_bridge_registration_skipped"
|
|
651
|
-
: "native_bridge_registration",
|
|
652
|
-
{
|
|
653
|
-
shard: shard.name,
|
|
654
|
-
sessionId: sessionConfig.id,
|
|
655
|
-
host: shard.host || "local",
|
|
656
|
-
displayName: registration?.displayName || null,
|
|
657
|
-
},
|
|
658
|
-
);
|
|
659
|
-
return registration;
|
|
660
|
-
} catch (err) {
|
|
661
|
-
eventLog.append("native_bridge_registration_failed", {
|
|
662
|
-
shard: shard.name,
|
|
663
|
-
sessionId: sessionConfig.id,
|
|
664
|
-
host: shard.host || "local",
|
|
665
|
-
error: err.message,
|
|
666
|
-
});
|
|
667
|
-
emitter.emit("warning", {
|
|
668
|
-
type: "native_bridge_registration_failed",
|
|
669
|
-
shard: shard.name,
|
|
670
|
-
host: shard.host || "local",
|
|
671
|
-
error: err.message,
|
|
672
|
-
});
|
|
673
|
-
return null;
|
|
674
|
-
}
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
function buildSessionConfig(shard) {
|
|
678
|
-
const config = {
|
|
679
|
-
id: buildSwarmSessionId(shard),
|
|
680
|
-
agent: shard.agent,
|
|
681
|
-
role: shard.role,
|
|
682
|
-
// #125: append Completion Protocol appendix so workers emit a
|
|
683
|
-
// sentinel-framed JSON payload that conductor can reliably capture.
|
|
684
|
-
prompt: buildWorkerPrompt(shard.prompt, {
|
|
685
|
-
leaseFiles: shard.files,
|
|
686
|
-
worktreePath: shard.worktreePath,
|
|
687
|
-
}),
|
|
688
|
-
workdir: shard.worktreePath || workdir,
|
|
689
|
-
mcpServers: shard.mcp,
|
|
690
|
-
worktreePath: shard.worktreePath || null,
|
|
691
|
-
branchName: shard.branchName || null,
|
|
692
|
-
// #90: shard는 worktree 격리가 본질. main 직접 커밋 방지 가드 env 주입.
|
|
693
|
-
branchGuard: true,
|
|
694
|
-
};
|
|
695
|
-
|
|
696
|
-
if (shard.worktreePath) {
|
|
697
|
-
return shard.host
|
|
698
|
-
? {
|
|
699
|
-
...config,
|
|
700
|
-
remote: true,
|
|
701
|
-
host: shard.host,
|
|
702
|
-
}
|
|
703
|
-
: config;
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
// Remote shard: use host's default_dir from hosts.json
|
|
707
|
-
if (shard.host && shard._remoteEnv) {
|
|
708
|
-
const hostConfig = getHostConfig(shard.host);
|
|
709
|
-
const remoteSrc = hostConfig?.default_dir || workdir;
|
|
710
|
-
const remoteDir = resolveRemoteDir(remoteSrc, shard._remoteEnv);
|
|
711
|
-
return {
|
|
712
|
-
...config,
|
|
713
|
-
remote: true,
|
|
714
|
-
host: shard.host,
|
|
715
|
-
workdir: remoteDir,
|
|
716
|
-
};
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
return config;
|
|
720
|
-
}
|
|
721
|
-
|
|
722
|
-
async function launchShard(shard, isRedundant = false) {
|
|
723
|
-
// Clone frozen shard so we can attach mutable runtime state (_remoteEnv)
|
|
724
|
-
shard = { ...shard };
|
|
725
|
-
let nativeBridgeRegistration = null;
|
|
726
|
-
|
|
727
|
-
const shardLogsDir = join(
|
|
728
|
-
logsDir,
|
|
729
|
-
isRedundant ? `${shard.name}-redundant` : shard.name,
|
|
730
|
-
);
|
|
731
|
-
mkdirSync(shardLogsDir, { recursive: true });
|
|
732
|
-
|
|
733
|
-
try {
|
|
734
|
-
// Remote shard: probe environment before conductor creation
|
|
735
|
-
if (shard.host && !shard._remoteEnv) {
|
|
736
|
-
try {
|
|
737
|
-
shard._remoteEnv = probeRemoteEnv(shard.host);
|
|
738
|
-
if (!shard._remoteEnv.claudePath) {
|
|
739
|
-
eventLog.append("remote_probe_no_claude", {
|
|
740
|
-
shard: shard.name,
|
|
741
|
-
host: shard.host,
|
|
742
|
-
});
|
|
743
|
-
if (!isRedundant) {
|
|
744
|
-
failures.set(shard.name, {
|
|
745
|
-
mode: FAILURE_MODES.F1_CRASH,
|
|
746
|
-
reason: `claude not found on ${shard.host}`,
|
|
747
|
-
});
|
|
748
|
-
}
|
|
749
|
-
return null;
|
|
750
|
-
}
|
|
751
|
-
eventLog.append("remote_probe_ok", {
|
|
752
|
-
shard: shard.name,
|
|
753
|
-
host: shard.host,
|
|
754
|
-
env: shard._remoteEnv,
|
|
755
|
-
});
|
|
756
|
-
} catch (err) {
|
|
757
|
-
eventLog.append("remote_probe_failed", {
|
|
758
|
-
shard: shard.name,
|
|
759
|
-
host: shard.host,
|
|
760
|
-
error: err.message,
|
|
761
|
-
});
|
|
762
|
-
if (!isRedundant) {
|
|
763
|
-
failures.set(shard.name, {
|
|
764
|
-
mode: FAILURE_MODES.F1_CRASH,
|
|
765
|
-
reason: `remote probe failed: ${err.message}`,
|
|
766
|
-
});
|
|
767
|
-
}
|
|
768
|
-
return null;
|
|
769
|
-
}
|
|
770
|
-
}
|
|
771
|
-
|
|
772
|
-
// Acquire file leases
|
|
773
|
-
if (!isRedundant) {
|
|
774
|
-
const leaseResult = lockManager.acquire(shard.name, shard.files);
|
|
775
|
-
if (!leaseResult.ok) {
|
|
776
|
-
eventLog.append("lease_denied", {
|
|
777
|
-
shard: shard.name,
|
|
778
|
-
conflicts: leaseResult.conflicts,
|
|
779
|
-
});
|
|
780
|
-
failures.set(shard.name, {
|
|
781
|
-
mode: FAILURE_MODES.F4_LEASE_VIOLATION,
|
|
782
|
-
conflicts: leaseResult.conflicts,
|
|
783
|
-
});
|
|
784
|
-
return null;
|
|
785
|
-
}
|
|
786
|
-
}
|
|
787
|
-
|
|
788
|
-
const worktreeMeta = await ensureWorktreeImpl({
|
|
789
|
-
slug: isRedundant ? `${shard.name}-redundant` : shard.name,
|
|
790
|
-
runId,
|
|
791
|
-
rootDir: workdir,
|
|
792
|
-
baseBranch,
|
|
793
|
-
host: shard.host,
|
|
794
|
-
remoteEnv: shard._remoteEnv,
|
|
795
|
-
});
|
|
796
|
-
shard.worktreePath = worktreeMeta.worktreePath;
|
|
797
|
-
shard.branchName = worktreeMeta.branchName;
|
|
798
|
-
|
|
799
|
-
const conductor = createConductorImpl({
|
|
800
|
-
logsDir: shardLogsDir,
|
|
801
|
-
maxRestarts,
|
|
802
|
-
graceMs,
|
|
803
|
-
probeOpts,
|
|
804
|
-
meshRegistry: sharedRegistry,
|
|
805
|
-
enableMesh: true,
|
|
806
|
-
});
|
|
807
|
-
|
|
808
|
-
// NOTE: conductor 는 `session.config.onCompleted({ sessionId })` 형태로
|
|
809
|
-
// 세션별 콜백만 호출한다 (factory 옵션은 무시). 따라서 sessionConfig 에
|
|
810
|
-
// 직접 주입해야 shardCompleted 이벤트가 발화된다. 2026-04-17 세션에서
|
|
811
|
-
// follow-up swarm 이 여기서 hang 했던 원인.
|
|
812
|
-
const sessionConfig = buildSessionConfig(shard);
|
|
813
|
-
nativeBridgeRegistration = await maybeRegisterNativeBridgeShard(
|
|
814
|
-
shard,
|
|
815
|
-
sessionConfig,
|
|
816
|
-
);
|
|
817
|
-
sessionConfig.onCompleted = ({ sessionId, completionPayload } = {}) =>
|
|
818
|
-
handleShardCompleted(
|
|
819
|
-
shard.name,
|
|
820
|
-
sessionId,
|
|
821
|
-
isRedundant,
|
|
822
|
-
completionPayload,
|
|
823
|
-
);
|
|
824
|
-
conductor.spawnSession(sessionConfig);
|
|
825
|
-
|
|
826
|
-
eventLog.append("shard_launched", {
|
|
827
|
-
shard: shard.name,
|
|
828
|
-
agent: shard.agent,
|
|
829
|
-
sessionId: sessionConfig.id,
|
|
830
|
-
isRedundant,
|
|
831
|
-
files: shard.files,
|
|
832
|
-
remote: Boolean(shard.host),
|
|
833
|
-
host: shard.host || null,
|
|
834
|
-
worktreePath: sessionConfig.worktreePath,
|
|
835
|
-
branchName: sessionConfig.branchName,
|
|
836
|
-
});
|
|
837
|
-
|
|
838
|
-
emitter.emit("shardLaunched", {
|
|
839
|
-
shardName: shard.name,
|
|
840
|
-
sessionId: sessionConfig.id,
|
|
841
|
-
agent: shard.agent,
|
|
842
|
-
isRedundant,
|
|
843
|
-
remote: Boolean(shard.host),
|
|
844
|
-
host: shard.host || null,
|
|
845
|
-
});
|
|
846
|
-
|
|
847
|
-
const entry = {
|
|
848
|
-
conductor,
|
|
849
|
-
shardConfig: shard,
|
|
850
|
-
sessionConfig,
|
|
851
|
-
startedAt: Date.now(),
|
|
852
|
-
worktreePath: sessionConfig.worktreePath,
|
|
853
|
-
branchName: sessionConfig.branchName,
|
|
854
|
-
nativeBridgeRegistration,
|
|
855
|
-
};
|
|
856
|
-
|
|
857
|
-
if (isRedundant) {
|
|
858
|
-
redundantWorkers.set(shard.name, entry);
|
|
859
|
-
} else {
|
|
860
|
-
workers.set(shard.name, entry);
|
|
861
|
-
}
|
|
862
|
-
|
|
863
|
-
// Listen for dead events (F1/F2/F3)
|
|
864
|
-
conductor.on("dead", ({ sessionId, reason }) => {
|
|
865
|
-
handleShardFailed(shard.name, sessionId, reason, isRedundant);
|
|
866
|
-
});
|
|
867
|
-
|
|
868
|
-
return entry;
|
|
869
|
-
} catch (err) {
|
|
870
|
-
if (nativeBridgeRegistration) {
|
|
871
|
-
await closeNativeBridgeRegistration(
|
|
872
|
-
{ nativeBridgeRegistration },
|
|
873
|
-
shard.name,
|
|
874
|
-
"launch_failed",
|
|
875
|
-
);
|
|
876
|
-
}
|
|
877
|
-
eventLog.append("shard_launch_failed", {
|
|
878
|
-
shard: shard.name,
|
|
879
|
-
isRedundant,
|
|
880
|
-
error: err.message,
|
|
881
|
-
});
|
|
882
|
-
if (!isRedundant) {
|
|
883
|
-
failures.set(shard.name, {
|
|
884
|
-
mode: FAILURE_MODES.F1_CRASH,
|
|
885
|
-
reason: err.message,
|
|
886
|
-
});
|
|
887
|
-
}
|
|
888
|
-
if (!isRedundant) {
|
|
889
|
-
lockManager.release(shard.name);
|
|
890
|
-
}
|
|
891
|
-
if (shard.worktreePath) {
|
|
892
|
-
await maybeCleanupWorktree(
|
|
893
|
-
shard.name,
|
|
894
|
-
{
|
|
895
|
-
worktreePath: shard.worktreePath,
|
|
896
|
-
branchName: shard.branchName,
|
|
897
|
-
},
|
|
898
|
-
shard,
|
|
899
|
-
{
|
|
900
|
-
branchName: shard.branchName,
|
|
901
|
-
force: true,
|
|
902
|
-
failureReason: `launch_failed:${err.message}`,
|
|
903
|
-
},
|
|
904
|
-
);
|
|
905
|
-
}
|
|
906
|
-
return null;
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
// ── Completion handling ─────────────────────────────────────
|
|
911
|
-
|
|
912
|
-
function handleShardCompleted(
|
|
913
|
-
shardName,
|
|
914
|
-
sessionId,
|
|
915
|
-
isRedundant,
|
|
916
|
-
completionPayload,
|
|
917
|
-
) {
|
|
918
|
-
eventLog.append("shard_completed", {
|
|
919
|
-
shard: shardName,
|
|
920
|
-
sessionId,
|
|
921
|
-
isRedundant,
|
|
922
|
-
hasPayload: completionPayload !== undefined,
|
|
923
|
-
});
|
|
924
|
-
const completedWorker = isRedundant
|
|
925
|
-
? redundantWorkers.get(shardName)
|
|
926
|
-
: workers.get(shardName);
|
|
927
|
-
|
|
928
|
-
// F7 — worker self-reported completion must include commits_made.
|
|
929
|
-
// Only enforce when payload is actually provided; legacy workers that
|
|
930
|
-
// don't emit a structured payload keep the F6 integration-time guard.
|
|
931
|
-
if (completionPayload !== undefined) {
|
|
932
|
-
const verdict = validateWorkerCompletion(completionPayload);
|
|
933
|
-
if (!verdict.ok) {
|
|
934
|
-
const reason = verdict.reason || "invalid_completion_payload";
|
|
935
|
-
|
|
936
|
-
if (isRedundant) {
|
|
937
|
-
eventLog.append("redundant_completion_rejected", {
|
|
938
|
-
shard: shardName,
|
|
939
|
-
sessionId,
|
|
940
|
-
reason,
|
|
941
|
-
});
|
|
942
|
-
redundantWorkers.delete(shardName);
|
|
943
|
-
if (completedWorker) {
|
|
944
|
-
void closeNativeBridgeRegistration(
|
|
945
|
-
completedWorker,
|
|
946
|
-
shardName,
|
|
947
|
-
"failed",
|
|
948
|
-
);
|
|
949
|
-
}
|
|
950
|
-
checkAllShardsCompleted();
|
|
951
|
-
return;
|
|
952
|
-
}
|
|
953
|
-
|
|
954
|
-
eventLog.append("no_worker_commit_report", {
|
|
955
|
-
shard: shardName,
|
|
956
|
-
sessionId,
|
|
957
|
-
reason,
|
|
958
|
-
});
|
|
959
|
-
if (completedWorker) {
|
|
960
|
-
void closeNativeBridgeRegistration(
|
|
961
|
-
completedWorker,
|
|
962
|
-
shardName,
|
|
963
|
-
"failed",
|
|
964
|
-
);
|
|
965
|
-
}
|
|
966
|
-
failures.set(shardName, {
|
|
967
|
-
mode: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
|
|
968
|
-
reason,
|
|
969
|
-
sessionId,
|
|
970
|
-
completionPayload,
|
|
971
|
-
});
|
|
972
|
-
lockManager.release(shardName);
|
|
973
|
-
|
|
974
|
-
const worker = workers.get(shardName);
|
|
975
|
-
const shard = plan?.shards.find((item) => item.name === shardName);
|
|
976
|
-
if (worker && shard) {
|
|
977
|
-
const cleanupPromise = maybeCleanupWorktree(
|
|
978
|
-
shardName,
|
|
979
|
-
worker,
|
|
980
|
-
shard,
|
|
981
|
-
{
|
|
982
|
-
force: true,
|
|
983
|
-
failureReason: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
|
|
984
|
-
},
|
|
985
|
-
);
|
|
986
|
-
worker.cleanupPromise = cleanupPromise;
|
|
987
|
-
void cleanupPromise
|
|
988
|
-
.finally(() => {
|
|
989
|
-
if (worker.cleanupPromise === cleanupPromise) {
|
|
990
|
-
worker.cleanupPromise = null;
|
|
991
|
-
}
|
|
992
|
-
})
|
|
993
|
-
.catch(() => {});
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
const redundant = redundantWorkers.get(shardName);
|
|
997
|
-
if (redundant) {
|
|
998
|
-
void redundant.conductor.shutdown("primary_failed_f7");
|
|
999
|
-
}
|
|
1000
|
-
|
|
1001
|
-
emitter.emit("shardFailed", {
|
|
1002
|
-
shardName,
|
|
1003
|
-
failureMode: FAILURE_MODES.F7_WORKER_DID_NOT_COMMIT,
|
|
1004
|
-
reason,
|
|
1005
|
-
});
|
|
1006
|
-
|
|
1007
|
-
cascadeDependencyFailure(shardName);
|
|
1008
|
-
checkAllShardsCompleted();
|
|
1009
|
-
return;
|
|
1010
|
-
}
|
|
1011
|
-
|
|
1012
|
-
if (
|
|
1013
|
-
completionPayload?.status === "failed" ||
|
|
1014
|
-
completionPayload?.status === "blocked"
|
|
1015
|
-
) {
|
|
1016
|
-
const detail =
|
|
1017
|
-
typeof completionPayload.reason === "string" &&
|
|
1018
|
-
completionPayload.reason.length > 0
|
|
1019
|
-
? completionPayload.reason
|
|
1020
|
-
: "unspecified";
|
|
1021
|
-
const reason = `worker_reported_${completionPayload.status}:${detail}`;
|
|
1022
|
-
|
|
1023
|
-
if (isRedundant) {
|
|
1024
|
-
eventLog.append("redundant_completion_rejected", {
|
|
1025
|
-
shard: shardName,
|
|
1026
|
-
sessionId,
|
|
1027
|
-
reason,
|
|
1028
|
-
});
|
|
1029
|
-
redundantWorkers.delete(shardName);
|
|
1030
|
-
if (completedWorker) {
|
|
1031
|
-
void closeNativeBridgeRegistration(
|
|
1032
|
-
completedWorker,
|
|
1033
|
-
shardName,
|
|
1034
|
-
"failed",
|
|
1035
|
-
);
|
|
1036
|
-
}
|
|
1037
|
-
checkAllShardsCompleted();
|
|
1038
|
-
return;
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
eventLog.append("worker_reported_non_ok_completion", {
|
|
1042
|
-
shard: shardName,
|
|
1043
|
-
sessionId,
|
|
1044
|
-
status: completionPayload.status,
|
|
1045
|
-
reason,
|
|
1046
|
-
});
|
|
1047
|
-
if (completedWorker) {
|
|
1048
|
-
void closeNativeBridgeRegistration(
|
|
1049
|
-
completedWorker,
|
|
1050
|
-
shardName,
|
|
1051
|
-
"failed",
|
|
1052
|
-
);
|
|
1053
|
-
}
|
|
1054
|
-
failures.set(shardName, {
|
|
1055
|
-
mode: classifyFailure(reason),
|
|
1056
|
-
reason,
|
|
1057
|
-
sessionId,
|
|
1058
|
-
completionPayload,
|
|
1059
|
-
});
|
|
1060
|
-
lockManager.release(shardName);
|
|
1061
|
-
|
|
1062
|
-
const redundant = redundantWorkers.get(shardName);
|
|
1063
|
-
if (redundant) {
|
|
1064
|
-
void redundant.conductor.shutdown("primary_reported_non_ok");
|
|
1065
|
-
}
|
|
1066
|
-
|
|
1067
|
-
emitter.emit("shardFailed", {
|
|
1068
|
-
shardName,
|
|
1069
|
-
failureMode: classifyFailure(reason),
|
|
1070
|
-
reason,
|
|
1071
|
-
});
|
|
1072
|
-
|
|
1073
|
-
cascadeDependencyFailure(shardName);
|
|
1074
|
-
checkAllShardsCompleted();
|
|
1075
|
-
return;
|
|
1076
|
-
}
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
if (isRedundant) {
|
|
1080
|
-
// #394: a redundant worker may only "win" (and SIGKILL the still-running
|
|
1081
|
-
// primary) once it has produced real, integratable commits. The earlier
|
|
1082
|
-
// F7 payload check trusts a self-report; agy/gemini redundant workers are
|
|
1083
|
-
// known to signal completion with zero commits (and legacy workers emit
|
|
1084
|
-
// no payload at all, bypassing F7). Gate the win on authoritative git
|
|
1085
|
-
// evidence collected now — before any kill — so a phantom completion can
|
|
1086
|
-
// never kill the worker that is doing the actual work.
|
|
1087
|
-
void finalizeRedundantWin(shardName, sessionId, completedWorker);
|
|
1088
|
-
return;
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
|
-
if (completedWorker) {
|
|
1092
|
-
void closeNativeBridgeRegistration(
|
|
1093
|
-
completedWorker,
|
|
1094
|
-
shardName,
|
|
1095
|
-
"completed",
|
|
1096
|
-
);
|
|
1097
|
-
}
|
|
1098
|
-
|
|
1099
|
-
completedShards.add(shardName);
|
|
1100
|
-
|
|
1101
|
-
// Primary completed — kill redundant if exists
|
|
1102
|
-
const redundant = redundantWorkers.get(shardName);
|
|
1103
|
-
if (redundant) {
|
|
1104
|
-
void redundant.conductor.shutdown("primary_completed_first");
|
|
1105
|
-
}
|
|
1106
|
-
|
|
1107
|
-
emitter.emit("shardCompleted", { shardName, sessionId, isRedundant });
|
|
1108
|
-
checkAllShardsCompleted();
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
/**
|
|
1112
|
-
* Gate a redundant worker's completion on authoritative git commit evidence
|
|
1113
|
-
* before it is allowed to win the shard and SIGKILL the primary (#394).
|
|
1114
|
-
*
|
|
1115
|
-
* A redundant worker with no commits ahead of the base branch is NOT a valid
|
|
1116
|
-
* winner: it is rejected, the primary is left running, and the shard is not
|
|
1117
|
-
* marked complete. Only a worker with `commitsAhead > 0` promotes itself and
|
|
1118
|
-
* terminates the primary.
|
|
1119
|
-
*
|
|
1120
|
-
* @param {string} shardName
|
|
1121
|
-
* @param {string} sessionId
|
|
1122
|
-
* @param {object} redundantWorker — entry fetched before this async hop
|
|
1123
|
-
*/
|
|
1124
|
-
async function finalizeRedundantWin(shardName, sessionId, redundantWorker) {
|
|
1125
|
-
const redundant = redundantWorker || redundantWorkers.get(shardName);
|
|
1126
|
-
if (!redundant) {
|
|
1127
|
-
// The primary already completed/failed and cleared the redundant entry
|
|
1128
|
-
// (primary_completed_first / primary_failed_f7). Nothing to promote.
|
|
1129
|
-
checkAllShardsCompleted();
|
|
1130
|
-
return;
|
|
1131
|
-
}
|
|
1132
|
-
|
|
1133
|
-
// A redundant worker may win only while the shard is still undecided. This
|
|
1134
|
-
// is checked TWICE — once up front (a cheap early-out before the git
|
|
1135
|
-
// round-trip) and again AFTER the async evidence hop. The second check is
|
|
1136
|
-
// the load-bearing one: the primary can complete/fail while
|
|
1137
|
-
// collectCommitEvidence() is in flight, and without re-checking, a slow git
|
|
1138
|
-
// probe would resume and flip an already-decided winner via
|
|
1139
|
-
// `workers.set(shardName, redundant)` (#394 TOCTOU race).
|
|
1140
|
-
const retireSupersededRedundant = (phase) => {
|
|
1141
|
-
eventLog.append("redundant_win_superseded", {
|
|
1142
|
-
shard: shardName,
|
|
1143
|
-
sessionId,
|
|
1144
|
-
phase,
|
|
1145
|
-
});
|
|
1146
|
-
redundantWorkers.delete(shardName);
|
|
1147
|
-
void closeNativeBridgeRegistration(redundant, shardName, "failed");
|
|
1148
|
-
checkAllShardsCompleted();
|
|
1149
|
-
};
|
|
1150
|
-
|
|
1151
|
-
if (completedShards.has(shardName) || failures.has(shardName)) {
|
|
1152
|
-
retireSupersededRedundant("before_evidence");
|
|
1153
|
-
return;
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
let evidence;
|
|
1157
|
-
try {
|
|
1158
|
-
evidence = await collectCommitEvidence(redundant, baseBranch);
|
|
1159
|
-
} catch (err) {
|
|
1160
|
-
evidence = { commitsAhead: 0, dirty: false, error: err.message };
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
// Re-check after the await: the primary may have won while we waited.
|
|
1164
|
-
if (completedShards.has(shardName) || failures.has(shardName)) {
|
|
1165
|
-
retireSupersededRedundant("after_evidence");
|
|
1166
|
-
return;
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
// A redundant worker may win only with CLEAN, integratable commits — the
|
|
1170
|
-
// exact bar integration applies (collectCommitEvidence.ok =
|
|
1171
|
-
// commitsAhead > 0 && !dirty). A weaker gate (commits-only) would promote a
|
|
1172
|
-
// dirty-but-committed redundant, kill the primary, then fail integration on
|
|
1173
|
-
// that same dirty evidence (F6) — losing the shard the primary could have
|
|
1174
|
-
// delivered (#394 re-review). Reject anything integration would not accept.
|
|
1175
|
-
if (!evidence?.ok) {
|
|
1176
|
-
const reason = evidence?.error
|
|
1177
|
-
? "evidence_error"
|
|
1178
|
-
: evidence?.dirty
|
|
1179
|
-
? "dirty_worktree"
|
|
1180
|
-
: "no_commit";
|
|
1181
|
-
eventLog.append("redundant_win_rejected", {
|
|
1182
|
-
shard: shardName,
|
|
1183
|
-
sessionId,
|
|
1184
|
-
reason,
|
|
1185
|
-
commitsAhead: evidence?.commitsAhead || 0,
|
|
1186
|
-
dirty: Boolean(evidence?.dirty),
|
|
1187
|
-
error: evidence?.error || null,
|
|
1188
|
-
});
|
|
1189
|
-
redundantWorkers.delete(shardName);
|
|
1190
|
-
void closeNativeBridgeRegistration(redundant, shardName, "failed");
|
|
1191
|
-
// Re-check in case the primary already reached a terminal state while we
|
|
1192
|
-
// were collecting evidence.
|
|
1193
|
-
checkAllShardsCompleted();
|
|
1194
|
-
return;
|
|
1195
|
-
}
|
|
1196
|
-
|
|
1197
|
-
// Clean, integratable commit evidence — promote the redundant worker and
|
|
1198
|
-
// kill the primary if it is still running.
|
|
1199
|
-
const primary = workers.get(shardName);
|
|
1200
|
-
workers.set(shardName, redundant);
|
|
1201
|
-
redundantWorkers.delete(shardName);
|
|
1202
|
-
completedShards.add(shardName);
|
|
1203
|
-
void closeNativeBridgeRegistration(redundant, shardName, "completed");
|
|
1204
|
-
if (primary && !isTerminal(primary)) {
|
|
1205
|
-
eventLog.append("redundant_wins", {
|
|
1206
|
-
shard: shardName,
|
|
1207
|
-
commitsAhead: evidence.commitsAhead,
|
|
1208
|
-
});
|
|
1209
|
-
void primary.conductor.shutdown("redundant_completed_first");
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
emitter.emit("shardCompleted", {
|
|
1213
|
-
shardName,
|
|
1214
|
-
sessionId,
|
|
1215
|
-
isRedundant: true,
|
|
1216
|
-
});
|
|
1217
|
-
checkAllShardsCompleted();
|
|
1218
|
-
}
|
|
1219
|
-
|
|
1220
|
-
function handleShardFailed(shardName, sessionId, reason, isRedundant) {
|
|
1221
|
-
const failureMode = classifyFailure(reason);
|
|
1222
|
-
|
|
1223
|
-
eventLog.append("shard_failed", {
|
|
1224
|
-
shard: shardName,
|
|
1225
|
-
sessionId,
|
|
1226
|
-
reason,
|
|
1227
|
-
failureMode,
|
|
1228
|
-
isRedundant,
|
|
1229
|
-
});
|
|
1230
|
-
|
|
1231
|
-
if (isRedundant) return; // redundant failure is non-critical
|
|
1232
|
-
const failedWorker = workers.get(shardName);
|
|
1233
|
-
if (failedWorker) {
|
|
1234
|
-
void closeNativeBridgeRegistration(failedWorker, shardName, "failed");
|
|
1235
|
-
}
|
|
1236
|
-
|
|
1237
|
-
// F2: Rate limit — try fallback agent
|
|
1238
|
-
if (failureMode === FAILURE_MODES.F2_RATE_LIMIT) {
|
|
1239
|
-
const shard = plan.shards.find((s) => s.name === shardName);
|
|
1240
|
-
if (shard) {
|
|
1241
|
-
const fallbackAgent = FALLBACK_AGENTS[shard.agent];
|
|
1242
|
-
if (fallbackAgent) {
|
|
1243
|
-
eventLog.append("fallback_agent", {
|
|
1244
|
-
shard: shardName,
|
|
1245
|
-
from: shard.agent,
|
|
1246
|
-
to: fallbackAgent,
|
|
1247
|
-
});
|
|
1248
|
-
const fallbackShard = { ...shard, agent: fallbackAgent };
|
|
1249
|
-
lockManager.release(shardName);
|
|
1250
|
-
void launchShard(fallbackShard);
|
|
1251
|
-
return;
|
|
1252
|
-
}
|
|
1253
|
-
}
|
|
1254
|
-
}
|
|
1255
|
-
|
|
1256
|
-
failures.set(shardName, { mode: failureMode, reason, sessionId });
|
|
1257
|
-
lockManager.release(shardName);
|
|
1258
|
-
|
|
1259
|
-
emitter.emit("shardFailed", { shardName, failureMode, reason });
|
|
1260
|
-
|
|
1261
|
-
// Cascade failure to dependents so they do not hold integration hostage
|
|
1262
|
-
// (BFS over plan.shards). Without this, a dead shard keeps dependents in
|
|
1263
|
-
// `pending` forever and integrateResults is never reachable.
|
|
1264
|
-
cascadeDependencyFailure(shardName);
|
|
1265
|
-
|
|
1266
|
-
checkAllShardsCompleted();
|
|
1267
|
-
}
|
|
1268
|
-
|
|
1269
|
-
function cascadeDependencyFailure(rootFailedShard) {
|
|
1270
|
-
const queue = [rootFailedShard];
|
|
1271
|
-
const visited = new Set([rootFailedShard]);
|
|
1272
|
-
while (queue.length > 0) {
|
|
1273
|
-
const failedName = queue.shift();
|
|
1274
|
-
for (const candidate of plan.shards) {
|
|
1275
|
-
if (visited.has(candidate.name)) continue;
|
|
1276
|
-
if (failures.has(candidate.name)) continue;
|
|
1277
|
-
if (completedShards.has(candidate.name)) continue;
|
|
1278
|
-
if (!candidate.depends?.includes(failedName)) continue;
|
|
1279
|
-
|
|
1280
|
-
visited.add(candidate.name);
|
|
1281
|
-
failures.set(candidate.name, {
|
|
1282
|
-
mode: FAILURE_MODES.F1_CRASH,
|
|
1283
|
-
reason: `dep_failed:${failedName}`,
|
|
1284
|
-
sessionId: null,
|
|
1285
|
-
});
|
|
1286
|
-
lockManager.release(candidate.name);
|
|
1287
|
-
eventLog.append("shard_blocked", {
|
|
1288
|
-
shard: candidate.name,
|
|
1289
|
-
failedDep: failedName,
|
|
1290
|
-
});
|
|
1291
|
-
emitter.emit("shardFailed", {
|
|
1292
|
-
shardName: candidate.name,
|
|
1293
|
-
failureMode: FAILURE_MODES.F1_CRASH,
|
|
1294
|
-
reason: `dep_failed:${failedName}`,
|
|
1295
|
-
});
|
|
1296
|
-
queue.push(candidate.name);
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
function classifyFailure(reason) {
|
|
1302
|
-
if (!reason) return FAILURE_MODES.F1_CRASH;
|
|
1303
|
-
const r = String(reason).toLowerCase();
|
|
1304
|
-
if (/rate.?limit|cooldown/u.test(r)) return FAILURE_MODES.F2_RATE_LIMIT;
|
|
1305
|
-
if (/stall|l1_stall|timeout/u.test(r)) return FAILURE_MODES.F3_STALL;
|
|
1306
|
-
if (/lease|violation/u.test(r)) return FAILURE_MODES.F4_LEASE_VIOLATION;
|
|
1307
|
-
if (/merge|conflict/u.test(r)) return FAILURE_MODES.F5_MERGE_CONFLICT;
|
|
1308
|
-
if (/no.?commit|dirty_worktree/u.test(r)) return FAILURE_MODES.F6_NO_COMMIT;
|
|
1309
|
-
return FAILURE_MODES.F1_CRASH;
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
function isTerminal(entry) {
|
|
1313
|
-
const snap = entry.conductor.getSnapshot();
|
|
1314
|
-
return snap.every(
|
|
1315
|
-
(s) => s.state === STATES.COMPLETED || s.state === STATES.DEAD,
|
|
1316
|
-
);
|
|
1317
|
-
}
|
|
1318
|
-
|
|
1319
|
-
// ── Integration ─────────────────────────────────────────────
|
|
1320
|
-
|
|
1321
|
-
function checkAllShardsCompleted() {
|
|
1322
|
-
if (state !== SWARM_STATES.RUNNING) return;
|
|
1323
|
-
|
|
1324
|
-
// `completedShards` + `failures` are set synchronously inside
|
|
1325
|
-
// handleShardCompleted / handleShardFailed, so they are the authoritative
|
|
1326
|
-
// terminal signal. `isTerminal(w)` queries conductor.getSnapshot() which
|
|
1327
|
-
// lags because the snapshot state transition is applied asynchronously
|
|
1328
|
-
// after the onCompleted callback chain that invoked this check. Using the
|
|
1329
|
-
// snapshot here races with its own producer and leaves integrateResults
|
|
1330
|
-
// unreachable (2026-04-17 swarm hang — required manual cherry-pick).
|
|
1331
|
-
const allDone = plan.mergeOrder.every(
|
|
1332
|
-
(name) => completedShards.has(name) || failures.has(name),
|
|
1333
|
-
);
|
|
1334
|
-
|
|
1335
|
-
if (allDone) {
|
|
1336
|
-
void integrateResults();
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
|
|
1340
|
-
/**
|
|
1341
|
-
* Validate a shard's output — check for file lease violations.
|
|
1342
|
-
* @param {string} shardName
|
|
1343
|
-
* @param {string[]} changedFiles — files the shard actually modified
|
|
1344
|
-
* @returns {{ ok: boolean, violations: Array }}
|
|
1345
|
-
*/
|
|
1346
|
-
function validateResult(shardName, changedFiles) {
|
|
1347
|
-
// Pass the shard's own lease so validateChanges can flag out-of-lease
|
|
1348
|
-
// writes to distribution-critical paths (regression of #115/#34 —
|
|
1349
|
-
// recovery patches were carrying `+++ /dev/null` deletions of
|
|
1350
|
-
// .claude-plugin/marketplace.json, undetected by lease-only checks).
|
|
1351
|
-
const ownLease = plan.leaseMap.get(shardName) || [];
|
|
1352
|
-
const violations = lockManager.validateChanges(shardName, changedFiles, {
|
|
1353
|
-
ownLease,
|
|
1354
|
-
});
|
|
1355
|
-
|
|
1356
|
-
eventLog.append("validate_result", {
|
|
1357
|
-
shard: shardName,
|
|
1358
|
-
changedFiles,
|
|
1359
|
-
violations,
|
|
1360
|
-
ok: violations.length === 0,
|
|
1361
|
-
});
|
|
1362
|
-
|
|
1363
|
-
return {
|
|
1364
|
-
ok: violations.length === 0,
|
|
1365
|
-
violations,
|
|
1366
|
-
};
|
|
1367
|
-
}
|
|
1368
|
-
|
|
1369
|
-
/**
|
|
1370
|
-
* Integrate results from all completed shards in merge order.
|
|
1371
|
-
* Uses git operations for conflict detection.
|
|
1372
|
-
*/
|
|
1373
|
-
async function integrateResults() {
|
|
1374
|
-
setState(SWARM_STATES.INTEGRATING, "all_shards_done");
|
|
1375
|
-
|
|
1376
|
-
const integrated = [];
|
|
1377
|
-
const integrationFailures = [];
|
|
1378
|
-
const integrationReport = [];
|
|
1379
|
-
const preIntegrationFailures = [...failures.keys()];
|
|
1380
|
-
let integrationBranch = null;
|
|
1381
|
-
|
|
1382
|
-
const appendIntegrationReport = (row) => {
|
|
1383
|
-
const normalized = normalizeIntegrationReportRow(row);
|
|
1384
|
-
integrationReport.push(normalized);
|
|
1385
|
-
eventLog.append("integration_shard_report", normalized);
|
|
1386
|
-
return normalized;
|
|
1387
|
-
};
|
|
1388
|
-
|
|
1389
|
-
const failureNotes = (shardName, fallback = "failed") => {
|
|
1390
|
-
const info = failures.get(shardName);
|
|
1391
|
-
if (!info) return fallback;
|
|
1392
|
-
return info.reason || info.mode || fallback;
|
|
1393
|
-
};
|
|
1394
|
-
|
|
1395
|
-
try {
|
|
1396
|
-
({ integrationBranch } = await prepareIntegrationBranchImpl({
|
|
1397
|
-
runId,
|
|
1398
|
-
baseBranch,
|
|
1399
|
-
rootDir: workdir,
|
|
1400
|
-
}));
|
|
1401
|
-
|
|
1402
|
-
for (const shardName of plan.mergeOrder) {
|
|
1403
|
-
if (failures.has(shardName)) {
|
|
1404
|
-
const worker = workers.get(shardName);
|
|
1405
|
-
if (worker?.cleanupPromise) await worker.cleanupPromise;
|
|
1406
|
-
eventLog.append("skip_failed_shard", { shard: shardName });
|
|
1407
|
-
appendIntegrationReport({
|
|
1408
|
-
shard: shardName,
|
|
1409
|
-
strategy: "failed",
|
|
1410
|
-
notes: `skipped: ${failureNotes(shardName)}`,
|
|
1411
|
-
});
|
|
1412
|
-
continue;
|
|
1413
|
-
}
|
|
1414
|
-
|
|
1415
|
-
const worker = workers.get(shardName);
|
|
1416
|
-
if (!worker) continue;
|
|
1417
|
-
|
|
1418
|
-
// Fetch remote shard branch to local (push-blocked hosts like Ultra4)
|
|
1419
|
-
const shard = plan.shards.find((s) => s.name === shardName);
|
|
1420
|
-
if (shard?.host && shard._remoteEnv) {
|
|
1421
|
-
const hostConfig = getHostConfig(shard.host, workdir);
|
|
1422
|
-
const sshUser = hostConfig?.ssh_user || shard.host;
|
|
1423
|
-
const remoteRepoPath = resolveRemoteDir(workdir, shard._remoteEnv);
|
|
1424
|
-
const fetchResult = await fetchRemoteShard({
|
|
1425
|
-
host: shard.host,
|
|
1426
|
-
sshUser,
|
|
1427
|
-
remoteRepoPath,
|
|
1428
|
-
branchName: worker.branchName || `swarm/${runId}/${shardName}`,
|
|
1429
|
-
rootDir: workdir,
|
|
1430
|
-
});
|
|
1431
|
-
|
|
1432
|
-
if (!fetchResult.ok) {
|
|
1433
|
-
eventLog.append("remote_fetch_failed", {
|
|
1434
|
-
shard: shardName,
|
|
1435
|
-
error: fetchResult.error,
|
|
1436
|
-
});
|
|
1437
|
-
const cleanup = await maybeCleanupWorktree(
|
|
1438
|
-
shardName,
|
|
1439
|
-
worker,
|
|
1440
|
-
shard,
|
|
1441
|
-
{
|
|
1442
|
-
force: true,
|
|
1443
|
-
failureReason: "remote_fetch_failed",
|
|
1444
|
-
},
|
|
1445
|
-
);
|
|
1446
|
-
integrationFailures.push(shardName);
|
|
1447
|
-
appendIntegrationReport({
|
|
1448
|
-
shard: shardName,
|
|
1449
|
-
strategy: "failed",
|
|
1450
|
-
notes: `remote_fetch_failed: ${fetchResult.error || "unknown"}`,
|
|
1451
|
-
recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
|
|
1452
|
-
});
|
|
1453
|
-
continue;
|
|
1454
|
-
}
|
|
1455
|
-
eventLog.append("remote_fetch_ok", {
|
|
1456
|
-
shard: shardName,
|
|
1457
|
-
headCommit: fetchResult.headCommit,
|
|
1458
|
-
});
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
// Read shard output log for changed files
|
|
1462
|
-
const commitEvidence = await collectCommitEvidence(
|
|
1463
|
-
worker,
|
|
1464
|
-
integrationBranch,
|
|
1465
|
-
);
|
|
1466
|
-
worker.commitEvidence = commitEvidence;
|
|
1467
|
-
eventLog.append("commit_evidence", {
|
|
1468
|
-
shard: shardName,
|
|
1469
|
-
...commitEvidence,
|
|
1470
|
-
});
|
|
1471
|
-
|
|
1472
|
-
const expectsCommitEvidence =
|
|
1473
|
-
Array.isArray(shard?.files) && shard.files.length > 0;
|
|
1474
|
-
if (expectsCommitEvidence && !commitEvidence.ok) {
|
|
1475
|
-
failures.set(shardName, {
|
|
1476
|
-
mode: FAILURE_MODES.F6_NO_COMMIT,
|
|
1477
|
-
reason: commitEvidence.error
|
|
1478
|
-
? `no_commit_evidence:${commitEvidence.error}`
|
|
1479
|
-
: commitEvidence.dirty
|
|
1480
|
-
? "dirty_worktree_without_commit"
|
|
1481
|
-
: "no_commit_evidence",
|
|
1482
|
-
commitEvidence,
|
|
1483
|
-
});
|
|
1484
|
-
eventLog.append("no_commit_guard_failed", {
|
|
1485
|
-
shard: shardName,
|
|
1486
|
-
...commitEvidence,
|
|
1487
|
-
});
|
|
1488
|
-
const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
|
|
1489
|
-
force: true,
|
|
1490
|
-
failureReason:
|
|
1491
|
-
failures.get(shardName)?.mode || "no_commit_evidence",
|
|
1492
|
-
});
|
|
1493
|
-
integrationFailures.push(shardName);
|
|
1494
|
-
appendIntegrationReport({
|
|
1495
|
-
shard: shardName,
|
|
1496
|
-
strategy: "failed",
|
|
1497
|
-
commits: commitEvidence.commitsAhead,
|
|
1498
|
-
finalSha: commitEvidence.headCommit,
|
|
1499
|
-
notes: failureNotes(shardName, "no_commit_evidence"),
|
|
1500
|
-
recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
|
|
1501
|
-
});
|
|
1502
|
-
continue;
|
|
1503
|
-
}
|
|
1504
|
-
|
|
1505
|
-
// Read shard output log for changed files
|
|
1506
|
-
const changedFiles = detectChangedFiles(shardName, worker);
|
|
1507
|
-
|
|
1508
|
-
// Validate against lease map
|
|
1509
|
-
const validation = validateResult(shardName, changedFiles);
|
|
1510
|
-
if (!validation.ok) {
|
|
1511
|
-
failures.set(shardName, {
|
|
1512
|
-
mode: FAILURE_MODES.F4_LEASE_VIOLATION,
|
|
1513
|
-
violations: validation.violations,
|
|
1514
|
-
});
|
|
1515
|
-
eventLog.append("lease_violation_revert", {
|
|
1516
|
-
shard: shardName,
|
|
1517
|
-
violations: validation.violations,
|
|
1518
|
-
});
|
|
1519
|
-
const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
|
|
1520
|
-
force: true,
|
|
1521
|
-
failureReason:
|
|
1522
|
-
failures.get(shardName)?.mode || "lease_violation_revert",
|
|
1523
|
-
});
|
|
1524
|
-
integrationFailures.push(shardName);
|
|
1525
|
-
appendIntegrationReport({
|
|
1526
|
-
shard: shardName,
|
|
1527
|
-
strategy: "failed",
|
|
1528
|
-
notes: failureNotes(shardName, "lease_violation_revert"),
|
|
1529
|
-
recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
|
|
1530
|
-
});
|
|
1531
|
-
continue;
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
const shardBranch = worker.branchName || `swarm/${runId}/${shardName}`;
|
|
1535
|
-
const rebaseResult = await rebaseShardOntoIntegrationImpl({
|
|
1536
|
-
shardBranch,
|
|
1537
|
-
integrationBranch,
|
|
1538
|
-
rootDir: workdir,
|
|
1539
|
-
});
|
|
1540
|
-
if (!rebaseResult.ok) {
|
|
1541
|
-
eventLog.append("integration_rebase_failed", {
|
|
1542
|
-
shard: shardName,
|
|
1543
|
-
shardBranch,
|
|
1544
|
-
integrationBranch,
|
|
1545
|
-
error: rebaseResult.error,
|
|
1546
|
-
});
|
|
1547
|
-
const cleanup = await maybeCleanupWorktree(shardName, worker, shard, {
|
|
1548
|
-
force: true,
|
|
1549
|
-
failureReason:
|
|
1550
|
-
rebaseResult.error || FAILURE_MODES.F5_MERGE_CONFLICT,
|
|
1551
|
-
});
|
|
1552
|
-
integrationFailures.push(shardName);
|
|
1553
|
-
appendIntegrationReport({
|
|
1554
|
-
shard: shardName,
|
|
1555
|
-
strategy: "failed",
|
|
1556
|
-
commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
|
|
1557
|
-
notes: rebaseResult.error || FAILURE_MODES.F5_MERGE_CONFLICT,
|
|
1558
|
-
recoveryPatch: cleanup?.recoveryPatch?.patchPath || null,
|
|
1559
|
-
});
|
|
1560
|
-
continue;
|
|
1561
|
-
}
|
|
1562
|
-
|
|
1563
|
-
results.set(shardName, {
|
|
1564
|
-
shard: shardName,
|
|
1565
|
-
changedFiles,
|
|
1566
|
-
branchName: shardBranch,
|
|
1567
|
-
worktreePath: worker.worktreePath || null,
|
|
1568
|
-
integrationBranch,
|
|
1569
|
-
headCommit: rebaseResult.headCommit,
|
|
1570
|
-
integrationStrategy: rebaseResult.strategy || "cherry-pick",
|
|
1571
|
-
commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
|
|
1572
|
-
notes: rebaseResult.notes || "",
|
|
1573
|
-
completedAt: Date.now(),
|
|
1574
|
-
});
|
|
1575
|
-
integrated.push(shardName);
|
|
1576
|
-
appendIntegrationReport({
|
|
1577
|
-
shard: shardName,
|
|
1578
|
-
strategy: rebaseResult.strategy || "cherry-pick",
|
|
1579
|
-
commits: rebaseResult.commits ?? commitEvidence.commitsAhead,
|
|
1580
|
-
finalSha: rebaseResult.headCommit,
|
|
1581
|
-
notes: rebaseResult.notes || "",
|
|
1582
|
-
});
|
|
1583
|
-
|
|
1584
|
-
await maybeCleanupWorktree(shardName, worker, shard, {
|
|
1585
|
-
branchName: shardBranch,
|
|
1586
|
-
});
|
|
1587
|
-
}
|
|
1588
|
-
} catch (err) {
|
|
1589
|
-
const unresolved = plan.mergeOrder.filter(
|
|
1590
|
-
(name) =>
|
|
1591
|
-
!integrated.includes(name) &&
|
|
1592
|
-
!preIntegrationFailures.includes(name) &&
|
|
1593
|
-
!integrationFailures.includes(name),
|
|
1594
|
-
);
|
|
1595
|
-
const failed = [
|
|
1596
|
-
...new Set([
|
|
1597
|
-
...preIntegrationFailures,
|
|
1598
|
-
...integrationFailures,
|
|
1599
|
-
...unresolved,
|
|
1600
|
-
]),
|
|
1601
|
-
];
|
|
1602
|
-
const skipped = preIntegrationFailures.filter(
|
|
1603
|
-
(name) => !integrationFailures.includes(name),
|
|
1604
|
-
);
|
|
1605
|
-
const reported = new Set(integrationReport.map((row) => row.shard));
|
|
1606
|
-
for (const shardName of failed) {
|
|
1607
|
-
if (reported.has(shardName)) continue;
|
|
1608
|
-
appendIntegrationReport({
|
|
1609
|
-
shard: shardName,
|
|
1610
|
-
strategy: "failed",
|
|
1611
|
-
notes: `integration_error: ${err.message}`,
|
|
1612
|
-
});
|
|
1613
|
-
}
|
|
1614
|
-
|
|
1615
|
-
eventLog.append("integration_complete", {
|
|
1616
|
-
integrated,
|
|
1617
|
-
failed,
|
|
1618
|
-
integrationFailures,
|
|
1619
|
-
integrationBranch,
|
|
1620
|
-
skipped,
|
|
1621
|
-
report: integrationReport,
|
|
1622
|
-
error: err.message,
|
|
1623
|
-
});
|
|
1624
|
-
|
|
1625
|
-
setState(SWARM_STATES.FAILED, "integration_error");
|
|
1626
|
-
const payload = settleIntegrationPromise({
|
|
1627
|
-
integrated,
|
|
1628
|
-
failed,
|
|
1629
|
-
integrationFailures,
|
|
1630
|
-
integrationBranch,
|
|
1631
|
-
skipped,
|
|
1632
|
-
report: integrationReport,
|
|
1633
|
-
results: [...results.values()],
|
|
1634
|
-
partial: true,
|
|
1635
|
-
error: err.message,
|
|
1636
|
-
});
|
|
1637
|
-
emitter.emit("integrationComplete", payload);
|
|
1638
|
-
return payload;
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
const skipped = preIntegrationFailures.filter(
|
|
1642
|
-
(name) => !integrationFailures.includes(name),
|
|
1643
|
-
);
|
|
1644
|
-
const failed = [...new Set([...skipped, ...integrationFailures])];
|
|
1645
|
-
|
|
1646
|
-
eventLog.append("integration_complete", {
|
|
1647
|
-
integrated,
|
|
1648
|
-
failed,
|
|
1649
|
-
integrationFailures,
|
|
1650
|
-
integrationBranch,
|
|
1651
|
-
skipped,
|
|
1652
|
-
report: integrationReport,
|
|
1653
|
-
});
|
|
1654
|
-
|
|
1655
|
-
if (failed.length > 0 && integrated.length === 0) {
|
|
1656
|
-
setState(SWARM_STATES.FAILED, "all_shards_failed_integration");
|
|
1657
|
-
} else {
|
|
1658
|
-
setState(
|
|
1659
|
-
SWARM_STATES.COMPLETED,
|
|
1660
|
-
`${integrated.length}/${plan.shards.length} integrated`,
|
|
1661
|
-
);
|
|
1662
|
-
}
|
|
1663
|
-
|
|
1664
|
-
const payload = settleIntegrationPromise({
|
|
1665
|
-
integrated,
|
|
1666
|
-
failed,
|
|
1667
|
-
integrationFailures,
|
|
1668
|
-
integrationBranch,
|
|
1669
|
-
skipped,
|
|
1670
|
-
report: integrationReport,
|
|
1671
|
-
results: [...results.values()],
|
|
1672
|
-
partial: failed.length > 0,
|
|
1673
|
-
});
|
|
1674
|
-
emitter.emit("integrationComplete", payload);
|
|
1675
|
-
return payload;
|
|
1676
|
-
}
|
|
1677
|
-
|
|
1678
|
-
async function maybeCleanupWorktree(
|
|
1679
|
-
shardName,
|
|
1680
|
-
worker,
|
|
1681
|
-
shard,
|
|
1682
|
-
{
|
|
1683
|
-
branchName = worker?.branchName,
|
|
1684
|
-
force = false,
|
|
1685
|
-
failureReason = null,
|
|
1686
|
-
} = {},
|
|
1687
|
-
) {
|
|
1688
|
-
const outcome = { recoveryPatch: null, processCleanup: null };
|
|
1689
|
-
if (!worker?.worktreePath || shard?.host) return outcome;
|
|
1690
|
-
if (failureReason && keepFailedWorktrees) return outcome;
|
|
1691
|
-
if (cleanedWorktreePaths.has(worker.worktreePath)) return outcome;
|
|
1692
|
-
|
|
1693
|
-
await closeNativeBridgeRegistration(
|
|
1694
|
-
worker,
|
|
1695
|
-
shardName,
|
|
1696
|
-
"before_worktree_cleanup",
|
|
1697
|
-
);
|
|
1698
|
-
|
|
1699
|
-
// Best-effort: preserve any uncommitted worker changes before removing the
|
|
1700
|
-
// worktree, on BOTH the failure and success teardown paths (#394). A worker
|
|
1701
|
-
// can exit 0 yet leave dirty, uncommitted work (the hub-observability case:
|
|
1702
|
-
// exit 0, commits_made []), and the success cleanup carries no
|
|
1703
|
-
// failureReason — without this it would silently delete that work.
|
|
1704
|
-
// preserveWorktreePatchImpl self-skips when the worktree is clean, so a
|
|
1705
|
-
// properly-committed shard saves nothing here.
|
|
1706
|
-
try {
|
|
1707
|
-
const preservation = await preserveWorktreePatchImpl({
|
|
1708
|
-
worktreePath: worker.worktreePath,
|
|
1709
|
-
shardId: shardName,
|
|
1710
|
-
recoveryDir: join(workdir, ".codex-swarm", "recovery"),
|
|
1711
|
-
});
|
|
1712
|
-
if (preservation?.ok && !preservation.skipped) {
|
|
1713
|
-
outcome.recoveryPatch = {
|
|
1714
|
-
patchPath: preservation.patchPath,
|
|
1715
|
-
manifestPath: preservation.manifestPath || null,
|
|
1716
|
-
};
|
|
1717
|
-
eventLog.append("recovery_patch_saved", {
|
|
1718
|
-
shard: shardName,
|
|
1719
|
-
worktreePath: worker.worktreePath,
|
|
1720
|
-
patchPath: preservation.patchPath,
|
|
1721
|
-
reason: failureReason || "dirty_on_cleanup",
|
|
1722
|
-
});
|
|
1723
|
-
}
|
|
1724
|
-
} catch {
|
|
1725
|
-
/* silent: preservation failures must not block cleanup */
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
|
-
try {
|
|
1729
|
-
const snapshot = worker.conductor?.getSnapshot?.();
|
|
1730
|
-
const processCleanup = await cleanupShardProcessesImpl({
|
|
1731
|
-
worktreePath: worker.worktreePath,
|
|
1732
|
-
sessionIds: [worker.sessionId || worker.sessionConfig?.id].filter(
|
|
1733
|
-
Boolean,
|
|
1734
|
-
),
|
|
1735
|
-
topPids: snapshot?.topPids ?? [],
|
|
1736
|
-
runId: plan?.runId || runId,
|
|
1737
|
-
shardName,
|
|
1738
|
-
});
|
|
1739
|
-
outcome.processCleanup = processCleanup;
|
|
1740
|
-
worker.processCleanup = processCleanup;
|
|
1741
|
-
recordProcessCleanup(processCleanup);
|
|
1742
|
-
eventLog.append("process_cleanup", {
|
|
1743
|
-
shard: shardName,
|
|
1744
|
-
worktreePath: worker.worktreePath,
|
|
1745
|
-
killed: processCleanup?.killed ?? 0,
|
|
1746
|
-
byCategory: processCleanup?.byCategory || {},
|
|
1747
|
-
});
|
|
1748
|
-
} catch (err) {
|
|
1749
|
-
eventLog.append("process_cleanup_failed", {
|
|
1750
|
-
shard: shardName,
|
|
1751
|
-
worktreePath: worker.worktreePath,
|
|
1752
|
-
error: err.message,
|
|
1753
|
-
});
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
|
-
try {
|
|
1757
|
-
await cleanupWorktreeImpl({
|
|
1758
|
-
worktreePath: worker.worktreePath,
|
|
1759
|
-
branchName,
|
|
1760
|
-
rootDir: workdir,
|
|
1761
|
-
force,
|
|
1762
|
-
});
|
|
1763
|
-
cleanedWorktreePaths.add(worker.worktreePath);
|
|
1764
|
-
if (failureReason) {
|
|
1765
|
-
eventLog.append("worktree_auto_cleanup", {
|
|
1766
|
-
shard: shardName,
|
|
1767
|
-
worktreePath: worker.worktreePath,
|
|
1768
|
-
reason: failureReason,
|
|
1769
|
-
});
|
|
1770
|
-
}
|
|
1771
|
-
} catch (err) {
|
|
1772
|
-
eventLog.append("worktree_cleanup_failed", {
|
|
1773
|
-
shard: shardName,
|
|
1774
|
-
worktreePath: worker.worktreePath,
|
|
1775
|
-
reason: failureReason || null,
|
|
1776
|
-
error: err.message,
|
|
1777
|
-
});
|
|
1778
|
-
}
|
|
1779
|
-
|
|
1780
|
-
return outcome;
|
|
1781
|
-
}
|
|
1782
|
-
|
|
1783
|
-
/**
|
|
1784
|
-
* Detect which files a shard modified by reading its output logs.
|
|
1785
|
-
* Falls back to an empty list if detection fails.
|
|
1786
|
-
* @param {string} shardName
|
|
1787
|
-
* @param {object} worker
|
|
1788
|
-
* @returns {string[]}
|
|
1789
|
-
*/
|
|
1790
|
-
function detectChangedFiles(shardName, worker) {
|
|
1791
|
-
// Best-effort: parse output log for file paths
|
|
1792
|
-
const _outPath = join(logsDir, shardName);
|
|
1793
|
-
try {
|
|
1794
|
-
const snap = worker.conductor.getSnapshot();
|
|
1795
|
-
for (const session of snap) {
|
|
1796
|
-
if (session.outPath && existsSync(session.outPath)) {
|
|
1797
|
-
const output = readFileSync(session.outPath, "utf8");
|
|
1798
|
-
return extractFilePathsFromOutput(
|
|
1799
|
-
output,
|
|
1800
|
-
plan.leaseMap.get(shardName) || [],
|
|
1801
|
-
);
|
|
1802
|
-
}
|
|
1803
|
-
}
|
|
1804
|
-
} catch {
|
|
1805
|
-
/* best-effort */
|
|
1806
|
-
}
|
|
1807
|
-
|
|
1808
|
-
// Fallback: trust the lease map (shard was allowed these files)
|
|
1809
|
-
return plan.leaseMap.get(shardName) || [];
|
|
1810
|
-
}
|
|
1811
|
-
|
|
1812
|
-
/**
|
|
1813
|
-
* Extract modified file paths from worker output text.
|
|
1814
|
-
* Looks for common patterns: "wrote file.mjs", "modified file.mjs", diff headers.
|
|
1815
|
-
* @param {string} output
|
|
1816
|
-
* @param {string[]} allowedFiles — lease map files to match against
|
|
1817
|
-
* @returns {string[]}
|
|
1818
|
-
*/
|
|
1819
|
-
function extractFilePathsFromOutput(output, allowedFiles) {
|
|
1820
|
-
if (!output) return allowedFiles;
|
|
1821
|
-
|
|
1822
|
-
const found = new Set();
|
|
1823
|
-
const lines = output.split(/\r?\n/);
|
|
1824
|
-
|
|
1825
|
-
for (const line of lines) {
|
|
1826
|
-
// Match common patterns
|
|
1827
|
-
const patterns = [
|
|
1828
|
-
/(?:wrote|created|modified|updated|edited)\s+['"]?([^\s'"]+\.\w+)/i,
|
|
1829
|
-
/^[+-]{3}\s+[ab]\/(.+)/, // diff headers
|
|
1830
|
-
/^diff --git a\/(.+)\s+b\//, // git diff headers
|
|
1831
|
-
];
|
|
1832
|
-
|
|
1833
|
-
for (const re of patterns) {
|
|
1834
|
-
const match = line.match(re);
|
|
1835
|
-
if (match) found.add(match[1]);
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
// Intersect with allowed files if we found anything
|
|
1840
|
-
if (found.size > 0) {
|
|
1841
|
-
return [...found].filter((f) =>
|
|
1842
|
-
allowedFiles.some((a) => f.endsWith(a) || a.endsWith(f) || f === a),
|
|
1843
|
-
);
|
|
1844
|
-
}
|
|
1845
|
-
|
|
1846
|
-
return allowedFiles;
|
|
1847
|
-
}
|
|
1848
|
-
|
|
1849
|
-
// ── Status monitor ──────────────────────────────────────────
|
|
1850
|
-
|
|
1851
|
-
/**
|
|
1852
|
-
* Get current swarm status snapshot.
|
|
1853
|
-
* @returns {SwarmStatus}
|
|
1854
|
-
*/
|
|
1855
|
-
function getStatus() {
|
|
1856
|
-
const workerStatuses = [];
|
|
1857
|
-
|
|
1858
|
-
for (const [name, w] of workers) {
|
|
1859
|
-
const snap = w.conductor.getSnapshot();
|
|
1860
|
-
const authoritative = computeAuthoritativeStatus(name, w, snap);
|
|
1861
|
-
workerStatuses.push({
|
|
1862
|
-
shard: name,
|
|
1863
|
-
agent: w.shardConfig.agent,
|
|
1864
|
-
sessions: snap,
|
|
1865
|
-
failed: failures.has(name),
|
|
1866
|
-
failureInfo: failures.get(name) || null,
|
|
1867
|
-
integrated: results.has(name),
|
|
1868
|
-
authoritativeStatus: authoritative.status,
|
|
1869
|
-
authoritativeReason: authoritative.reason,
|
|
1870
|
-
commitEvidence: w.commitEvidence || null,
|
|
1871
|
-
});
|
|
1872
|
-
}
|
|
1873
|
-
|
|
1874
|
-
return Object.freeze({
|
|
1875
|
-
state,
|
|
1876
|
-
totalShards: plan?.shards.length || 0,
|
|
1877
|
-
completedShards: completedShards.size,
|
|
1878
|
-
failedShards: failures.size,
|
|
1879
|
-
workers: workerStatuses,
|
|
1880
|
-
mergeOrder: plan?.mergeOrder || [],
|
|
1881
|
-
criticalShards: plan?.criticalShards || [],
|
|
1882
|
-
locks: lockManager?.snapshot() || [],
|
|
1883
|
-
integrationPromise: Object.freeze({
|
|
1884
|
-
state: integrationPromiseState.state,
|
|
1885
|
-
startedAt: integrationPromiseState.startedAt,
|
|
1886
|
-
settledAt: integrationPromiseState.settledAt,
|
|
1887
|
-
partial: integrationPromiseState.partial,
|
|
1888
|
-
integrated: [...integrationPromiseState.integrated],
|
|
1889
|
-
failed: [...integrationPromiseState.failed],
|
|
1890
|
-
integrationFailures: [...integrationPromiseState.integrationFailures],
|
|
1891
|
-
skipped: [...integrationPromiseState.skipped],
|
|
1892
|
-
integrationBranch: integrationPromiseState.integrationBranch,
|
|
1893
|
-
report: integrationPromiseState.report.map((row) => ({ ...row })),
|
|
1894
|
-
processCleanup: Object.freeze({
|
|
1895
|
-
totalKilled: integrationPromiseState.processCleanup.totalKilled,
|
|
1896
|
-
byCategory: Object.freeze({
|
|
1897
|
-
...integrationPromiseState.processCleanup.byCategory,
|
|
1898
|
-
}),
|
|
1899
|
-
}),
|
|
1900
|
-
error: integrationPromiseState.error,
|
|
1901
|
-
}),
|
|
1902
|
-
});
|
|
1903
|
-
}
|
|
1904
|
-
|
|
1905
|
-
// ── Public API ──────────────────────────────────────────────
|
|
1906
|
-
|
|
1907
|
-
/**
|
|
1908
|
-
* Launch the swarm from a pre-built plan.
|
|
1909
|
-
* @param {SwarmPlan} swarmPlan — from planSwarm()
|
|
1910
|
-
* @returns {Promise<SwarmStatus>}
|
|
1911
|
-
*/
|
|
1912
|
-
/** Hub keepalive — Hub crash recovery (idle timeout은 기본 비활성이므로 crash 복구 용도) */
|
|
1913
|
-
let hubKeepaliveTimer = null;
|
|
1914
|
-
let lockRenewTimer = null;
|
|
1915
|
-
function startHubKeepalive() {
|
|
1916
|
-
// 5분마다 Hub /status 핑 — crash 감지 시 ensureHubAlive로 재시작
|
|
1917
|
-
hubKeepaliveTimer = setInterval(
|
|
1918
|
-
async () => {
|
|
1919
|
-
try {
|
|
1920
|
-
const hub = getHubInfoImpl ? await getHubInfoImpl() : null;
|
|
1921
|
-
const statusUrl = resolveHubStatusUrl(hub);
|
|
1922
|
-
if (!statusUrl) {
|
|
1923
|
-
if (ensureHubAliveImpl) {
|
|
1924
|
-
eventLog.append("hub_keepalive_restart", {
|
|
1925
|
-
reason: "hub_url_unresolved",
|
|
1926
|
-
});
|
|
1927
|
-
await ensureHubAliveImpl();
|
|
1928
|
-
}
|
|
1929
|
-
return;
|
|
1930
|
-
}
|
|
1931
|
-
|
|
1932
|
-
const resp = await fetch(statusUrl, {
|
|
1933
|
-
signal: AbortSignal.timeout(5000),
|
|
1934
|
-
});
|
|
1935
|
-
if (!resp.ok && ensureHubAliveImpl) {
|
|
1936
|
-
eventLog.append("hub_keepalive_restart", {});
|
|
1937
|
-
await ensureHubAliveImpl();
|
|
1938
|
-
}
|
|
1939
|
-
} catch {
|
|
1940
|
-
// Hub 다운 — 재시작 시도
|
|
1941
|
-
if (ensureHubAliveImpl) {
|
|
1942
|
-
eventLog.append("hub_keepalive_restart", {
|
|
1943
|
-
reason: "fetch_failed",
|
|
1944
|
-
});
|
|
1945
|
-
try {
|
|
1946
|
-
await ensureHubAliveImpl();
|
|
1947
|
-
} catch {
|
|
1948
|
-
eventLog.append("hub_restart_failed", {});
|
|
1949
|
-
}
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
1952
|
-
},
|
|
1953
|
-
5 * 60 * 1000,
|
|
1954
|
-
);
|
|
1955
|
-
}
|
|
1956
|
-
|
|
1957
|
-
function stopHubKeepalive() {
|
|
1958
|
-
if (hubKeepaliveTimer) {
|
|
1959
|
-
clearInterval(hubKeepaliveTimer);
|
|
1960
|
-
hubKeepaliveTimer = null;
|
|
1961
|
-
}
|
|
1962
|
-
}
|
|
1963
|
-
|
|
1964
|
-
function startLockRenewal() {
|
|
1965
|
-
lockRenewTimer = setInterval(() => {
|
|
1966
|
-
if (!lockManager) return;
|
|
1967
|
-
for (const shardName of workers.keys()) {
|
|
1968
|
-
if (!completedShards.has(shardName) && !failures.has(shardName))
|
|
1969
|
-
lockManager.renew(shardName);
|
|
1970
|
-
}
|
|
1971
|
-
}, DEFAULT_SWARM_LOCK_TTL_MS / 2);
|
|
1972
|
-
lockRenewTimer.unref?.();
|
|
1973
|
-
}
|
|
1974
|
-
|
|
1975
|
-
function stopLockRenewal() {
|
|
1976
|
-
if (lockRenewTimer) clearInterval(lockRenewTimer);
|
|
1977
|
-
lockRenewTimer = null;
|
|
1978
|
-
}
|
|
1979
|
-
|
|
1980
|
-
async function launch(swarmPlan) {
|
|
1981
|
-
if (state !== SWARM_STATES.PLANNING) {
|
|
1982
|
-
throw new Error(`Cannot launch in state "${state}"`);
|
|
1983
|
-
}
|
|
1984
|
-
|
|
1985
|
-
plan = swarmPlan;
|
|
1986
|
-
markIntegrationPromisePending();
|
|
1987
|
-
|
|
1988
|
-
// Phase 1 dynamic routing wire-up (opt-in env TRIFLUX_DYNAMIC_ROUTING).
|
|
1989
|
-
// launch() 는 plan-time, 이미 async 라 D-1 facade 의 await routeRequest 를
|
|
1990
|
-
// 그대로 사용한다 (conductor.spawnSession 의 sync invariant 충돌 없음).
|
|
1991
|
-
// team_size = plan.shards.length 매핑 → S5 leaseMany 시나리오 자연 매핑.
|
|
1992
|
-
// decision.shards[i].cli 가 plan.shards[i].agent 와 다르면 1:1 override.
|
|
1993
|
-
if (
|
|
1994
|
-
isDynamicRoutingEnabled(process.env) &&
|
|
1995
|
-
Array.isArray(plan?.shards) &&
|
|
1996
|
-
plan.shards.length > 0
|
|
1997
|
-
) {
|
|
1998
|
-
try {
|
|
1999
|
-
const router = createDynamicRouter();
|
|
2000
|
-
const decision = await router.routeRequest({
|
|
2001
|
-
task_id: runId,
|
|
2002
|
-
team_size: plan.shards.length,
|
|
2003
|
-
agent_hint: plan.shards[0]?.agent || "codex",
|
|
2004
|
-
});
|
|
2005
|
-
eventLog.append("dynamic_route_plan_decision", {
|
|
2006
|
-
decisionId: decision.decisionId ?? null,
|
|
2007
|
-
scenario: decision.scenario ?? "unknown",
|
|
2008
|
-
lane: decision.lane ?? null,
|
|
2009
|
-
mode: decision.mode ?? null,
|
|
2010
|
-
shardCount: decision.shards?.length ?? 0,
|
|
2011
|
-
planShardCount: plan.shards.length,
|
|
2012
|
-
snapshotAgeMs: decision.snapshotAgeMs ?? null,
|
|
2013
|
-
});
|
|
2014
|
-
const overrides = [];
|
|
2015
|
-
const decShards = Array.isArray(decision.shards) ? decision.shards : [];
|
|
2016
|
-
for (let i = 0; i < plan.shards.length; i++) {
|
|
2017
|
-
const shard = plan.shards[i];
|
|
2018
|
-
const decShard = decShards[i];
|
|
2019
|
-
if (decShard?.cli && decShard.cli !== shard.agent) {
|
|
2020
|
-
overrides.push({
|
|
2021
|
-
shard: shard.name,
|
|
2022
|
-
original: shard.agent,
|
|
2023
|
-
override: decShard.cli,
|
|
2024
|
-
accountId: decShard.accountId ?? null,
|
|
2025
|
-
});
|
|
2026
|
-
shard.agent = decShard.cli;
|
|
2027
|
-
}
|
|
2028
|
-
}
|
|
2029
|
-
if (overrides.length > 0) {
|
|
2030
|
-
eventLog.append("dynamic_route_plan_overrides", {
|
|
2031
|
-
decisionId: decision.decisionId ?? null,
|
|
2032
|
-
count: overrides.length,
|
|
2033
|
-
overrides,
|
|
2034
|
-
});
|
|
2035
|
-
}
|
|
2036
|
-
} catch (err) {
|
|
2037
|
-
eventLog.append("dynamic_route_plan_error", {
|
|
2038
|
-
error: err?.message || String(err),
|
|
2039
|
-
});
|
|
2040
|
-
}
|
|
2041
|
-
}
|
|
2042
|
-
|
|
2043
|
-
// Hub alive 확인 — 죽어있으면 재시작
|
|
2044
|
-
if (ensureHubAliveImpl) {
|
|
2045
|
-
ensureHubAliveImpl()
|
|
2046
|
-
.then((hub) => {
|
|
2047
|
-
eventLog.append("hub_ensured", { port: hub?.port });
|
|
2048
|
-
})
|
|
2049
|
-
.catch((err) => {
|
|
2050
|
-
eventLog.append("hub_ensure_failed", { error: err.message });
|
|
2051
|
-
emitter.emit("warning", {
|
|
2052
|
-
type: "hub_unavailable",
|
|
2053
|
-
error: err.message,
|
|
2054
|
-
});
|
|
2055
|
-
});
|
|
2056
|
-
}
|
|
2057
|
-
|
|
2058
|
-
// Warn about file conflicts but don't block
|
|
2059
|
-
if (plan.conflicts.length > 0) {
|
|
2060
|
-
eventLog.append("file_conflicts_warning", { conflicts: plan.conflicts });
|
|
2061
|
-
emitter.emit("warning", {
|
|
2062
|
-
type: "file_conflicts",
|
|
2063
|
-
conflicts: plan.conflicts,
|
|
2064
|
-
});
|
|
2065
|
-
}
|
|
2066
|
-
|
|
2067
|
-
// Initialize lock manager
|
|
2068
|
-
lockManager = createSwarmLocks({
|
|
2069
|
-
repoRoot: workdir,
|
|
2070
|
-
persistPath: join(workdir, ".triflux", "swarm-locks.json"),
|
|
2071
|
-
});
|
|
2072
|
-
|
|
2073
|
-
startLockRenewal();
|
|
2074
|
-
// Hub keepalive 시작
|
|
2075
|
-
startHubKeepalive();
|
|
2076
|
-
|
|
2077
|
-
setState(SWARM_STATES.LAUNCHING, `${plan.shards.length} shards`);
|
|
2078
|
-
|
|
2079
|
-
// Launch shards respecting dependency order
|
|
2080
|
-
const launched = new Set();
|
|
2081
|
-
const pending = new Set(plan.mergeOrder);
|
|
2082
|
-
|
|
2083
|
-
async function launchReady() {
|
|
2084
|
-
for (const name of pending) {
|
|
2085
|
-
const shard = plan.shards.find((s) => s.name === name);
|
|
2086
|
-
if (!shard) continue;
|
|
2087
|
-
|
|
2088
|
-
// Check all dependencies are completed (not just launched)
|
|
2089
|
-
const depsReady = shard.depends.every((d) => completedShards.has(d));
|
|
2090
|
-
if (!depsReady) continue;
|
|
2091
|
-
|
|
2092
|
-
pending.delete(name);
|
|
2093
|
-
launched.add(name);
|
|
2094
|
-
await launchShard(shard);
|
|
2095
|
-
|
|
2096
|
-
// Launch redundant worker only when explicitly requested.
|
|
2097
|
-
const redundantAgent = resolveRedundantAgent(shard);
|
|
2098
|
-
if (redundantAgent) {
|
|
2099
|
-
const redundantShard = {
|
|
2100
|
-
...shard,
|
|
2101
|
-
agent: redundantAgent,
|
|
2102
|
-
};
|
|
2103
|
-
await launchShard(redundantShard, true);
|
|
2104
|
-
}
|
|
2105
|
-
}
|
|
2106
|
-
}
|
|
2107
|
-
|
|
2108
|
-
await launchReady();
|
|
2109
|
-
|
|
2110
|
-
// Re-check pending on each shard completion (dependency chains)
|
|
2111
|
-
emitter.on("shardCompleted", () => {
|
|
2112
|
-
if (pending.size > 0) void launchReady();
|
|
2113
|
-
});
|
|
2114
|
-
|
|
2115
|
-
setState(
|
|
2116
|
-
SWARM_STATES.RUNNING,
|
|
2117
|
-
`${launched.size} launched, ${pending.size} pending deps`,
|
|
2118
|
-
);
|
|
2119
|
-
|
|
2120
|
-
return { ...getStatus(), done: integrationPromise };
|
|
2121
|
-
}
|
|
2122
|
-
|
|
2123
|
-
/**
|
|
2124
|
-
* Graceful shutdown — kill all workers and release locks.
|
|
2125
|
-
* @param {string} [reason]
|
|
2126
|
-
*/
|
|
2127
|
-
async function shutdown(reason = "shutdown") {
|
|
2128
|
-
stopHubKeepalive();
|
|
2129
|
-
stopLockRenewal();
|
|
2130
|
-
eventLog.append("swarm_shutdown", { reason, state });
|
|
2131
|
-
|
|
2132
|
-
const shutdowns = [];
|
|
2133
|
-
for (const [, w] of workers) {
|
|
2134
|
-
shutdowns.push(w.conductor.shutdown(reason));
|
|
2135
|
-
shutdowns.push(
|
|
2136
|
-
closeNativeBridgeRegistration(w, w.shardConfig?.name, reason),
|
|
2137
|
-
);
|
|
2138
|
-
}
|
|
2139
|
-
for (const [, w] of redundantWorkers) {
|
|
2140
|
-
shutdowns.push(w.conductor.shutdown(reason));
|
|
2141
|
-
shutdowns.push(
|
|
2142
|
-
closeNativeBridgeRegistration(w, w.shardConfig?.name, reason),
|
|
2143
|
-
);
|
|
2144
|
-
}
|
|
2145
|
-
|
|
2146
|
-
await Promise.allSettled(shutdowns);
|
|
2147
|
-
|
|
2148
|
-
if (!keepFailedWorktrees) {
|
|
2149
|
-
const cleanupCandidates = new Map();
|
|
2150
|
-
for (const [shardName, worker] of workers) {
|
|
2151
|
-
cleanupCandidates.set(shardName, worker);
|
|
2152
|
-
}
|
|
2153
|
-
for (const [shardName, worker] of redundantWorkers) {
|
|
2154
|
-
cleanupCandidates.set(`${shardName}:redundant`, worker);
|
|
2155
|
-
}
|
|
2156
|
-
|
|
2157
|
-
for (const [candidateName, worker] of cleanupCandidates) {
|
|
2158
|
-
const shardName = worker?.shardConfig?.name || candidateName;
|
|
2159
|
-
const failureInfo = failures.get(shardName);
|
|
2160
|
-
const shard =
|
|
2161
|
-
worker?.shardConfig ||
|
|
2162
|
-
plan?.shards.find((item) => item.name === shardName);
|
|
2163
|
-
if (!worker?.worktreePath) continue;
|
|
2164
|
-
await maybeCleanupWorktree(shardName, worker, shard, {
|
|
2165
|
-
force: true,
|
|
2166
|
-
failureReason: failureInfo?.mode || failureInfo?.reason || reason,
|
|
2167
|
-
});
|
|
2168
|
-
}
|
|
2169
|
-
}
|
|
2170
|
-
|
|
2171
|
-
sharedRegistry.clear();
|
|
2172
|
-
lockManager?.releaseAll();
|
|
2173
|
-
await eventLog.flush();
|
|
2174
|
-
await eventLog.close();
|
|
2175
|
-
|
|
2176
|
-
if (state !== SWARM_STATES.COMPLETED && state !== SWARM_STATES.FAILED) {
|
|
2177
|
-
setState(SWARM_STATES.FAILED, reason);
|
|
2178
|
-
}
|
|
2179
|
-
|
|
2180
|
-
emitter.emit("shutdown", { reason });
|
|
2181
|
-
}
|
|
2182
|
-
|
|
2183
|
-
return Object.freeze({
|
|
2184
|
-
launch,
|
|
2185
|
-
shutdown,
|
|
2186
|
-
getStatus,
|
|
2187
|
-
integrationComplete() {
|
|
2188
|
-
return integrationPromise;
|
|
2189
|
-
},
|
|
2190
|
-
getMeshRegistry() {
|
|
2191
|
-
return sharedRegistry;
|
|
2192
|
-
},
|
|
2193
|
-
validateResult,
|
|
2194
|
-
on: emitter.on.bind(emitter),
|
|
2195
|
-
off: emitter.off.bind(emitter),
|
|
2196
|
-
get state() {
|
|
2197
|
-
return state;
|
|
2198
|
-
},
|
|
2199
|
-
get plan() {
|
|
2200
|
-
return plan;
|
|
2201
|
-
},
|
|
2202
|
-
get eventLogPath() {
|
|
2203
|
-
return eventLog.filePath;
|
|
2204
|
-
},
|
|
2205
|
-
});
|
|
2206
|
-
}
|