@bridge_gpt/mcp-server 0.2.41 → 0.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +330 -191
- package/build/agent-capabilities/cli.js +2 -1
- package/build/agent-launchers/claude-executor-adapter.js +17 -4
- package/build/agents.generated.js +2 -2
- package/build/claude-review-workflow.js +510 -45
- package/build/claude-user-config-doctor.js +42 -11
- package/build/cli-release.js +2 -1
- package/build/commands.generated.js +6 -5
- package/build/conduct-epic/bridge-client.js +354 -113
- package/build/conduct-epic/checkpoint-store.js +17 -0
- package/build/conduct-epic/cli.js +947 -99
- package/build/conduct-epic/cut-protocol.js +327 -0
- package/build/conduct-epic/spawn.js +14 -2
- package/build/conductor/bridge-api-client.js +148 -1
- package/build/conductor/cli.js +109 -1
- package/build/conductor/doctor.js +101 -16
- package/build/conductor/epic-reconcile.js +72 -19
- package/build/conductor/epic-runtime.js +15 -3
- package/build/conductor/errors.js +47 -0
- package/build/conductor/git-hooks.js +205 -11
- package/build/conductor/install-doctor.js +230 -1
- package/build/conductor/local-merge.js +130 -28
- package/build/conductor/recovery-cli.js +313 -0
- package/build/conductor/recovery-operations.js +219 -0
- package/build/conductor/tools.js +32 -3
- package/build/conductor/worker-ledger-cli.js +27 -1
- package/build/conductor-bin.js +20 -16
- package/build/credentials-cli.js +3 -2
- package/build/docs.generated.js +2 -1
- package/build/doctor.js +120 -44
- package/build/drive-epic.js +375 -0
- package/build/executor/cli.js +48 -1
- package/build/executor/env.js +21 -0
- package/build/executor/http-client.js +71 -3
- package/build/executor/index-scope.js +39 -0
- package/build/executor/job-errors.js +9 -0
- package/build/executor/job-log-registry.js +69 -0
- package/build/executor/job-runner.js +198 -29
- package/build/executor/live-worker-registry.js +83 -0
- package/build/executor/observation.js +259 -6
- package/build/executor/platform.js +147 -3
- package/build/executor/process.js +58 -14
- package/build/executor/runner.js +454 -48
- package/build/executor/test-clock.js +3 -2
- package/build/executor/worker-finalization.js +233 -56
- package/build/executor/worktree.js +8 -1
- package/build/index-scope-contract.js +96 -0
- package/build/index.js +2277 -270
- package/build/init.js +83 -22
- package/build/install-bridge-conductor.js +323 -14
- package/build/install-bridge.js +225 -47
- package/build/install-doctor.js +23 -9
- package/build/install-reexec.js +2 -1
- package/build/launcher-config-inspection.js +83 -22
- package/build/mcp-host-config.js +331 -67
- package/build/mcp-host-targets.js +45 -21
- package/build/mcp-identity.js +92 -0
- package/build/mcp-install-state.js +94 -1
- package/build/mcp-invoke.js +2 -1
- package/build/mcp-provisioning.js +45 -12
- package/build/mcp-registration-doctor.js +35 -13
- package/build/mcp-server-invocation.js +4 -2
- package/build/merge-pull-request.js +208 -9
- package/build/pipelines.generated.js +305 -15
- package/build/plane/cli.js +73 -7
- package/build/plane/defaults.js +18 -5
- package/build/plane/manifest.js +90 -0
- package/build/plane/preflight.js +100 -10
- package/build/plane/shutdown.js +71 -3
- package/build/plane/test-fakes.js +9 -1
- package/build/readme.generated.js +1 -1
- package/build/regression-check.js +3 -2
- package/build/review-tickets.js +8 -7
- package/build/run-unit-tests-launcher.js +149 -6
- package/build/schedule-run.js +3 -2
- package/build/setup-epic.js +531 -82
- package/build/sfcc/tool-wrapper.js +15 -0
- package/build/start-tickets-prereqs.js +11 -6
- package/build/start-tickets.js +91 -85
- package/build/update-check.js +3 -2
- package/build/upgrade-advice.js +2 -1
- package/build/upgrade-cli.js +50 -18
- package/build/version.generated.js +2 -1
- package/build/worktree-core.js +31 -17
- package/docs/CONDUCTOR.md +22 -0
- package/docs/install/mcp-tool-integrations.md +19 -3
- package/package.json +2 -2
- package/pipelines/greenfield-setup.json +286 -0
package/build/executor/runner.js
CHANGED
|
@@ -8,7 +8,90 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { collectExecutorPreflight, buildClaimManifest, createDenyProbeCache, } from "./preflight.js";
|
|
10
10
|
import { runClaimedJob } from "./job-runner.js";
|
|
11
|
+
import { appendToActiveJobLogs } from "./job-log-registry.js";
|
|
12
|
+
import { createLiveWorkerRegistry } from "./live-worker-registry.js";
|
|
11
13
|
import { sweepExecutorWorktrees } from "./worktree-gc.js";
|
|
14
|
+
/**
|
|
15
|
+
* Signals that request a graceful executor shutdown (BAPI-828).
|
|
16
|
+
*
|
|
17
|
+
* Both, not just `SIGTERM`: an operator stopping a foreground executor types
|
|
18
|
+
* Ctrl-C, so a `SIGINT` that orphaned the worker would be the common case rather
|
|
19
|
+
* than the rare one. `SIGKILL` is deliberately absent — it cannot be trapped, and
|
|
20
|
+
* the worktree lock's stale-owner recovery is what covers it.
|
|
21
|
+
*/
|
|
22
|
+
export const EXECUTOR_SHUTDOWN_SIGNALS = ["SIGTERM", "SIGINT"];
|
|
23
|
+
/**
|
|
24
|
+
* Extra wall-clock slack, beyond three poll intervals, before a COMPLETED poll
|
|
25
|
+
* sleep is called a host suspend (BAPI-828).
|
|
26
|
+
*
|
|
27
|
+
* The threshold is `3 * pollIntervalMs + SUSPEND_GAP_SLACK_MS`, so it scales with
|
|
28
|
+
* however the operator configured polling instead of pinning a constant that a
|
|
29
|
+
* long interval would trivially exceed and a short one would never reach. Three
|
|
30
|
+
* intervals plus a full minute is far outside anything ordinary scheduling jitter
|
|
31
|
+
* or a slow GC sweep produces, which keeps the diagnostic rare enough to mean
|
|
32
|
+
* something when it does appear.
|
|
33
|
+
*/
|
|
34
|
+
export const SUSPEND_GAP_SLACK_MS = 60_000;
|
|
35
|
+
/**
|
|
36
|
+
* Render the fixed suspend-gap diagnostic.
|
|
37
|
+
*
|
|
38
|
+
* Exported so the stderr line and the worker-log annotation are the same string
|
|
39
|
+
* by construction — an operator correlating one against the other is the whole
|
|
40
|
+
* reason both exist.
|
|
41
|
+
*/
|
|
42
|
+
export function formatSuspendGapDiagnostic(elapsedMs) {
|
|
43
|
+
return `executor: host appears to have been suspended for ~${Math.round(elapsedMs / 1000)} s (poll gap)`;
|
|
44
|
+
}
|
|
45
|
+
function normalizeDispatcherAvailability(value) {
|
|
46
|
+
if (value === "fresh")
|
|
47
|
+
return "fresh";
|
|
48
|
+
if (value === "stale" || value === "never_seen")
|
|
49
|
+
return "absent";
|
|
50
|
+
return "unknown";
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The unavailable diagnostic. Exported so the message has ONE definition that a
|
|
54
|
+
* test can assert against rather than a copy that drifts.
|
|
55
|
+
*
|
|
56
|
+
* Calm on purpose, and it says what the executor is going to do next: this is
|
|
57
|
+
* not a failure of the executor, it keeps polling, and the fix is on the server
|
|
58
|
+
* side. An executor that exited here would turn a restartable dispatcher outage
|
|
59
|
+
* into a second outage.
|
|
60
|
+
*/
|
|
61
|
+
export function formatDispatcherUnavailableDiagnostic(liveness) {
|
|
62
|
+
return (`executor: no fresh reconciler/dispatcher heartbeat is available ` +
|
|
63
|
+
`(reconciler_liveness=${liveness}); the dispatcher may not be running. ` +
|
|
64
|
+
"Check DISABLE_SCHEDULER and GET /automation/health; see " +
|
|
65
|
+
"docs/claude/epic-conductor-v2-operator-runbook.md. Continuing to poll.");
|
|
66
|
+
}
|
|
67
|
+
/** The recovery diagnostic, emitted once when an absent dispatcher comes back. */
|
|
68
|
+
export function formatDispatcherRecoveredDiagnostic() {
|
|
69
|
+
return ("executor: reconciler/dispatcher heartbeat is fresh again; continuing to poll.");
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* How often the executor publishes PER-PROCESS liveness (BAPI-871).
|
|
73
|
+
*
|
|
74
|
+
* The server calls an executor stale at 300 s (`EXECUTOR_STALE_SECONDS`). 60 s is
|
|
75
|
+
* comfortably below that — four consecutive misses before anyone is alarmed —
|
|
76
|
+
* which keeps a transient network blip, a VPN reconnect, or one slow request from
|
|
77
|
+
* manufacturing an outage while still detecting a genuinely dead process within
|
|
78
|
+
* the server's own window.
|
|
79
|
+
*
|
|
80
|
+
* Deliberately INDEPENDENT of claim results and of any active job. An executor
|
|
81
|
+
* that is idle is exactly the executor whose liveness was previously invisible,
|
|
82
|
+
* so a cadence that paused while nothing was claimed would rebuild the blind spot
|
|
83
|
+
* this endpoint exists to remove.
|
|
84
|
+
*/
|
|
85
|
+
export const PROCESS_HEARTBEAT_INTERVAL_MS = 60_000;
|
|
86
|
+
/** Delivery-failure diagnostic. Bounded, credential-free, stderr-only. */
|
|
87
|
+
export function formatProcessHeartbeatFailureDiagnostic() {
|
|
88
|
+
return ("executor: process-heartbeat delivery is failing; the server may report this " +
|
|
89
|
+
"executor as stale. Claiming and running jobs are unaffected.");
|
|
90
|
+
}
|
|
91
|
+
/** Emitted once when delivery starts working again. */
|
|
92
|
+
export function formatProcessHeartbeatRecoveredDiagnostic() {
|
|
93
|
+
return "executor: process-heartbeat delivery has recovered.";
|
|
94
|
+
}
|
|
12
95
|
/**
|
|
13
96
|
* Run the executor loop. Returns 0 on the `once` path (a single cycle drained);
|
|
14
97
|
* in continuous mode it does not return (the process is long-lived).
|
|
@@ -37,6 +120,50 @@ export async function runExecutor(options, deps, httpClient, seams = {}) {
|
|
|
37
120
|
// the sole eligibility authority. Declared here, outside both loops below, so
|
|
38
121
|
// it persists across poll cycles for the lifetime of this `runExecutor` call.
|
|
39
122
|
let lastClaimedEpicRunId = null;
|
|
123
|
+
// BAPI-871 — the last normalized dispatcher availability OBSERVED on a
|
|
124
|
+
// successful claim response, retained for the lifetime of this `runExecutor`
|
|
125
|
+
// call. Declared here, outside both loops, so the suppression below spans poll
|
|
126
|
+
// cycles: an absent dispatcher must produce one diagnostic, not one per poll.
|
|
127
|
+
//
|
|
128
|
+
// `null` means "nothing EVALUABLE observed yet", which is what makes the FIRST
|
|
129
|
+
// absent observation announce itself rather than being treated as an unchanged
|
|
130
|
+
// state. Only `fresh` and `absent` are ever stored — see the transparency rule
|
|
131
|
+
// for `unknown` in the observer below.
|
|
132
|
+
//
|
|
133
|
+
// Read-only diagnostics, exactly like `lastClaimedEpicRunId` above: it never
|
|
134
|
+
// feeds claim eligibility, cadence, retries, dispatch, or exit.
|
|
135
|
+
let lastDispatcherAvailability = null;
|
|
136
|
+
/**
|
|
137
|
+
* Announce a dispatcher availability CHANGE, and only a change.
|
|
138
|
+
*
|
|
139
|
+
* Called after every successful claim response and before the runner branches
|
|
140
|
+
* on claimed-versus-no-job, because the no-job branch is precisely the one that
|
|
141
|
+
* used to say nothing at all.
|
|
142
|
+
*/
|
|
143
|
+
const observeDispatcherLiveness = (liveness) => {
|
|
144
|
+
const availability = normalizeDispatcherAvailability(liveness);
|
|
145
|
+
// `unknown` is TRANSPARENT: it emits nothing and, just as importantly, does
|
|
146
|
+
// not overwrite what was last actually established. An operator who saw
|
|
147
|
+
// "the dispatcher may not be running", then a stretch of unevaluable polls,
|
|
148
|
+
// then a fresh heartbeat, must still be told it recovered — and swallowing
|
|
149
|
+
// that recovery because an unevaluable poll sat in between would leave the
|
|
150
|
+
// absence warning standing as the last word.
|
|
151
|
+
if (availability === "unknown")
|
|
152
|
+
return;
|
|
153
|
+
const previous = lastDispatcherAvailability;
|
|
154
|
+
lastDispatcherAvailability = availability;
|
|
155
|
+
if (availability === previous)
|
|
156
|
+
return;
|
|
157
|
+
if (availability === "absent") {
|
|
158
|
+
// stderr, never `console.log` — stdout is the MCP protocol channel.
|
|
159
|
+
deps.errorLog(formatDispatcherUnavailableDiagnostic(liveness));
|
|
160
|
+
}
|
|
161
|
+
else if (previous === "absent") {
|
|
162
|
+
// Only from `absent`. A first-ever `fresh` observation is the ordinary
|
|
163
|
+
// healthy start-up and deserves no announcement.
|
|
164
|
+
deps.errorLog(formatDispatcherRecoveredDiagnostic());
|
|
165
|
+
}
|
|
166
|
+
};
|
|
40
167
|
// BAPI-722: ONE deny-probe cache per `runExecutor` invocation, created OUTSIDE
|
|
41
168
|
// the claim loop below — that scope is the whole feature. A cache created inside
|
|
42
169
|
// the loop would be discarded every cycle and re-probe exactly as before; a
|
|
@@ -50,6 +177,188 @@ export async function runExecutor(options, deps, httpClient, seams = {}) {
|
|
|
50
177
|
denyProbeCache: createDenyProbeCache(),
|
|
51
178
|
...(seams.preflightSeams ?? {}),
|
|
52
179
|
};
|
|
180
|
+
// --- Graceful shutdown state (BAPI-828) -------------------------------
|
|
181
|
+
// ONE registry per invocation — see `live-worker-registry.ts` for why this is
|
|
182
|
+
// not module-global. Every dispatched job receives it, so a worker spawned by
|
|
183
|
+
// any of them is reachable from the signal handlers installed below.
|
|
184
|
+
const liveWorkers = createLiveWorkerRegistry();
|
|
185
|
+
const control = { liveWorkers };
|
|
186
|
+
let shutdownRequested = false;
|
|
187
|
+
/**
|
|
188
|
+
* Wake functions for whichever poll sleep is currently in flight. A list whose
|
|
189
|
+
* entries are REMOVED when their sleep completes normally, rather than one
|
|
190
|
+
* long-lived promise every cycle races against: the latter would accumulate one
|
|
191
|
+
* pending reaction per poll for the entire life of a long-running executor.
|
|
192
|
+
*/
|
|
193
|
+
const shutdownWaiters = [];
|
|
194
|
+
/**
|
|
195
|
+
* Handle the first shutdown signal. Idempotent across repeated and mixed
|
|
196
|
+
* `SIGTERM`/`SIGINT`: a second signal must not send a second `SIGTERM` to a
|
|
197
|
+
* worker, install a second grace timer, or start a second drain.
|
|
198
|
+
*
|
|
199
|
+
* Deliberately does NOT call `process.exit` and does NOT signal any child
|
|
200
|
+
* directly. The loop below is allowed to unwind normally so worker supervision,
|
|
201
|
+
* worker-log closure, and the ownership-checked worktree-lock release all still
|
|
202
|
+
* run — exiting here is precisely what would leave the lock behind.
|
|
203
|
+
*/
|
|
204
|
+
const beginShutdown = () => {
|
|
205
|
+
if (shutdownRequested)
|
|
206
|
+
return;
|
|
207
|
+
shutdownRequested = true;
|
|
208
|
+
deps.errorLog("executor: shutdown signal received; terminating live workers and draining");
|
|
209
|
+
liveWorkers.requestShutdown();
|
|
210
|
+
for (const wake of shutdownWaiters.splice(0, shutdownWaiters.length))
|
|
211
|
+
wake();
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* Sleep one poll interval, or return early once shutdown is requested.
|
|
215
|
+
*
|
|
216
|
+
* The return value is load-bearing: only a `"slept"` result may feed the
|
|
217
|
+
* suspend-gap calculation. A shutdown wake is an INTERRUPTED wait whose elapsed
|
|
218
|
+
* time says nothing about the host, and reporting one as a suspend would
|
|
219
|
+
* manufacture a diagnostic out of an ordinary Ctrl-C.
|
|
220
|
+
*/
|
|
221
|
+
const sleepUntilPollOrShutdown = async (ms) => {
|
|
222
|
+
if (shutdownRequested)
|
|
223
|
+
return "shutdown";
|
|
224
|
+
let wake;
|
|
225
|
+
const interrupted = new Promise((resolve) => {
|
|
226
|
+
wake = resolve;
|
|
227
|
+
});
|
|
228
|
+
shutdownWaiters.push(wake);
|
|
229
|
+
try {
|
|
230
|
+
return await Promise.race([
|
|
231
|
+
deps.sleep(ms).then(() => "slept"),
|
|
232
|
+
interrupted.then(() => "shutdown"),
|
|
233
|
+
]);
|
|
234
|
+
}
|
|
235
|
+
finally {
|
|
236
|
+
const index = shutdownWaiters.indexOf(wake);
|
|
237
|
+
if (index !== -1)
|
|
238
|
+
shutdownWaiters.splice(index, 1);
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
// The heartbeat loop's OWN stop signal (BAPI-871). Deliberately separate from
|
|
242
|
+
// `shutdownRequested`: winding the loop down at the end of a `once` run must
|
|
243
|
+
// not flip the runner's shutdown flag, because `beginShutdown` is idempotent
|
|
244
|
+
// on that flag and a signal arriving late — after the loop ended but before the
|
|
245
|
+
// handlers are unsubscribed — must still reach the live workers.
|
|
246
|
+
let heartbeatStopRequested = false;
|
|
247
|
+
const heartbeatWaiters = [];
|
|
248
|
+
const stopProcessHeartbeats = () => {
|
|
249
|
+
heartbeatStopRequested = true;
|
|
250
|
+
for (const wake of heartbeatWaiters.splice(0, heartbeatWaiters.length))
|
|
251
|
+
wake();
|
|
252
|
+
};
|
|
253
|
+
/**
|
|
254
|
+
* Sleep one heartbeat interval, returning early on either stop condition.
|
|
255
|
+
*
|
|
256
|
+
* Races the interval against a real shutdown AND the loop's own wind-down, so
|
|
257
|
+
* process exit never waits out a full cadence.
|
|
258
|
+
*/
|
|
259
|
+
const sleepUntilHeartbeatOrStop = async (ms) => {
|
|
260
|
+
if (heartbeatStopRequested || shutdownRequested)
|
|
261
|
+
return "stop";
|
|
262
|
+
let wake;
|
|
263
|
+
const interrupted = new Promise((resolve) => {
|
|
264
|
+
wake = resolve;
|
|
265
|
+
});
|
|
266
|
+
heartbeatWaiters.push(wake);
|
|
267
|
+
shutdownWaiters.push(wake);
|
|
268
|
+
try {
|
|
269
|
+
return await Promise.race([
|
|
270
|
+
deps.sleep(ms).then(() => "slept"),
|
|
271
|
+
interrupted.then(() => "stop"),
|
|
272
|
+
]);
|
|
273
|
+
}
|
|
274
|
+
finally {
|
|
275
|
+
for (const list of [heartbeatWaiters, shutdownWaiters]) {
|
|
276
|
+
const index = list.indexOf(wake);
|
|
277
|
+
if (index !== -1)
|
|
278
|
+
list.splice(index, 1);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
/**
|
|
283
|
+
* Publish per-process liveness on a fixed cadence until shutdown (BAPI-871).
|
|
284
|
+
*
|
|
285
|
+
* Runs CONCURRENTLY with the poll loop rather than inside it, because the poll
|
|
286
|
+
* loop's cadence is the operator's to configure and its cycle time varies with
|
|
287
|
+
* whatever work it picks up — neither of which should be able to change how
|
|
288
|
+
* often this process proves it is alive.
|
|
289
|
+
*
|
|
290
|
+
* Every failure mode is contained here: delivery is non-critical monitoring, so
|
|
291
|
+
* a failure is reported to stderr and the loop keeps going. It cannot fail a
|
|
292
|
+
* claim, fail a job, or end the run.
|
|
293
|
+
*/
|
|
294
|
+
const runProcessHeartbeatLoop = async () => {
|
|
295
|
+
const publish = httpClient.processHeartbeat;
|
|
296
|
+
// A client without the capability (every object-literal test fake) simply
|
|
297
|
+
// does not heartbeat. Nothing else about the run changes.
|
|
298
|
+
if (typeof publish !== "function")
|
|
299
|
+
return;
|
|
300
|
+
// `repoName` FIRST so the client authenticates with the primary repo's key in
|
|
301
|
+
// multi-repo mode, then the remaining configured repos, de-duplicated.
|
|
302
|
+
const repoNames = [options.repoName, ...options.repos].filter((repo, index, all) => repo && all.indexOf(repo) === index);
|
|
303
|
+
const request = {
|
|
304
|
+
component: "executor",
|
|
305
|
+
instance_id: options.executorId,
|
|
306
|
+
repo_names: repoNames,
|
|
307
|
+
};
|
|
308
|
+
// Suppression state: one message per CHANGE of delivery state, so a long
|
|
309
|
+
// outage produces one line rather than one per minute.
|
|
310
|
+
let lastFailed = false;
|
|
311
|
+
for (;;) {
|
|
312
|
+
let outcome;
|
|
313
|
+
try {
|
|
314
|
+
outcome = await publish.call(httpClient, request);
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
// The client already contains its own errors; this is belt-and-braces so
|
|
318
|
+
// an unexpected throw can never escape into the run.
|
|
319
|
+
outcome = "failed";
|
|
320
|
+
}
|
|
321
|
+
if (outcome === "failed" && !lastFailed) {
|
|
322
|
+
deps.errorLog(formatProcessHeartbeatFailureDiagnostic());
|
|
323
|
+
}
|
|
324
|
+
else if (outcome === "delivered" && lastFailed) {
|
|
325
|
+
deps.errorLog(formatProcessHeartbeatRecoveredDiagnostic());
|
|
326
|
+
}
|
|
327
|
+
lastFailed = outcome === "failed";
|
|
328
|
+
if (heartbeatStopRequested || shutdownRequested)
|
|
329
|
+
return;
|
|
330
|
+
// Woken by the same waiter list a real shutdown fires, so process exit does
|
|
331
|
+
// not have to wait out a full interval.
|
|
332
|
+
if ((await sleepUntilHeartbeatOrStop(PROCESS_HEARTBEAT_INTERVAL_MS)) === "stop") {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
/**
|
|
338
|
+
* Annotate an abnormally long COMPLETED poll gap (BAPI-828).
|
|
339
|
+
*
|
|
340
|
+
* ADVISORY BY CONSTRUCTION: it logs and appends, and touches nothing else.
|
|
341
|
+
* Preflight, claim eligibility, heartbeat cadence, leases, active jobs, retries,
|
|
342
|
+
* and worktree handling are all deliberately left alone — the host having slept
|
|
343
|
+
* is a fact for the operator reading the log, never an input to scheduling. The
|
|
344
|
+
* whole point is that an executor whose MacBook idled to sleep mid-run left
|
|
345
|
+
* nothing in its logs saying so, and the freeze was misdiagnosed for hours.
|
|
346
|
+
*/
|
|
347
|
+
const reportSuspendGap = async (elapsedMs) => {
|
|
348
|
+
if (elapsedMs <= 3 * options.pollIntervalMs + SUSPEND_GAP_SLACK_MS)
|
|
349
|
+
return;
|
|
350
|
+
const diagnostic = formatSuspendGapDiagnostic(elapsedMs);
|
|
351
|
+
deps.errorLog(diagnostic);
|
|
352
|
+
const appendFile = deps.appendFile;
|
|
353
|
+
if (!appendFile)
|
|
354
|
+
return;
|
|
355
|
+
try {
|
|
356
|
+
await appendToActiveJobLogs(diagnostic, { appendFile });
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
/* annotation is never allowed to disturb the poll loop */
|
|
360
|
+
}
|
|
361
|
+
};
|
|
53
362
|
/**
|
|
54
363
|
* Run the conservative worktree GC sweep ONLY while the runner owns no active
|
|
55
364
|
* job worktree. Failures are logged to stderr and never abort claiming
|
|
@@ -69,7 +378,12 @@ export async function runExecutor(options, deps, httpClient, seams = {}) {
|
|
|
69
378
|
function dispatch(job, report) {
|
|
70
379
|
const promise = (async () => {
|
|
71
380
|
try {
|
|
72
|
-
|
|
381
|
+
// `undefined` for the behavior seams (the job runner defaults them), then
|
|
382
|
+
// the runner-scoped control object carrying the live-worker registry. A
|
|
383
|
+
// job dispatched after shutdown was requested still runs through here so
|
|
384
|
+
// its worktree lock unwinds through the normal `finally`; the registry's
|
|
385
|
+
// sticky flag terminates whatever it manages to spawn.
|
|
386
|
+
await runJob(job, httpClient, options, deps, report, undefined, control);
|
|
73
387
|
}
|
|
74
388
|
catch (err) {
|
|
75
389
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -81,58 +395,150 @@ export async function runExecutor(options, deps, httpClient, seams = {}) {
|
|
|
81
395
|
})();
|
|
82
396
|
active.set(job.id, promise);
|
|
83
397
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
for (const
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
398
|
+
// Handlers go on BEFORE preflight and before the first claim (BAPI-828).
|
|
399
|
+
// Preflight can take seconds and the first claim can spawn a worker immediately
|
|
400
|
+
// after it, so registering any later would leave a real window in which a
|
|
401
|
+
// `SIGTERM` reached the executor and nothing owned the child it had just made.
|
|
402
|
+
const unsubscribers = [];
|
|
403
|
+
const onSignal = seams.onSignal;
|
|
404
|
+
if (onSignal) {
|
|
405
|
+
for (const signal of EXECUTOR_SHUTDOWN_SIGNALS) {
|
|
406
|
+
try {
|
|
407
|
+
unsubscribers.push(onSignal(signal, beginShutdown));
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
// A host that refuses a handler for one signal must not stop the executor
|
|
411
|
+
// from running, or from handling the other signal.
|
|
412
|
+
}
|
|
96
413
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
414
|
+
}
|
|
415
|
+
// Started after the FIRST successful preflight and joined before the runner
|
|
416
|
+
// returns. Declared here so the `finally` below can await it.
|
|
417
|
+
let processHeartbeats = null;
|
|
418
|
+
try {
|
|
419
|
+
for (;;) {
|
|
420
|
+
const report = await collectPreflight(options, deps, preflightSeams);
|
|
421
|
+
// First heartbeat goes out immediately once preflight passes, so a freshly
|
|
422
|
+
// started executor is visible to the server without waiting one interval.
|
|
423
|
+
// Gated on `report.ok` because an executor that preflight refuses is not
|
|
424
|
+
// going to do any work, and advertising it as live would be a lie.
|
|
425
|
+
if (report.ok && processHeartbeats === null && !shutdownRequested) {
|
|
426
|
+
processHeartbeats = runProcessHeartbeatLoop();
|
|
427
|
+
}
|
|
428
|
+
// BAPI-727: preflight warnings were previously collected but never emitted, so
|
|
429
|
+
// a non-fatal finding — an overridden MCP-shadowing collision, an unreadable
|
|
430
|
+
// ~/.claude.json — was invisible to the operator. Emit them before the claim
|
|
431
|
+
// decision; this only logs and never changes the existing `!report.ok`
|
|
432
|
+
// claim-skipping behavior below.
|
|
433
|
+
for (const warning of report.warnings) {
|
|
434
|
+
deps.errorLog(`executor preflight warning: ${warning}`);
|
|
435
|
+
}
|
|
436
|
+
if (!report.ok) {
|
|
437
|
+
deps.errorLog(`executor preflight refused claiming: ${report.fatalFindings.join("; ")}`);
|
|
438
|
+
}
|
|
439
|
+
else if (shutdownRequested) {
|
|
440
|
+
// Shutdown arrived during preflight. Claiming now would take on work this
|
|
441
|
+
// executor is about to stop doing, and the server would have to wait out
|
|
442
|
+
// the whole lease before anyone else could have it.
|
|
443
|
+
deps.errorLog("executor: shutdown requested; not claiming new work");
|
|
444
|
+
}
|
|
445
|
+
else {
|
|
446
|
+
let claiming = true;
|
|
447
|
+
while (claiming && !shutdownRequested && options.maxConcurrent - active.size > 0) {
|
|
448
|
+
const freeSlots = options.maxConcurrent - active.size;
|
|
449
|
+
const manifest = buildClaimManifest(report, options, freeSlots);
|
|
450
|
+
const result = await httpClient.claim(manifest);
|
|
451
|
+
// Evaluated on every SUCCESSFUL claim response — 200 and 204 alike —
|
|
452
|
+
// ahead of the branch below. An error result carries no verdict and is
|
|
453
|
+
// not evidence about the dispatcher, so it is deliberately not observed.
|
|
454
|
+
if (result.kind === "claimed" || result.kind === "none") {
|
|
455
|
+
observeDispatcherLiveness(result.reconcilerLiveness);
|
|
456
|
+
}
|
|
457
|
+
if (result.kind === "claimed") {
|
|
458
|
+
const currentEpicRunId = result.job.epic_run_id ?? null;
|
|
459
|
+
if (lastClaimedEpicRunId !== null &&
|
|
460
|
+
currentEpicRunId !== null &&
|
|
461
|
+
currentEpicRunId !== lastClaimedEpicRunId) {
|
|
462
|
+
deps.errorLog(`executor claimed a job for a different epic run than the previous claim ` +
|
|
463
|
+
`(repo=${result.job.repo_name} previous_epic_run_id=${lastClaimedEpicRunId} ` +
|
|
464
|
+
`current_epic_run_id=${currentEpicRunId} scoped=${options.epicRunIds !== undefined})`);
|
|
465
|
+
}
|
|
466
|
+
if (currentEpicRunId !== null)
|
|
467
|
+
lastClaimedEpicRunId = currentEpicRunId;
|
|
468
|
+
// Dispatched even when the signal landed while this claim was in
|
|
469
|
+
// flight (BAPI-828). The job is OURS the moment the server answered,
|
|
470
|
+
// so dropping it here would strand a claimed row under a live lease
|
|
471
|
+
// and skip the worktree-lock unwind. The registry's sticky shutdown
|
|
472
|
+
// terminates whatever it spawns, immediately, on registration.
|
|
473
|
+
dispatch(result.job, report);
|
|
474
|
+
}
|
|
475
|
+
else if (result.kind === "none") {
|
|
476
|
+
claiming = false;
|
|
477
|
+
}
|
|
478
|
+
else {
|
|
479
|
+
deps.errorLog(`executor claim ${result.kind}: ${result.error}`);
|
|
480
|
+
claiming = false;
|
|
111
481
|
}
|
|
112
|
-
if (currentEpicRunId !== null)
|
|
113
|
-
lastClaimedEpicRunId = currentEpicRunId;
|
|
114
|
-
dispatch(result.job, report);
|
|
115
|
-
}
|
|
116
|
-
else if (result.kind === "none") {
|
|
117
|
-
claiming = false;
|
|
118
|
-
}
|
|
119
|
-
else {
|
|
120
|
-
deps.errorLog(`executor claim ${result.kind}: ${result.error}`);
|
|
121
|
-
claiming = false;
|
|
122
482
|
}
|
|
483
|
+
// Conservative worktree GC runs once per cycle, AFTER the claim attempt and
|
|
484
|
+
// only while idle (`active.size === 0` — no owned job worktree). This covers
|
|
485
|
+
// both the startup cycle and between-poll-cycle cadence without ever
|
|
486
|
+
// sweeping while a job is running. Kept off the pre-claim path so it never
|
|
487
|
+
// delays claiming a ready job.
|
|
488
|
+
await maybeSweepWorktrees();
|
|
489
|
+
}
|
|
490
|
+
if (options.once || shutdownRequested)
|
|
491
|
+
break;
|
|
492
|
+
// Sample the clock immediately around the ACTUAL sleep, so the measured gap
|
|
493
|
+
// is the wait itself and not the claim/GC work on either side of it.
|
|
494
|
+
const sleepStartedAt = deps.now();
|
|
495
|
+
const wake = await sleepUntilPollOrShutdown(options.pollIntervalMs);
|
|
496
|
+
if (wake === "shutdown")
|
|
497
|
+
break;
|
|
498
|
+
await reportSuspendGap(deps.now() - sleepStartedAt);
|
|
499
|
+
}
|
|
500
|
+
// Drain every dispatched job — on the `once` path and the shutdown path alike.
|
|
501
|
+
// Terminated workers still have to finish supervision, flush final telemetry,
|
|
502
|
+
// close their worker logs, deregister, and release their worktree locks;
|
|
503
|
+
// returning ahead of that is exactly what would leave a lock owned by a
|
|
504
|
+
// process that has already exited.
|
|
505
|
+
//
|
|
506
|
+
// INSIDE the try, so the handlers are still installed while it runs: a second
|
|
507
|
+
// `SIGTERM` arriving mid-drain must reach `beginShutdown` (an idempotent no-op
|
|
508
|
+
// by then) rather than Node's default terminate action, which would kill the
|
|
509
|
+
// executor in the middle of releasing its locks.
|
|
510
|
+
await Promise.all(active.values());
|
|
511
|
+
}
|
|
512
|
+
finally {
|
|
513
|
+
// Join the heartbeat loop before returning, via its OWN stop signal. A `once`
|
|
514
|
+
// run breaks out of the poll loop with `shutdownRequested` still false, and
|
|
515
|
+
// it must stay false: `beginShutdown` is idempotent on that flag, so setting
|
|
516
|
+
// it here would silently swallow a signal that arrives during this wind-down.
|
|
517
|
+
if (processHeartbeats !== null) {
|
|
518
|
+
stopProcessHeartbeats();
|
|
519
|
+
try {
|
|
520
|
+
await processHeartbeats;
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
/* a monitoring loop must never change the runner's outcome */
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
// Unsubscribe in a runner-level `finally`, so a direct or repeated
|
|
527
|
+
// `runExecutor` call — an embedded runner, a `--once` invocation in a loop, a
|
|
528
|
+
// test suite — cannot leave stale listeners bound to a registry that has since
|
|
529
|
+
// gone out of scope.
|
|
530
|
+
for (const unsubscribe of unsubscribers) {
|
|
531
|
+
try {
|
|
532
|
+
unsubscribe();
|
|
533
|
+
}
|
|
534
|
+
catch {
|
|
535
|
+
/* a failed unsubscribe must not mask the runner's own outcome */
|
|
123
536
|
}
|
|
124
|
-
// Conservative worktree GC runs once per cycle, AFTER the claim attempt and
|
|
125
|
-
// only while idle (`active.size === 0` — no owned job worktree). This covers
|
|
126
|
-
// both the startup cycle and between-poll-cycle cadence without ever
|
|
127
|
-
// sweeping while a job is running. Kept off the pre-claim path so it never
|
|
128
|
-
// delays claiming a ready job.
|
|
129
|
-
await maybeSweepWorktrees();
|
|
130
537
|
}
|
|
131
|
-
if (options.once)
|
|
132
|
-
break;
|
|
133
|
-
await deps.sleep(options.pollIntervalMs);
|
|
134
538
|
}
|
|
135
|
-
//
|
|
136
|
-
|
|
539
|
+
// A graceful signal shutdown is a NORMAL executor exit, not an error: the run did
|
|
540
|
+
// exactly what it was asked to do. A distinct nonzero code here would make every
|
|
541
|
+
// deliberate operator stop look like a crash to launchd, systemd, and the CLI
|
|
542
|
+
// boundary alike.
|
|
137
543
|
return 0;
|
|
138
544
|
}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { resolveAgentSpec } from "../agent-registry.js";
|
|
10
10
|
import { createClaudeExecutorAdapter } from "../agent-launchers/claude-executor-adapter.js";
|
|
11
11
|
import { validateExecutorAdapterCapabilities } from "../agent-launchers/executor-adapter.js";
|
|
12
|
+
import { MCP_SERVER_NAME } from "../mcp-identity.js";
|
|
12
13
|
export class VirtualClock {
|
|
13
14
|
t = 0;
|
|
14
15
|
timers = [];
|
|
@@ -70,7 +71,7 @@ async function flushMicrotasks() {
|
|
|
70
71
|
export const DEFAULT_WORKER_INIT_EVENT_LINE = `${JSON.stringify({
|
|
71
72
|
type: "system",
|
|
72
73
|
subtype: "init",
|
|
73
|
-
mcp_servers: [{ name:
|
|
74
|
+
mcp_servers: [{ name: MCP_SERVER_NAME, status: "connected" }],
|
|
74
75
|
})}\n`;
|
|
75
76
|
/** A single-chunk stdout stream carrying {@link DEFAULT_WORKER_INIT_EVENT_LINE}. */
|
|
76
77
|
async function* defaultWorkerStdout() {
|
|
@@ -136,7 +137,7 @@ export function makeFakeExecutorDeps(clock, overrides = {}) {
|
|
|
136
137
|
// registration must therefore override this read explicitly — it will not
|
|
137
138
|
// get that state by accident, which is the point.
|
|
138
139
|
if (typeof filePath === "string" && filePath.endsWith(".mcp.json")) {
|
|
139
|
-
return JSON.stringify({ mcpServers: {
|
|
140
|
+
return JSON.stringify({ mcpServers: { [MCP_SERVER_NAME]: { command: "node", args: [] } } });
|
|
140
141
|
}
|
|
141
142
|
// Mirror real fs/promises: a missing-file rejection carries code "ENOENT"
|
|
142
143
|
// (so BAPI-664 command provisioning treats absent files as fillable).
|