@bridge_gpt/mcp-server 0.2.53 → 0.2.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -56
- package/build/agent-guidance.generated.js +3 -0
- package/build/agent-launchers/claude-executor-adapter.js +3 -0
- package/build/agent-launchers/claude.js +3 -3
- package/build/agent-launchers/prompt.js +8 -11
- package/build/agent-notes.js +178 -0
- package/build/agent-registry.js +5 -2
- package/build/agent-utils.js +58 -0
- package/build/agents.generated.js +1 -1
- package/build/base-ref.js +33 -9
- package/build/bounded-wait.js +174 -0
- package/build/codex-skill-adapter.js +55 -0
- package/build/commands.generated.js +6 -5
- package/build/conductor/bridge-api-client.js +235 -14
- package/build/conductor/bring-up-facts.js +187 -0
- package/build/conductor/claude-hook.js +7 -5
- package/build/conductor/cli.js +28 -0
- package/build/conductor/doctor.js +80 -9
- package/build/conductor/epic-implementer-cli.js +1298 -0
- package/build/conductor/epic-runtime.js +134 -98
- package/build/conductor/errors.js +2 -2
- package/build/conductor/git-ci-types.js +1 -1
- package/build/conductor/git-hooks.js +28 -14
- package/build/conductor/install-doctor.js +11 -5
- package/build/conductor/readiness-cli.js +10 -10
- package/build/conductor/readiness-sections.js +58 -9
- package/build/conductor/readiness.js +120 -4
- package/build/conductor/recovery-cli.js +289 -10
- package/build/conductor/recovery-operations.js +125 -2
- package/build/conductor/repair-contract.js +58 -0
- package/build/conductor/run-branch.js +137 -0
- package/build/conductor/store.js +2 -2
- package/build/conductor/supervisor-runtime.js +1 -1
- package/build/conductor/test-run-branch-vectors.js +165 -0
- package/build/conductor-bin.js +2 -139
- package/build/conductor-claude-hook-bin.js +2 -2
- package/build/conductor-claude-hook-removed-stub-bin.js +31 -0
- package/build/conductor-removed-stub-bin.js +30 -0
- package/build/docs.generated.js +1 -1
- package/build/doctor.js +145 -18
- package/build/drive-epic.js +752 -90
- package/build/epic-implementer-bin.js +145 -0
- package/build/epic-implementer-bundle-cli.js +264 -0
- package/build/epic-implementer-claude-hook-bin.js +3 -0
- package/build/epic-integration-pr.js +5 -3
- package/build/executor/claim-scope.js +104 -0
- package/build/executor/cli.js +14 -25
- package/build/executor/env-file-guard.js +82 -3
- package/build/executor/env.js +6 -0
- package/build/executor/interrupted-worktree.js +60 -0
- package/build/executor/job-errors.js +45 -0
- package/build/executor/job-runner.js +334 -9
- package/build/executor/job-types.js +25 -9
- package/build/executor/merge-tree-classifier.js +171 -0
- package/build/executor/reconcile-overlap-governance.js +129 -0
- package/build/executor/reconcile-overlap-job.js +989 -0
- package/build/executor/reconcile-overlap-types.js +14 -0
- package/build/executor/spawn-job-driver.js +1 -0
- package/build/executor/types.js +2 -0
- package/build/executor/worker-finalization.js +25 -2
- package/build/executor/worker-guard-hook.js +15 -7
- package/build/implement-epic/bridge-client.js +773 -0
- package/build/implement-epic/checkpoint-store.js +542 -0
- package/build/implement-epic/cli.js +3158 -0
- package/build/implement-epic/cut-protocol.js +392 -0
- package/build/implement-epic/lock.js +302 -0
- package/build/implement-epic/pr-state.js +286 -0
- package/build/implement-epic/spawn.js +113 -0
- package/build/index.js +657 -481
- package/build/init.js +72 -8
- package/build/install-bridge-conductor.js +5 -5
- package/build/install-bridge.js +403 -70
- package/build/local-artifact-storage.js +130 -0
- package/build/mcp-host-config.js +22 -60
- package/build/mcp-host-entry-adapter.js +18 -0
- package/build/mcp-host-targets.js +1 -21
- package/build/merge-pull-request.js +1 -1
- package/build/pipelines.generated.js +23 -16
- package/build/plan-epic-conductor-eligibility.js +1 -1
- package/build/plane/cli.js +321 -41
- package/build/plane/manifest.js +209 -1
- package/build/plane/member-roster.js +70 -0
- package/build/plane/preflight.js +128 -12
- package/build/plane/shutdown.js +18 -5
- package/build/plane/status.js +35 -1
- package/build/plane/supervisor.js +546 -164
- package/build/plane/types.js +25 -2
- package/build/polling-policy.js +72 -0
- package/build/readiness-check.js +3 -3
- package/build/readme.generated.js +1 -1
- package/build/review-generation.js +219 -0
- package/build/run-unit-tests-launcher.js +6 -1
- package/build/setup-epic.js +567 -38
- package/build/start-tickets-conductor.js +8 -7
- package/build/ticket-key-utils.js +4 -3
- package/build/ticket-review-artifact-gate.js +461 -0
- package/build/upgrade-cli.js +5 -26
- package/build/version.generated.js +3 -3
- package/build/worker-guard-hook-bin.js +1 -1
- package/docs/CONDUCTOR.md +8 -6
- package/docs/install/mcp-tool-integrations.md +23 -1
- package/package.json +5 -3
- package/pipelines/review-ticket.json +17 -4
|
@@ -25,11 +25,12 @@
|
|
|
25
25
|
* mystery.
|
|
26
26
|
*/
|
|
27
27
|
import path from "path";
|
|
28
|
-
import { PLANE_ID_ENV_VAR, PLANE_RUNTIME_LOG_FILENAME, PLANE_RUNTIME_LOG_PATH, } from "./types.js";
|
|
29
|
-
import { getPlanePaths, readPlaneManifest, writePlaneManifest } from "./manifest.js";
|
|
28
|
+
import { PLANE_ID_ENV_VAR, PLANE_RUNTIME_ACTION, PLANE_RUNTIME_LOG_FILENAME, PLANE_RUNTIME_LOG_PATH, } from "./types.js";
|
|
29
|
+
import { appendPlaneManifestMembers, getPlanePaths, planeManifestLifecycle, readPlaneManifest, transitionPlaneManifestLifecycle, writePlaneManifest, } from "./manifest.js";
|
|
30
|
+
import { executorClaimScopeArgs, } from "../executor/claim-scope.js";
|
|
30
31
|
import { attachMemberOutput, openMemberLog, openPlaneRuntimeLog, writeMemberStartupFailure, writeRuntimeTraceLine, PLANE_MEMBER_STARTUP_FAILURES, PLANE_SUPERVISOR_LOG_PREFIX, } from "./member-logs.js";
|
|
31
32
|
import { absoluteLogPath, describePlaneMember } from "./member-roster.js";
|
|
32
|
-
import { shutdownPlane } from "./shutdown.js";
|
|
33
|
+
import { PLANE_SHUTDOWN_GRACE_MS, PLANE_SHUTDOWN_KILL_WAIT_MS, PLANE_SHUTDOWN_POLL_MS, shutdownPlane, } from "./shutdown.js";
|
|
33
34
|
/** How long the runtime waits for the launcher to record its pid. */
|
|
34
35
|
export const PLANE_HANDSHAKE_TIMEOUT_MS = 10_000;
|
|
35
36
|
/** Handshake poll interval. */
|
|
@@ -178,7 +179,7 @@ export function reportMemberCrash(sinks, report) {
|
|
|
178
179
|
* rather than a per-field judgement call.
|
|
179
180
|
*/
|
|
180
181
|
export async function launchPlaneSupervisor(params, deps) {
|
|
181
|
-
const { context, manifest, executors } = params;
|
|
182
|
+
const { context, manifest, executors, claimScope, twoPhase } = params;
|
|
182
183
|
const paths = getPlanePaths(context.repoRoot);
|
|
183
184
|
const tracePath = path.join(paths.planeDir, PLANE_RUNTIME_LOG_FILENAME);
|
|
184
185
|
// BAPI-778: the subscription OAuth token is a bearer secret of the same class as
|
|
@@ -218,7 +219,18 @@ export async function launchPlaneSupervisor(params, deps) {
|
|
|
218
219
|
runtimeEnv.BAPI_API_KEY = context.bridgeApiKey;
|
|
219
220
|
let child;
|
|
220
221
|
try {
|
|
221
|
-
child = deps.spawn(deps.nodeExecutable, [
|
|
222
|
+
child = deps.spawn(deps.nodeExecutable, [
|
|
223
|
+
deps.entrypoint,
|
|
224
|
+
"plane",
|
|
225
|
+
PLANE_RUNTIME_ACTION,
|
|
226
|
+
"--executors",
|
|
227
|
+
String(executors),
|
|
228
|
+
// BAPI-1102. In two-phase mode the scope is absent here and arrives
|
|
229
|
+
// through the manifest instead — the runtime is told WHICH lifecycle to
|
|
230
|
+
// run, and the lifecycle decides where the scope comes from.
|
|
231
|
+
...(twoPhase ? [PLANE_RUNTIME_TWO_PHASE_FLAG] : []),
|
|
232
|
+
...(claimScope ? executorClaimScopeArgs(claimScope) : []),
|
|
233
|
+
], { cwd: context.repoRoot, env: runtimeEnv, detached: true });
|
|
222
234
|
}
|
|
223
235
|
catch {
|
|
224
236
|
// Fixed prose only. The thrown value can carry the spawn's argv and env,
|
|
@@ -249,7 +261,18 @@ export async function launchPlaneSupervisor(params, deps) {
|
|
|
249
261
|
// The runtime's stdout feed is the operator's view of the plane. It is
|
|
250
262
|
// streamed through rather than teed: per-member detail already lives in the
|
|
251
263
|
// member logs, and this channel carries only attributed events and banners.
|
|
264
|
+
let controlReadyAnnounced = false;
|
|
265
|
+
let lanesFailedAnnounced = false;
|
|
252
266
|
const readStdout = createLineReader((line) => {
|
|
267
|
+
if (line === PLANE_CONTROL_READY_MARKER) {
|
|
268
|
+
// Private, exactly like the ready marker: forwarding it would put an
|
|
269
|
+
// internal token in the operator's terminal.
|
|
270
|
+
if (!controlReadyAnnounced) {
|
|
271
|
+
controlReadyAnnounced = true;
|
|
272
|
+
deps.onControlPlaneReady?.({ supervisorPid, supervisorPgid: supervisorPid });
|
|
273
|
+
}
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
253
276
|
if (line === PLANE_READY_MARKER) {
|
|
254
277
|
// Private. Forwarding it would put an internal token in the operator's
|
|
255
278
|
// terminal and, worse, invite something downstream to parse for it.
|
|
@@ -259,6 +282,16 @@ export async function launchPlaneSupervisor(params, deps) {
|
|
|
259
282
|
}
|
|
260
283
|
return;
|
|
261
284
|
}
|
|
285
|
+
if (line === PLANE_LANES_FAILED_MARKER) {
|
|
286
|
+
// Private, like the other two. The operator-facing explanation was already
|
|
287
|
+
// written to stderr by the runtime, so this token carries no text of its
|
|
288
|
+
// own — only the fact, to whoever is waiting on it.
|
|
289
|
+
if (!lanesFailedAnnounced) {
|
|
290
|
+
lanesFailedAnnounced = true;
|
|
291
|
+
deps.onLanesFailed?.({ supervisorPid, supervisorPgid: supervisorPid });
|
|
292
|
+
}
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
262
295
|
deps.sinks.stdout(line);
|
|
263
296
|
});
|
|
264
297
|
const readStderr = createLineReader((line) => {
|
|
@@ -353,6 +386,37 @@ export async function launchPlaneSupervisor(params, deps) {
|
|
|
353
386
|
exitCode: outcome.code ?? 0,
|
|
354
387
|
};
|
|
355
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* Emitted on stdout the moment the CONTROL PLANE is ready (BAPI-1102).
|
|
391
|
+
*
|
|
392
|
+
* A second private marker beside {@link PLANE_READY_MARKER}, for the same reason
|
|
393
|
+
* that one exists: the launcher needs to know a phase completed, and the only
|
|
394
|
+
* channel to a detached child is its stdout. Never forwarded to the operator's
|
|
395
|
+
* terminal.
|
|
396
|
+
*/
|
|
397
|
+
export const PLANE_CONTROL_READY_MARKER = "__bapi_plane_control_ready__";
|
|
398
|
+
/**
|
|
399
|
+
* Phase two failed and the control plane was HELD (BAPI-1102).
|
|
400
|
+
*
|
|
401
|
+
* The third private marker, and the only one that reports a failure. It exists
|
|
402
|
+
* because the held path is the one bring-up outcome that neither reaches
|
|
403
|
+
* {@link PLANE_READY_MARKER} nor ends the runtime process: a launcher racing
|
|
404
|
+
* "ready" against "the runtime exited" would wait forever on a plane that is
|
|
405
|
+
* alive, serving a committed run, and never going to start a lane.
|
|
406
|
+
*/
|
|
407
|
+
export const PLANE_LANES_FAILED_MARKER = "__bapi_plane_lanes_failed__";
|
|
408
|
+
/**
|
|
409
|
+
* Private argv flag selecting the two-phase lifecycle on `plane __runtime`.
|
|
410
|
+
*
|
|
411
|
+
* Never part of the documented surface, exactly like the action it modifies:
|
|
412
|
+
* the two-phase lifecycle is composed by `drive-epic`, and an operator running
|
|
413
|
+
* `plane up` by hand wants the single-phase bring-up they already know.
|
|
414
|
+
*/
|
|
415
|
+
export const PLANE_RUNTIME_TWO_PHASE_FLAG = "--two-phase";
|
|
416
|
+
/** Default poll interval while waiting for a phase-two request. */
|
|
417
|
+
export const PLANE_PHASE_REQUEST_POLL_MS = 250;
|
|
418
|
+
/** Default bound on the phase-two wait. */
|
|
419
|
+
export const PLANE_PHASE_REQUEST_TIMEOUT_MS = 15 * 60_000;
|
|
356
420
|
/**
|
|
357
421
|
* The detached supervisor loop: adopt the manifest, spawn the roster, watch it,
|
|
358
422
|
* and wind down on a signal.
|
|
@@ -365,19 +429,37 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
|
|
|
365
429
|
const adopted = await awaitManifestOwnership(repoRoot, deps);
|
|
366
430
|
if (!adopted.ok)
|
|
367
431
|
return { ok: false, error: adopted.error, exitCode: 1 };
|
|
432
|
+
/** One member's initial manifest record: listed, not yet spawned. */
|
|
433
|
+
const pendingMember = (spec) => ({
|
|
434
|
+
name: spec.name,
|
|
435
|
+
pid: null,
|
|
436
|
+
state: "spawning",
|
|
437
|
+
exitCode: null,
|
|
438
|
+
exitSignal: null,
|
|
439
|
+
logPath: spec.logPath,
|
|
440
|
+
});
|
|
368
441
|
let manifest = {
|
|
369
442
|
...adopted.manifest,
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
exitSignal: null,
|
|
376
|
-
logPath: spec.logPath,
|
|
377
|
-
})),
|
|
443
|
+
// In two-phase mode `roster` is the CONTROL-PLANE cohort; later cohorts are
|
|
444
|
+
// appended as each phase begins, so `plane status` and `plane down` never
|
|
445
|
+
// list a member that has not been asked to start.
|
|
446
|
+
members: roster.map(pendingMember),
|
|
447
|
+
...(deps.twoPhase ? { lifecycle: "control-plane-starting" } : {}),
|
|
378
448
|
updatedAt: deps.clock.now().toISOString(),
|
|
379
449
|
};
|
|
380
450
|
const started = [];
|
|
451
|
+
/**
|
|
452
|
+
* Has {@link PLANE_CONTROL_READY_MARKER} been announced? (BAPI-1102)
|
|
453
|
+
*
|
|
454
|
+
* The single boundary between "roll the whole plane back" and "hold the
|
|
455
|
+
* control plane and stop only what phase two started". It flips at the
|
|
456
|
+
* announcement rather than at the manifest transition, because the
|
|
457
|
+
* announcement is what lets the composed caller go create a run — before it,
|
|
458
|
+
* nothing outside this process can depend on the plane existing.
|
|
459
|
+
*/
|
|
460
|
+
let phaseTwoBegun = false;
|
|
461
|
+
/** Members started by phase two: the ONLY cohort a held failure tears down. */
|
|
462
|
+
const phaseTwoMembers = [];
|
|
381
463
|
// ---- Serialized manifest persistence (BAPI-950) --------------------------
|
|
382
464
|
//
|
|
383
465
|
// `writePlaneManifest` publishes through a temporary file and an atomic
|
|
@@ -481,175 +563,381 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
|
|
|
481
563
|
selfPid: deps.selfPid,
|
|
482
564
|
});
|
|
483
565
|
};
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
});
|
|
518
|
-
if (outcome.kind === "failed") {
|
|
519
|
-
// Terminal before the spawn: a rejected credential or a moved contract
|
|
520
|
-
// cannot be waited out, and starting the member first would leave a live
|
|
521
|
-
// process to roll back for no reason.
|
|
522
|
-
return finishPartialStart(spec.name, describeBaselineFailure(outcome.reason), {
|
|
523
|
-
log: opened.stream,
|
|
524
|
-
failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
|
|
525
|
-
});
|
|
526
|
-
}
|
|
527
|
-
baseline = outcome.baseline;
|
|
528
|
-
baselineElapsedMs = outcome.elapsedMs;
|
|
566
|
+
/**
|
|
567
|
+
* Members phase two stopped on purpose, so their exits are not reported as
|
|
568
|
+
* crashes. Per-member rather than the plane-wide `shuttingDown` flag: the
|
|
569
|
+
* control plane stays up and a genuine crash of ITS members must still be
|
|
570
|
+
* reported after this.
|
|
571
|
+
*/
|
|
572
|
+
const intentionallyStopped = new Set();
|
|
573
|
+
/**
|
|
574
|
+
* Stop exactly the cohort phase two started, and nothing else (BAPI-1102).
|
|
575
|
+
*
|
|
576
|
+
* SIGTERM, a grace window on the injected clock, then SIGKILL for whatever is
|
|
577
|
+
* still alive — the same escalation `shutdownPlane` uses, applied to a subset
|
|
578
|
+
* rather than to the plane. It deliberately does NOT touch the manifest's
|
|
579
|
+
* ownership, the supervisor, the server, or the reconciler: the whole point of
|
|
580
|
+
* the held path is that those survive.
|
|
581
|
+
*
|
|
582
|
+
* Liveness is read from the members' own `close` handlers, which is what makes
|
|
583
|
+
* this correct without retaining child handles: the handler patches
|
|
584
|
+
* `state: "exited"` on the manifest projection, and that projection is the one
|
|
585
|
+
* record of member lifecycle everything else here reads.
|
|
586
|
+
*/
|
|
587
|
+
const stopPhaseTwoCohort = async () => {
|
|
588
|
+
const outstanding = () => manifest.members.filter((m) => phaseTwoMembers.includes(m.name) && m.pid !== null && m.state !== "exited");
|
|
589
|
+
const targets = outstanding();
|
|
590
|
+
if (targets.length === 0)
|
|
591
|
+
return;
|
|
592
|
+
for (const m of targets)
|
|
593
|
+
intentionallyStopped.add(m.name);
|
|
594
|
+
deps.sinks.stderr(`Stopping ${targets.length} executor-lane member(s) started by this phase. ` +
|
|
595
|
+
"The control plane is NOT being rolled back.");
|
|
596
|
+
for (const m of targets) {
|
|
597
|
+
if (m.pid !== null)
|
|
598
|
+
deps.proc.signal(m.pid, "SIGTERM");
|
|
529
599
|
}
|
|
530
|
-
let
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
env: spec.env,
|
|
535
|
-
// Members stay in the RUNTIME's process group, which is what makes one
|
|
536
|
-
// group signal reach the whole plane.
|
|
537
|
-
detached: false,
|
|
538
|
-
});
|
|
600
|
+
let waited = 0;
|
|
601
|
+
while (waited < PLANE_SHUTDOWN_GRACE_MS && outstanding().length > 0) {
|
|
602
|
+
await deps.clock.sleep(PLANE_SHUTDOWN_POLL_MS);
|
|
603
|
+
waited += PLANE_SHUTDOWN_POLL_MS;
|
|
539
604
|
}
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
});
|
|
605
|
+
const survivors = outstanding();
|
|
606
|
+
for (const m of survivors) {
|
|
607
|
+
if (m.pid !== null)
|
|
608
|
+
deps.proc.signal(m.pid, "SIGKILL");
|
|
545
609
|
}
|
|
546
|
-
if (
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
610
|
+
if (survivors.length > 0) {
|
|
611
|
+
let killWaited = 0;
|
|
612
|
+
while (killWaited < PLANE_SHUTDOWN_KILL_WAIT_MS && outstanding().length > 0) {
|
|
613
|
+
await deps.clock.sleep(PLANE_SHUTDOWN_POLL_MS);
|
|
614
|
+
killWaited += PLANE_SHUTDOWN_POLL_MS;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
const stubborn = outstanding();
|
|
618
|
+
if (stubborn.length > 0) {
|
|
619
|
+
deps.sinks.stderr(`${stubborn.length} executor-lane member(s) did not exit: ` +
|
|
620
|
+
`${stubborn.map((m) => `${m.name} (pid ${m.pid})`).join(", ")}. ` +
|
|
621
|
+
"They are still recorded in the manifest and `plane down` will reach them.");
|
|
551
622
|
}
|
|
552
|
-
attachMemberOutput({ stdout: child.stdout, stderr: child.stderr }, opened.stream);
|
|
553
|
-
started.push(spec.name);
|
|
554
|
-
patchMember(spec.name, { pid: child.pid, state: "running" });
|
|
555
623
|
await persist();
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
//
|
|
564
|
-
//
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
624
|
+
};
|
|
625
|
+
/** Record the terminal held phase, best-effort. */
|
|
626
|
+
const recordLanesFailed = async () => {
|
|
627
|
+
const current = manifest.lifecycle ?? "ready";
|
|
628
|
+
const written = await transitionPlaneManifestLifecycle(repoRoot, deps.planeId, current, "lanes-failed", deps.fs);
|
|
629
|
+
// Best-effort on purpose: the phase is how `plane status` describes a held
|
|
630
|
+
// plane, not a fact anything decides on. A manifest that cannot be written
|
|
631
|
+
// is already the subject of its own reported failure, and turning that into
|
|
632
|
+
// a second one would bury the first.
|
|
633
|
+
if (written.ok) {
|
|
634
|
+
manifest = { ...written.manifest, members: manifest.members };
|
|
635
|
+
await persist();
|
|
636
|
+
}
|
|
637
|
+
else {
|
|
638
|
+
deps.sinks.stderr(`The held plane's lifecycle could not be recorded (${written.reason}); ` +
|
|
639
|
+
"`plane status` may still describe it as starting lanes.");
|
|
640
|
+
}
|
|
641
|
+
};
|
|
642
|
+
// ---- Spawn one cohort of members, in order ------------------------------
|
|
643
|
+
//
|
|
644
|
+
// BAPI-1102 extracted this from a single top-level loop so the two-phase
|
|
645
|
+
// lifecycle can run it three times — control plane, scoped lanes, observer —
|
|
646
|
+
// against the SAME closure state. Every property the single pass had is
|
|
647
|
+
// preserved by construction rather than re-established: one `started` list, one
|
|
648
|
+
// serialized manifest writer, one rollback, one set of member events.
|
|
649
|
+
//
|
|
650
|
+
// Returns `null` on success, or the terminal failure result. A cohort failure
|
|
651
|
+
// rolls back EVERY member started so far, across all cohorts, which is the
|
|
652
|
+
// behavior a partially-started plane needs and the reason `started` is shared.
|
|
653
|
+
const startMembers = async (cohort) => {
|
|
654
|
+
for (const spec of cohort) {
|
|
655
|
+
const absoluteLog = absoluteLogPath(repoRoot, spec);
|
|
656
|
+
const opened = openMemberLog(spec.name, paths.planeDir, absoluteLog, deps.log);
|
|
657
|
+
if (!opened.ok) {
|
|
658
|
+
return finishPartialStart(spec.name, `log could not be opened (${opened.error})`);
|
|
659
|
+
}
|
|
660
|
+
// Probed BEFORE the spawn, for every heartbeat-gated member (BAPI-1036;
|
|
661
|
+
// BAPI-1029 took one such probe, for the reconciler only, and used it merely
|
|
662
|
+
// to disclose an ambiguity). The baseline is what turns "a fresh heartbeat
|
|
663
|
+
// exists" into "a heartbeat landed after we started this process" — see
|
|
664
|
+
// `PlaneHeartbeatBaseline`.
|
|
665
|
+
//
|
|
666
|
+
// Taken AFTER the member's log is opened, so a terminal baseline failure is
|
|
667
|
+
// annotated in the log an operator will actually open for that member —
|
|
668
|
+
// still before the spawn, which is the property that matters.
|
|
669
|
+
//
|
|
670
|
+
// An executor lane is scoped here by the identity the roster already minted;
|
|
671
|
+
// the reconciler's local identity needs a pid, so its pre-spawn baseline is
|
|
672
|
+
// the UNSCOPED observation, which is a sound lower bound: the unscoped read
|
|
673
|
+
// returns the newest row across the deployment, so any row this member later
|
|
674
|
+
// publishes must be at least as new. A `DYNO` deployment scopes exactly,
|
|
675
|
+
// because that identity needs no pid.
|
|
676
|
+
let baseline = { observedAtMs: null };
|
|
677
|
+
let baselineElapsedMs = 0;
|
|
678
|
+
if (spec.readiness?.kind === "process-heartbeat") {
|
|
679
|
+
const readiness = spec.readiness;
|
|
680
|
+
const baselineInstanceId = readiness.instanceId ?? resolveMemberInstanceId(spec.env, 0, undefined);
|
|
681
|
+
const outcome = await probeHeartbeatBaseline({
|
|
682
|
+
deps,
|
|
683
|
+
spec,
|
|
684
|
+
readiness,
|
|
685
|
+
instanceId: baselineInstanceId,
|
|
686
|
+
});
|
|
687
|
+
if (outcome.kind === "failed") {
|
|
688
|
+
// Terminal before the spawn: a rejected credential or a moved contract
|
|
689
|
+
// cannot be waited out, and starting the member first would leave a live
|
|
690
|
+
// process to roll back for no reason.
|
|
691
|
+
return finishPartialStart(spec.name, describeBaselineFailure(outcome.reason), {
|
|
692
|
+
log: opened.stream,
|
|
693
|
+
failure: PLANE_MEMBER_STARTUP_FAILURES.readinessUnestablished,
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
baseline = outcome.baseline;
|
|
697
|
+
baselineElapsedMs = outcome.elapsedMs;
|
|
698
|
+
}
|
|
699
|
+
let child;
|
|
700
|
+
try {
|
|
701
|
+
child = deps.spawn(spec.command, spec.args, {
|
|
702
|
+
cwd: spec.cwd,
|
|
703
|
+
env: spec.env,
|
|
704
|
+
// Members stay in the RUNTIME's process group, which is what makes one
|
|
705
|
+
// group signal reach the whole plane.
|
|
706
|
+
detached: false,
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
catch {
|
|
710
|
+
return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.spawnThrew, {
|
|
711
|
+
log: opened.stream,
|
|
712
|
+
failure: PLANE_MEMBER_STARTUP_FAILURES.spawnThrew,
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
if (child.pid === null || child.pid === undefined) {
|
|
716
|
+
return finishPartialStart(spec.name, PLANE_MEMBER_STARTUP_FAILURES.noPid, {
|
|
717
|
+
log: opened.stream,
|
|
718
|
+
failure: PLANE_MEMBER_STARTUP_FAILURES.noPid,
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
attachMemberOutput({ stdout: child.stdout, stderr: child.stderr }, opened.stream);
|
|
722
|
+
started.push(spec.name);
|
|
723
|
+
// Ledgered separately so a held failure can stop exactly this cohort and
|
|
724
|
+
// leave the control plane running (BAPI-1102).
|
|
725
|
+
if (phaseTwoBegun)
|
|
726
|
+
phaseTwoMembers.push(spec.name);
|
|
727
|
+
patchMember(spec.name, { pid: child.pid, state: "running" });
|
|
728
|
+
await persist();
|
|
573
729
|
emitMemberEvent(deps.sinks, {
|
|
574
730
|
member: spec.name,
|
|
575
|
-
kind: "
|
|
576
|
-
timestamp,
|
|
731
|
+
kind: "start",
|
|
732
|
+
timestamp: deps.clock.now().toISOString(),
|
|
577
733
|
logPath: spec.logPath,
|
|
578
|
-
detail:
|
|
734
|
+
detail: describePlaneMember(spec),
|
|
579
735
|
});
|
|
580
|
-
|
|
581
|
-
|
|
736
|
+
// Exit watching is registered BEFORE readiness so a member that dies during
|
|
737
|
+
// its readiness window is reported as a crash rather than as a silent
|
|
738
|
+
// readiness timeout.
|
|
739
|
+
child.on("close", ((code, signal) => {
|
|
740
|
+
const timestamp = deps.clock.now().toISOString();
|
|
741
|
+
patchMember(spec.name, { state: "exited", exitCode: code, exitSignal: signal });
|
|
742
|
+
// Enrolled synchronously so simultaneous exits queue in the order their
|
|
743
|
+
// patches were applied, and tracked so the write is awaited — never
|
|
744
|
+
// abandoned — before the runtime returns.
|
|
745
|
+
trackPersist(persist());
|
|
746
|
+
emitMemberEvent(deps.sinks, {
|
|
582
747
|
member: spec.name,
|
|
748
|
+
kind: "exit",
|
|
583
749
|
timestamp,
|
|
584
|
-
exitCode: code,
|
|
585
|
-
exitSignal: signal,
|
|
586
750
|
logPath: spec.logPath,
|
|
751
|
+
detail: signal !== null ? `signal ${signal}` : `code ${code ?? "unknown"}`,
|
|
587
752
|
});
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
//
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
753
|
+
if (!shuttingDown && !intentionallyStopped.has(spec.name)) {
|
|
754
|
+
reportMemberCrash(deps.sinks, {
|
|
755
|
+
member: spec.name,
|
|
756
|
+
timestamp,
|
|
757
|
+
exitCode: code,
|
|
758
|
+
exitSignal: signal,
|
|
759
|
+
logPath: spec.logPath,
|
|
760
|
+
});
|
|
761
|
+
}
|
|
762
|
+
// NOTHING follows. No respawn, no retry, no backoff timer, and no
|
|
763
|
+
// shutdown of the survivors: one member's crash is one member's crash.
|
|
764
|
+
}));
|
|
765
|
+
child.on("error", (() => {
|
|
766
|
+
/* surfaced through the close handler's exit status */
|
|
767
|
+
}));
|
|
768
|
+
if (spec.readiness) {
|
|
769
|
+
let resolvedInstanceId;
|
|
770
|
+
if (spec.readiness.kind === "process-heartbeat") {
|
|
771
|
+
// Resolved HERE, not on the roster: an executor lane's identity is
|
|
772
|
+
// minted before the spawn, but the local reconciler's is
|
|
773
|
+
// `<hostname>:<pid>` and the pid only exists now. The window between
|
|
774
|
+
// `spawn` returning and this probe is milliseconds, and the child cannot
|
|
775
|
+
// publish inside it — it has to open its database pools and start its
|
|
776
|
+
// scheduler first — but it is a window, and it is documented rather than
|
|
777
|
+
// asserted away. Supplying the identity TO the child would be race-free
|
|
778
|
+
// and would change a durable heartbeat writer, which this work forbids.
|
|
779
|
+
resolvedInstanceId =
|
|
780
|
+
spec.readiness.instanceId ??
|
|
781
|
+
resolveMemberInstanceId(spec.env, child.pid, deps.hostname);
|
|
782
|
+
// Announced BEFORE the wait, not after it. This is the slowest step in
|
|
783
|
+
// bring-up by an order of magnitude, and an operator watching a silent
|
|
784
|
+
// terminal for two minutes has no way to tell waiting from wedged. The
|
|
785
|
+
// wording is the status itself — no colour, no spinner, no symbol — so
|
|
786
|
+
// it survives a pipe, a CI log, and a screen reader identically. The
|
|
787
|
+
// resolved identity is NOT rendered: it is scoping, not news.
|
|
788
|
+
emitMemberEvent(deps.sinks, {
|
|
789
|
+
member: spec.name,
|
|
790
|
+
kind: "start",
|
|
791
|
+
timestamp: deps.clock.now().toISOString(),
|
|
792
|
+
logPath: spec.logPath,
|
|
793
|
+
detail: `Starting — awaiting this member's first post-start ${spec.readiness.component} ` +
|
|
794
|
+
`heartbeat (expected within ${Math.round(spec.readiness.timeoutMs / 1000)}s; watch ${spec.logPath})`,
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
// `memberExited` is read through the closure the `close` handler patches,
|
|
798
|
+
// so a child that dies mid-wait ends the wait immediately instead of
|
|
799
|
+
// burning the remaining budget and reporting a timeout it did not have.
|
|
800
|
+
const outcome = await waitForMemberReady(spec, deps, 250, () => !memberExited(spec.name), {
|
|
801
|
+
baseline,
|
|
802
|
+
instanceId: resolvedInstanceId,
|
|
803
|
+
elapsedMs: baselineElapsedMs,
|
|
622
804
|
});
|
|
805
|
+
if (outcome.kind !== "ready") {
|
|
806
|
+
return finishPartialStart(spec.name, describeReadinessFailure(spec, outcome), {
|
|
807
|
+
log: opened.stream,
|
|
808
|
+
failure: spec.readiness.kind === "process-heartbeat"
|
|
809
|
+
? PLANE_MEMBER_STARTUP_FAILURES.heartbeatReadinessTimeout
|
|
810
|
+
: PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout,
|
|
811
|
+
});
|
|
812
|
+
}
|
|
623
813
|
}
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
814
|
+
patchMember(spec.name, { state: "ready" });
|
|
815
|
+
await persist();
|
|
816
|
+
emitMemberEvent(deps.sinks, {
|
|
817
|
+
member: spec.name,
|
|
818
|
+
kind: "ready",
|
|
819
|
+
timestamp: deps.clock.now().toISOString(),
|
|
820
|
+
logPath: spec.logPath,
|
|
631
821
|
});
|
|
632
|
-
if (outcome.kind !== "ready") {
|
|
633
|
-
return finishPartialStart(spec.name, describeReadinessFailure(spec, outcome), {
|
|
634
|
-
log: opened.stream,
|
|
635
|
-
failure: spec.readiness.kind === "process-heartbeat"
|
|
636
|
-
? PLANE_MEMBER_STARTUP_FAILURES.heartbeatReadinessTimeout
|
|
637
|
-
: PLANE_MEMBER_STARTUP_FAILURES.readinessTimeout,
|
|
638
|
-
});
|
|
639
|
-
}
|
|
640
822
|
}
|
|
641
|
-
|
|
823
|
+
return null;
|
|
824
|
+
};
|
|
825
|
+
// ---- Phase one: the roster this runtime was handed ------------------------
|
|
826
|
+
const firstCohort = await startMembers(roster);
|
|
827
|
+
if (firstCohort !== null)
|
|
828
|
+
return firstCohort;
|
|
829
|
+
/**
|
|
830
|
+
* Phase two: scoped lanes, then the observer (BAPI-1102).
|
|
831
|
+
*
|
|
832
|
+
* The runtime stays alive inside this for as long as run creation takes.
|
|
833
|
+
* Nothing is spawned in the meantime, and the observer is deliberately NOT yet
|
|
834
|
+
* running: a plane with zero executor lanes is exactly the state its executor
|
|
835
|
+
* sweep would alert on.
|
|
836
|
+
*
|
|
837
|
+
* Returns `null` on success. A failure comes back as the terminal result, and
|
|
838
|
+
* the caller reads `heldControlPlane` to tell the two failure shapes apart:
|
|
839
|
+
* one rolled the plane back, the other stopped only what phase two started.
|
|
840
|
+
* Extracted into a function purely so those two can be distinguished at ONE
|
|
841
|
+
* place, instead of at each of the six early returns inside it.
|
|
842
|
+
*/
|
|
843
|
+
const bringUpPhaseTwo = async (twoPhase) => {
|
|
844
|
+
const controlReady = await transitionPlaneManifestLifecycle(repoRoot, deps.planeId, "control-plane-starting", "control-plane-ready", deps.fs);
|
|
845
|
+
if (!controlReady.ok) {
|
|
846
|
+
return finishFailedStart(`the control-plane phase could not be recorded in the manifest (${controlReady.reason}).`);
|
|
847
|
+
}
|
|
848
|
+
manifest = { ...manifest, lifecycle: "control-plane-ready" };
|
|
849
|
+
// Announced only after the transition is DURABLE, so a caller that reads the
|
|
850
|
+
// manifest the instant it sees this marker cannot observe a stale phase.
|
|
851
|
+
//
|
|
852
|
+
// Set BEFORE the announcement: from the instant a caller can act on this
|
|
853
|
+
// marker, every later bring-up failure holds the control plane instead of
|
|
854
|
+
// rolling it back. Setting it after would leave a window in which a failure
|
|
855
|
+
// tore down a plane the caller was already creating a run against.
|
|
856
|
+
phaseTwoBegun = true;
|
|
857
|
+
deps.sinks.stdout(PLANE_CONTROL_READY_MARKER);
|
|
858
|
+
const requested = await awaitLaneScopeRequest(repoRoot, deps, twoPhase);
|
|
859
|
+
if (!requested.ok) {
|
|
860
|
+
// Nothing was spawned for phase two, so there is nothing to roll back
|
|
861
|
+
// beyond the control plane — which `finishPartialStart` handles.
|
|
862
|
+
return finishFailedStart(`plane bring-up did not complete: ${requested.error}.`);
|
|
863
|
+
}
|
|
864
|
+
const laneRoster = twoPhase.buildLaneRoster(requested.scope);
|
|
865
|
+
const lanesStarting = await transitionPlaneManifestLifecycle(repoRoot, deps.planeId, "control-plane-ready", "lanes-starting", deps.fs);
|
|
866
|
+
if (!lanesStarting.ok) {
|
|
867
|
+
return finishFailedStart(`the executor-lane phase could not be recorded in the manifest (${lanesStarting.reason}).`);
|
|
868
|
+
}
|
|
869
|
+
// Re-adopt the manifest the guarded write produced. It carries `epicRunId`
|
|
870
|
+
// and `laneScope`, both written by the OTHER process, and dropping them here
|
|
871
|
+
// would make the next `persist()` publish a record that had forgotten which
|
|
872
|
+
// run this plane serves — the exact binding `plane down` reads to stop it.
|
|
873
|
+
manifest = { ...lanesStarting.manifest, members: manifest.members };
|
|
874
|
+
await appendPlaneManifestMembers(repoRoot, deps.planeId, laneRoster.map(pendingMember), deps.fs);
|
|
875
|
+
manifest = { ...manifest, members: [...manifest.members, ...laneRoster.map(pendingMember)] };
|
|
642
876
|
await persist();
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
877
|
+
const lanes = await startMembers(laneRoster);
|
|
878
|
+
if (lanes !== null)
|
|
879
|
+
return lanes;
|
|
880
|
+
const lanesReady = await transitionPlaneManifestLifecycle(repoRoot, deps.planeId, "lanes-starting", "lanes-ready", deps.fs);
|
|
881
|
+
if (!lanesReady.ok) {
|
|
882
|
+
return finishFailedStart(`executor-lane readiness could not be recorded in the manifest (${lanesReady.reason}).`);
|
|
883
|
+
}
|
|
884
|
+
manifest = { ...manifest, lifecycle: "lanes-ready" };
|
|
885
|
+
// Every lane's own per-instance heartbeat gate has closed. Only now is the
|
|
886
|
+
// observer's first sweep truthful.
|
|
887
|
+
manifest = {
|
|
888
|
+
...manifest,
|
|
889
|
+
members: [...manifest.members, ...twoPhase.observerRoster.map(pendingMember)],
|
|
890
|
+
};
|
|
891
|
+
await appendPlaneManifestMembers(repoRoot, deps.planeId, twoPhase.observerRoster.map(pendingMember), deps.fs);
|
|
892
|
+
await persist();
|
|
893
|
+
const observer = await startMembers(twoPhase.observerRoster);
|
|
894
|
+
if (observer !== null)
|
|
895
|
+
return observer;
|
|
896
|
+
const ready = await transitionPlaneManifestLifecycle(repoRoot, deps.planeId, "lanes-ready", "ready", deps.fs);
|
|
897
|
+
if (!ready.ok) {
|
|
898
|
+
return finishFailedStart(`plane readiness could not be recorded in the manifest (${ready.reason}).`);
|
|
899
|
+
}
|
|
900
|
+
manifest = { ...manifest, lifecycle: "ready" };
|
|
901
|
+
return null;
|
|
902
|
+
};
|
|
903
|
+
let heldFailure = null;
|
|
904
|
+
if (deps.twoPhase) {
|
|
905
|
+
const outcome = await bringUpPhaseTwo(deps.twoPhase);
|
|
906
|
+
if (outcome !== null) {
|
|
907
|
+
// A failure BEFORE the control plane was announced ready is terminal, and
|
|
908
|
+
// has already rolled the whole plane back. One AFTER it is held: the
|
|
909
|
+
// caller may by then have created and approved a run against this plane,
|
|
910
|
+
// and `plane down` stops the manifest-bound run before it terminates
|
|
911
|
+
// processes — so tearing down here would stop a committed run because a
|
|
912
|
+
// local lane failed to start.
|
|
913
|
+
if (outcome.ok || outcome.heldControlPlane !== true)
|
|
914
|
+
return outcome;
|
|
915
|
+
heldFailure = outcome.error;
|
|
916
|
+
await stopPhaseTwoCohort();
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
if (heldFailure !== null) {
|
|
920
|
+
// Recorded before it is announced, for the same reason the ready phases are:
|
|
921
|
+
// a caller that reads the manifest the instant it sees this marker must not
|
|
922
|
+
// observe a phase that says lanes are still starting.
|
|
923
|
+
await recordLanesFailed();
|
|
924
|
+
deps.sinks.stderr("The control plane (server, reconciler) is STILL RUNNING and was not rolled back. " +
|
|
925
|
+
"No executor lane is claiming jobs, and the dead-man observer was not started. " +
|
|
926
|
+
"Inspect it with `plane status`, and wind it down with `plane down` once you have " +
|
|
927
|
+
"decided what to do with the run.");
|
|
928
|
+
// The failure counterpart of PLANE_READY_MARKER, and the reason it exists:
|
|
929
|
+
// the runtime does NOT exit here, so a launcher waiting on readiness or on
|
|
930
|
+
// the process lifetime would otherwise wait for something that never comes.
|
|
931
|
+
deps.sinks.stdout(PLANE_LANES_FAILED_MARKER);
|
|
932
|
+
}
|
|
933
|
+
else {
|
|
934
|
+
// Every member is spawned, alive, and (where it listens) accepting. Only now
|
|
935
|
+
// has the launcher been told anything it may print a success banner over.
|
|
936
|
+
// Emitted EXACTLY ONCE, in both modes, and in two-phase mode only after the
|
|
937
|
+
// observer is up — the banner claims a whole plane, so it must not be printed
|
|
938
|
+
// over one that is still half-built.
|
|
939
|
+
deps.sinks.stdout(PLANE_READY_MARKER);
|
|
649
940
|
}
|
|
650
|
-
// Every member is spawned, alive, and (where it listens) accepting. Only now
|
|
651
|
-
// has the launcher been told anything it may print a success banner over.
|
|
652
|
-
deps.sinks.stdout(PLANE_READY_MARKER);
|
|
653
941
|
// ---- Attended lifetime ---------------------------------------------------
|
|
654
942
|
const exitCode = await new Promise((resolve) => {
|
|
655
943
|
let resolved = false;
|
|
@@ -690,8 +978,33 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
|
|
|
690
978
|
async function finishPartialStart(member, reason, annotate) {
|
|
691
979
|
if (annotate)
|
|
692
980
|
writeMemberStartupFailure(annotate.log, member, annotate.failure);
|
|
693
|
-
|
|
981
|
+
return finishFailedStart(`plane member '${member}' failed to start: ${reason}`);
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* A bring-up failure that is NOT attributable to one member (BAPI-1102).
|
|
985
|
+
*
|
|
986
|
+
* The lifecycle transitions and the phase-two wait can fail with every member
|
|
987
|
+
* healthy — an unreadable manifest, a plane claimed by someone else, a caller
|
|
988
|
+
* that died before requesting lanes. Reporting those through
|
|
989
|
+
* {@link finishPartialStart} would print "plane member 'worker' failed to
|
|
990
|
+
* start" over a worker that started perfectly well and is still running, which
|
|
991
|
+
* sends an operator to the wrong log.
|
|
992
|
+
*
|
|
993
|
+
* Rollback is identical, deliberately: whatever the reason, a bring-up that did
|
|
994
|
+
* not complete must not leave half a plane behind.
|
|
995
|
+
*/
|
|
996
|
+
async function finishFailedStart(message) {
|
|
694
997
|
deps.sinks.stderr(message);
|
|
998
|
+
// BAPI-1102 — once the control plane has been ANNOUNCED ready, a bring-up
|
|
999
|
+
// failure must not tear the plane down. The composed caller has by then gone
|
|
1000
|
+
// and created (and possibly approved) a run against this very plane, and
|
|
1001
|
+
// `plane down` stops the manifest-bound run before it terminates processes:
|
|
1002
|
+
// rolling back here would stop a committed run because a local lane failed
|
|
1003
|
+
// to start. The lane cohort is stopped by the caller of this function and the
|
|
1004
|
+
// server, reconciler, and manifest are left exactly as they are.
|
|
1005
|
+
if (phaseTwoBegun) {
|
|
1006
|
+
return { ok: false, error: message, exitCode: 1, heldControlPlane: true };
|
|
1007
|
+
}
|
|
695
1008
|
if (started.length > 0) {
|
|
696
1009
|
deps.sinks.stderr(`Rolling back ${started.length} member(s) that had already started.`);
|
|
697
1010
|
shuttingDown = true;
|
|
@@ -708,6 +1021,75 @@ export async function runPlaneRuntime(repoRoot, roster, deps) {
|
|
|
708
1021
|
return { ok: false, error: message, exitCode: 1 };
|
|
709
1022
|
}
|
|
710
1023
|
}
|
|
1024
|
+
/**
|
|
1025
|
+
* Wait for the composed caller to record a phase-two request in the manifest.
|
|
1026
|
+
*
|
|
1027
|
+
* BAPI-1102. The runtime and the requester are DIFFERENT PROCESSES — the
|
|
1028
|
+
* requester may even exit before phase two finishes — so the durable manifest is
|
|
1029
|
+
* the only channel between them. Polling it is the honest implementation of that:
|
|
1030
|
+
* there is no shared memory to signal through, and a filesystem watch would add a
|
|
1031
|
+
* platform-dependent dependency to a loop that runs for seconds.
|
|
1032
|
+
*
|
|
1033
|
+
* Bounded, and the bound is not a deadline an operator should ever reach: the
|
|
1034
|
+
* composed caller requests phase two within seconds of the control plane going
|
|
1035
|
+
* ready. It exists so a caller that DIED between the phases leaves a runtime that
|
|
1036
|
+
* eventually reports why, rather than one polling forever.
|
|
1037
|
+
*
|
|
1038
|
+
* Every exit from here is a HELD failure, not a rollback: the control plane has
|
|
1039
|
+
* already been announced ready, so by this point the caller may have created and
|
|
1040
|
+
* approved a run against it. The runtime reports, stops nothing it did not start,
|
|
1041
|
+
* and goes on supervising the server and reconciler until `plane down`.
|
|
1042
|
+
*
|
|
1043
|
+
* A manifest that stops being readable, or whose identity stops matching, ends
|
|
1044
|
+
* the wait immediately — both mean this runtime no longer owns the plane, and
|
|
1045
|
+
* continuing to wait would be waiting for an instruction that can never arrive.
|
|
1046
|
+
* Those two are held as well, deliberately: a runtime that has lost its manifest
|
|
1047
|
+
* cannot safely run `shutdownPlane`, because the manifest it would read and act
|
|
1048
|
+
* on now describes somebody else's plane. It reports and leaves its own members
|
|
1049
|
+
* running for an operator to resolve.
|
|
1050
|
+
*/
|
|
1051
|
+
async function awaitLaneScopeRequest(repoRoot, deps, twoPhase) {
|
|
1052
|
+
const pollMs = twoPhase.pollIntervalMs ?? PLANE_PHASE_REQUEST_POLL_MS;
|
|
1053
|
+
const timeoutMs = twoPhase.waitTimeoutMs ?? PLANE_PHASE_REQUEST_TIMEOUT_MS;
|
|
1054
|
+
const sleep = twoPhase.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
1055
|
+
const startedAt = deps.clock.now().getTime();
|
|
1056
|
+
for (;;) {
|
|
1057
|
+
const read = await readPlaneManifest(repoRoot, deps.fs);
|
|
1058
|
+
if (read.kind !== "valid") {
|
|
1059
|
+
return {
|
|
1060
|
+
ok: false,
|
|
1061
|
+
error: "the plane manifest became unreadable while waiting for executor lanes",
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
if (read.manifest.planeId !== deps.planeId) {
|
|
1065
|
+
return {
|
|
1066
|
+
ok: false,
|
|
1067
|
+
error: "the plane manifest was claimed by a different plane while waiting for executor lanes",
|
|
1068
|
+
};
|
|
1069
|
+
}
|
|
1070
|
+
const scope = read.manifest.laneScope;
|
|
1071
|
+
if (scope !== undefined)
|
|
1072
|
+
return { ok: true, scope };
|
|
1073
|
+
// A phase that has already moved past `control-plane-ready` with no scope
|
|
1074
|
+
// recorded is incoherent — nothing else writes those phases — so it is
|
|
1075
|
+
// reported rather than waited out.
|
|
1076
|
+
const phase = planeManifestLifecycle(read.manifest);
|
|
1077
|
+
if (phase !== "control-plane-ready" && phase !== "control-plane-starting") {
|
|
1078
|
+
return {
|
|
1079
|
+
ok: false,
|
|
1080
|
+
error: `the plane advanced to ${phase} without recording an executor claim scope`,
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
if (deps.clock.now().getTime() - startedAt >= timeoutMs) {
|
|
1084
|
+
return {
|
|
1085
|
+
ok: false,
|
|
1086
|
+
error: `no executor claim scope was requested within ${Math.round(timeoutMs / 1000)}s ` +
|
|
1087
|
+
"of the control plane becoming ready",
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
await sleep(pollMs);
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
711
1093
|
/**
|
|
712
1094
|
* The pre-spawn baseline for one heartbeat-gated member, or a terminal reason.
|
|
713
1095
|
*
|