omp-conductor 0.15.2 → 0.15.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.ts +7 -2
- package/src/daemon.ts +146 -0
- package/src/gitops.ts +98 -16
- package/src/lifecycle.ts +149 -29
- package/src/omp.ts +34 -0
- package/src/setup-host.ts +35 -7
- package/src/setup-wizard.ts +9 -4
- package/src/types.ts +1 -0
- package/src/upgrade.ts +116 -5
- package/src/worker.ts +41 -15
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
package/src/cli.ts
CHANGED
|
@@ -155,7 +155,9 @@ usage:
|
|
|
155
155
|
that hits --timeout restarts nothing and leaves dispatch paused. On boot
|
|
156
156
|
the new process salvages dirty live worktrees before orphaning those
|
|
157
157
|
rows — see README "Deploying a new package onto a busy fleet". Goes
|
|
158
|
-
through systemctl when the unit owns the live pid
|
|
158
|
+
through systemctl when the unit owns the live pid; a failed
|
|
159
|
+
installed unit is reset and restarted through systemd, never
|
|
160
|
+
replaced by an unmanaged daemon.
|
|
159
161
|
status layered fleet report: dispatch (running|paused|stopped), ticks and
|
|
160
162
|
next due time, pane, herdr, Telegram bot/API health, daemon, caps
|
|
161
163
|
and active runs.
|
|
@@ -682,7 +684,10 @@ try {
|
|
|
682
684
|
// Inherit the running daemon's port and project: a restart that quietly
|
|
683
685
|
// moved to the default port would leave every existing health check
|
|
684
686
|
// pointing at nothing. When the unit owns the live pid, restartDaemon
|
|
685
|
-
// goes through systemctl so the replacement stays supervised
|
|
687
|
+
// goes through systemctl so the replacement stays supervised; a failed
|
|
688
|
+
// installed unit is reset and started through systemd the same way and
|
|
689
|
+
// is never replaced by an unmanaged daemon. Success is only reported
|
|
690
|
+
// after the manager is proven to own the reported pid.
|
|
686
691
|
const { previous, record, via } = await restartDaemon({
|
|
687
692
|
port: portFlag(argv),
|
|
688
693
|
project: projectFlag,
|
package/src/daemon.ts
CHANGED
|
@@ -171,11 +171,25 @@ export interface DaemonOpts {
|
|
|
171
171
|
* model or caps — `handleIssue` destructures them at dispatch time. The rest
|
|
172
172
|
* are resolved once at startup so a tick never re-reads config mid-flight and
|
|
173
173
|
* changes its own limits underneath itself. */
|
|
174
|
+
export interface DrainSignal {
|
|
175
|
+
draining: boolean;
|
|
176
|
+
}
|
|
177
|
+
|
|
174
178
|
interface Deps {
|
|
175
179
|
project: ProjectConfig;
|
|
176
180
|
caps: Caps;
|
|
177
181
|
tracker: Tracker;
|
|
178
182
|
store: Store;
|
|
183
|
+
/**
|
|
184
|
+
* Set true the moment a daemon stop (SIGTERM/SIGINT) begins. The run loop
|
|
185
|
+
* only sees `stopping` between whole ticks, so a pass that was already in
|
|
186
|
+
* flight when the stop landed must re-check this flag itself — before it
|
|
187
|
+
* claims and before it launches — or it creates exactly the work the
|
|
188
|
+
* shutdown is about to wait for and then lose to the stop timeout (#374).
|
|
189
|
+
* Optional only so tests that never exercise shutdown can omit it;
|
|
190
|
+
* `runDaemon` always wires the real one.
|
|
191
|
+
*/
|
|
192
|
+
drain?: DrainSignal;
|
|
179
193
|
/** False after a live config reload fails; autonomous delivery then holds
|
|
180
194
|
* fail-closed until a later tick validates the config again. */
|
|
181
195
|
deliveryPolicyValid?: boolean;
|
|
@@ -444,6 +458,40 @@ export function pauseProvenance(
|
|
|
444
458
|
}
|
|
445
459
|
}
|
|
446
460
|
|
|
461
|
+
/**
|
|
462
|
+
* One pause sentinel read as a single identity: who set it, why, and the
|
|
463
|
+
* creation instant, all from the SAME file that was selected. Unlike pairing
|
|
464
|
+
* {@link pauseProvenance} with {@link pausedAt} — which can describe
|
|
465
|
+
* different files when a project pause coexists with the legacy global
|
|
466
|
+
* sentinel, letting a stale global timestamp mask a recreated project pause —
|
|
467
|
+
* this reads provenance and timestamp from one sentinel, so a caller can prove
|
|
468
|
+
* "the pause I set still exists" instead of "some pause with the same labels
|
|
469
|
+
* still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
|
|
470
|
+
*/
|
|
471
|
+
export function pauseInstance(
|
|
472
|
+
project?: string,
|
|
473
|
+
): { source: string; reason?: string; since: number } | undefined {
|
|
474
|
+
const paths =
|
|
475
|
+
project === undefined
|
|
476
|
+
? [pausedPath()]
|
|
477
|
+
: [pausedPath(project), pausedPath()];
|
|
478
|
+
const path = paths.find((candidate) => existsSync(candidate));
|
|
479
|
+
if (path === undefined) return undefined;
|
|
480
|
+
try {
|
|
481
|
+
const [line1, line2] = readFileSync(path, "utf8").split("\n");
|
|
482
|
+
const since = Date.parse(line1?.trim() ?? "");
|
|
483
|
+
if (!Number.isFinite(since)) return undefined;
|
|
484
|
+
if (line2 === undefined) return undefined;
|
|
485
|
+
const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
|
|
486
|
+
if (match === null) return undefined;
|
|
487
|
+
const source = match[1]!;
|
|
488
|
+
const reason = match[2];
|
|
489
|
+
return { source, since, ...(reason === undefined ? {} : { reason }) };
|
|
490
|
+
} catch {
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
447
495
|
export function setPaused(
|
|
448
496
|
v: boolean,
|
|
449
497
|
why?: { source: string; reason?: string },
|
|
@@ -1309,7 +1357,65 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1309
1357
|
return true;
|
|
1310
1358
|
};
|
|
1311
1359
|
|
|
1360
|
+
/**
|
|
1361
|
+
* The daemon-wide stop fence can land after the claim but before `runWorker`
|
|
1362
|
+
* exposes its controller — the same window `settleStopBeforeSession` closes
|
|
1363
|
+
* for a per-run operator stop, entered from the shutdown path instead of the
|
|
1364
|
+
* pause control. Settle identically (close the claim, never launch) so the
|
|
1365
|
+
* drain waits only for workers admitted before the stop request, and the
|
|
1366
|
+
* shutdown cannot extend its own workload (#374).
|
|
1367
|
+
*/
|
|
1368
|
+
const settleDrainBeforeSession = async (): Promise<boolean> => {
|
|
1369
|
+
if (d.drain?.draining !== true || run === undefined || workerSessionInstalled) return false;
|
|
1370
|
+
turnLimit?.close();
|
|
1371
|
+
turnLimit = undefined;
|
|
1372
|
+
const settlement =
|
|
1373
|
+
worktreePath === undefined
|
|
1374
|
+
? undefined
|
|
1375
|
+
: await settleWorktree({
|
|
1376
|
+
issue,
|
|
1377
|
+
attempt,
|
|
1378
|
+
ending: "daemon shutdown began while the run was being prepared",
|
|
1379
|
+
worktree: worktreePath,
|
|
1380
|
+
branch,
|
|
1381
|
+
publish,
|
|
1382
|
+
tree: "remove",
|
|
1383
|
+
mirrorPath,
|
|
1384
|
+
});
|
|
1385
|
+
recordOperatorStop(store, {
|
|
1386
|
+
project: project.name,
|
|
1387
|
+
issue,
|
|
1388
|
+
runId: run.id,
|
|
1389
|
+
inProgress,
|
|
1390
|
+
reason: "daemon shutdown began after the claim",
|
|
1391
|
+
patch: {
|
|
1392
|
+
endedAt: Date.now(),
|
|
1393
|
+
turns: run.turns,
|
|
1394
|
+
spendUsd: run.spendUsd,
|
|
1395
|
+
worktree: worktreePath ?? run.worktree,
|
|
1396
|
+
report: [
|
|
1397
|
+
"Daemon shutdown began while this run was still being prepared; the worker",
|
|
1398
|
+
"never started and the claim was closed so the shutdown could complete.",
|
|
1399
|
+
...(settlement?.lines ?? []),
|
|
1400
|
+
].join("\n"),
|
|
1401
|
+
...settlement?.patch,
|
|
1402
|
+
},
|
|
1403
|
+
});
|
|
1404
|
+
log(`#${issue} not launched: daemon shutdown began after the claim`);
|
|
1405
|
+
return true;
|
|
1406
|
+
};
|
|
1407
|
+
|
|
1312
1408
|
try {
|
|
1409
|
+
// The claim-side of the stop fence (#374): the run row is the boundary the
|
|
1410
|
+
// shutdown drain waits on, so the claim itself refuses once the daemon is
|
|
1411
|
+
// draining. The tick re-checks before the dispatch tail; this covers the
|
|
1412
|
+
// same fence for any caller that reaches `handleIssue` without one, and
|
|
1413
|
+
// makes "no post-stop run rows" a property of the claim, not of its caller.
|
|
1414
|
+
if (d.drain?.draining === true) {
|
|
1415
|
+
log(`#${issue} not claimed: daemon is draining`);
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1313
1419
|
// Claim on the STORE first, before anything that can fail. The run row —
|
|
1314
1420
|
// not the label — is the crash-safe guard against double dispatch: rows
|
|
1315
1421
|
// are local, written before any network call, and the startup orphan
|
|
@@ -1351,6 +1457,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1351
1457
|
turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
|
|
1352
1458
|
workerControl = d.workerControls.open(project.name, issue, runId);
|
|
1353
1459
|
if (await settleStopBeforeSession()) return;
|
|
1460
|
+
if (await settleDrainBeforeSession()) return;
|
|
1354
1461
|
|
|
1355
1462
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
1356
1463
|
// an existing path, so a retry — or a tree kept from a failed attempt — has
|
|
@@ -1360,6 +1467,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1360
1467
|
// would cost a second network fetch per attempt.
|
|
1361
1468
|
await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
|
|
1362
1469
|
if (await settleStopBeforeSession()) return;
|
|
1470
|
+
if (await settleDrainBeforeSession()) return;
|
|
1363
1471
|
const provisioned = await addRunRepo(
|
|
1364
1472
|
r.repo,
|
|
1365
1473
|
project.mirrorRoot,
|
|
@@ -1370,6 +1478,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1370
1478
|
worktreePath = provisioned.path;
|
|
1371
1479
|
runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
|
|
1372
1480
|
if (await settleStopBeforeSession()) return;
|
|
1481
|
+
if (await settleDrainBeforeSession()) return;
|
|
1373
1482
|
|
|
1374
1483
|
// The SDK names the transcript itself, so the daemon supplies the parent
|
|
1375
1484
|
// directory and learns the real path back from the result. Inventing one
|
|
@@ -1409,6 +1518,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1409
1518
|
{ ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
|
|
1410
1519
|
);
|
|
1411
1520
|
if (await settleStopBeforeSession()) return;
|
|
1521
|
+
if (await settleDrainBeforeSession()) return;
|
|
1412
1522
|
|
|
1413
1523
|
store.updateRun(runId, { worktree: worktreePath, state: "running" });
|
|
1414
1524
|
|
|
@@ -1425,6 +1535,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1425
1535
|
: {}),
|
|
1426
1536
|
});
|
|
1427
1537
|
if (await settleStopBeforeSession()) return;
|
|
1538
|
+
if (await settleDrainBeforeSession()) return;
|
|
1428
1539
|
|
|
1429
1540
|
const repoSlug = githubRepo(r.repo.cloneUrl);
|
|
1430
1541
|
|
|
@@ -1470,6 +1581,12 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
|
|
|
1470
1581
|
// and a path written at completion is a path nobody can follow live. The
|
|
1471
1582
|
// completion-time update below writes the same value again, harmlessly.
|
|
1472
1583
|
onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
|
|
1584
|
+
// The last fence (#374): every pre-launch settle check above has
|
|
1585
|
+
// passed, but the stop can still land while the session socket is
|
|
1586
|
+
// binding inside `createSession`. This gate is re-checked there,
|
|
1587
|
+
// immediately before the child spawn, and closes the run as stopped
|
|
1588
|
+
// instead of launching a worker the shutdown would have to wait for.
|
|
1589
|
+
maySpawn: () => d.drain?.draining !== true,
|
|
1473
1590
|
}, d.workerDeps);
|
|
1474
1591
|
} finally {
|
|
1475
1592
|
// This is the authoritative settlement edge for `extend`: close before
|
|
@@ -3273,6 +3390,21 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
3273
3390
|
}
|
|
3274
3391
|
|
|
3275
3392
|
const pass = await admitCandidates(d, routed, slots);
|
|
3393
|
+
|
|
3394
|
+
// The stop fence (#374): the run loop only re-reads `stopping` between
|
|
3395
|
+
// whole ticks, so a pass already in flight when SIGTERM/SIGINT landed must
|
|
3396
|
+
// re-check the drain signal here — after candidate admission, before any
|
|
3397
|
+
// claim or launch — or the shutdown path admits exactly the work it is
|
|
3398
|
+
// about to wait for and then loses to the stop timeout. The candidates keep
|
|
3399
|
+
// their queue labels; the next daemon start re-dispatches them.
|
|
3400
|
+
if (d.drain?.draining === true) {
|
|
3401
|
+
recordDispatch(0, [
|
|
3402
|
+
...routingHolds,
|
|
3403
|
+
...pass.holds,
|
|
3404
|
+
...pass.admitted.map((a) => ({ issue: a.r.issue.number, reason: "shutting-down" as const })),
|
|
3405
|
+
]);
|
|
3406
|
+
return;
|
|
3407
|
+
}
|
|
3276
3408
|
recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
|
|
3277
3409
|
|
|
3278
3410
|
if (pass.admitted.length === 0) return;
|
|
@@ -4553,6 +4685,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4553
4685
|
const alive = livingDaemon();
|
|
4554
4686
|
const runtimes: ProjectRuntime[] = [];
|
|
4555
4687
|
|
|
4688
|
+
// The stop fence (#374): one signal shared by every project's tick.
|
|
4689
|
+
// `stopping` gates the run loop between whole ticks; `drain.draining` is
|
|
4690
|
+
// the same event visible *inside* a tick already in flight, so a pass that
|
|
4691
|
+
// was mid-admission when the stop landed re-checks it before claiming or
|
|
4692
|
+
// launching. Created here, before the loop, so `stop` below and every
|
|
4693
|
+
// project's `Deps` reference the same object.
|
|
4694
|
+
const drain: DrainSignal = { draining: false };
|
|
4695
|
+
|
|
4556
4696
|
log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`);
|
|
4557
4697
|
log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
|
|
4558
4698
|
|
|
@@ -4675,6 +4815,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4675
4815
|
caps,
|
|
4676
4816
|
tracker,
|
|
4677
4817
|
store,
|
|
4818
|
+
drain,
|
|
4678
4819
|
deliveryPolicyValid: false,
|
|
4679
4820
|
usage,
|
|
4680
4821
|
escalate: (event) => escalator.escalate(event),
|
|
@@ -4754,6 +4895,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
4754
4895
|
const stop = (): void => {
|
|
4755
4896
|
if (stopping) return;
|
|
4756
4897
|
stopping = true;
|
|
4898
|
+
// Close the admission fence before the loop is told, so a tick already in
|
|
4899
|
+
// flight — parked at an await when the signal landed — sees `draining`
|
|
4900
|
+
// when it resumes and cannot create the work this shutdown is about to
|
|
4901
|
+
// wait for (#374).
|
|
4902
|
+
drain.draining = true;
|
|
4757
4903
|
log("shutting down after active workers finish");
|
|
4758
4904
|
wake?.();
|
|
4759
4905
|
};
|
package/src/gitops.ts
CHANGED
|
@@ -112,17 +112,30 @@ export function repoSlugFor(repo: RepoTarget): string {
|
|
|
112
112
|
* hop that touches the network happens here, so the dispatcher observes the
|
|
113
113
|
* exact sha it recorded.
|
|
114
114
|
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
115
|
+
* The run-repo hop carries **no leading `+`** — that single character is the
|
|
116
|
+
* difference between "advance the mirror's copy of this branch" and "make the
|
|
117
|
+
* mirror's copy whatever the run says it is", and a run repo is the one place
|
|
118
|
+
* in this system that model-executed code fully controls. The push carries no
|
|
119
|
+
* `--force` and no `--force-with-lease`: a lease is still a force, and a
|
|
120
|
+
* rejected push is a decision for a human, not a retry.
|
|
121
|
+
*
|
|
122
|
+
* The one exception is narrower than it looks. The mirror's copy of a run
|
|
123
|
+
* branch can be stale in a way that is not the run's fault:
|
|
124
|
+
* `conductor_pr_update_branch` moves — and on a rewritten history, replaces —
|
|
125
|
+
* the remote branch server-side after a run's last push, so a continuation
|
|
126
|
+
* that incorporated the reviewed baseline can propose a head the *live remote*
|
|
127
|
+
* is an ancestor of while the mirror's old copy is not. That is a legitimate
|
|
128
|
+
* fast-forward, not a divergence, so when the plain copy is refused this push
|
|
129
|
+
* reconciles the mirror's remote-tracking ref of the run branch with the live
|
|
130
|
+
* remote and asks git whether the live head is an ancestor of the proposed
|
|
131
|
+
* head. Only that proof lets the mirror's copy be refreshed, and the push hop
|
|
132
|
+
* still re-checks the ancestor relation against the live remote itself.
|
|
133
|
+
* Everything else is refused carrying the exact proposed and live SHAs so a
|
|
134
|
+
* real divergence is diagnosable.
|
|
122
135
|
*
|
|
123
136
|
* A rejection — non-fast-forward, protected ref, unauthorised — comes back as a
|
|
124
|
-
* value carrying
|
|
125
|
-
*
|
|
137
|
+
* value carrying the reason, because the caller settles the run `failed` on it
|
|
138
|
+
* and a paraphrase is worthless in that report.
|
|
126
139
|
*/
|
|
127
140
|
export async function pushRunBranch(
|
|
128
141
|
project: Pick<ProjectConfig, "mirrorRoot">,
|
|
@@ -131,24 +144,93 @@ export async function pushRunBranch(
|
|
|
131
144
|
): Promise<PushOutcome> {
|
|
132
145
|
const mirror = mirrorPath(project, run.repo);
|
|
133
146
|
const ref = `refs/heads/${run.branch}`;
|
|
147
|
+
const tracked = `refs/remotes/origin/${run.branch}`;
|
|
134
148
|
const env = credentialedEnv();
|
|
135
149
|
|
|
150
|
+
// Copy the run's branch into the mirror, fast-forward only.
|
|
136
151
|
const fetched = await exec(["git", "-C", mirror, "fetch", "--no-tags", run.runRepoPath, `${ref}:${ref}`], { env });
|
|
137
|
-
|
|
138
|
-
|
|
152
|
+
|
|
153
|
+
// The proposed head is read from the run repo itself: when the copy above
|
|
154
|
+
// was refused, the mirror's copy of the branch is exactly the stale one.
|
|
155
|
+
const proposedRun = await exec(["git", "-C", run.runRepoPath, "rev-parse", ref], { env });
|
|
156
|
+
if (proposedRun.code !== 0) {
|
|
157
|
+
return { ok: false, stderr: scrubUserinfo(proposedRun.stderr.trim() || `git rev-parse ${ref} exited ${String(proposedRun.code)}`) };
|
|
158
|
+
}
|
|
159
|
+
const proposed = proposedRun.stdout.trim();
|
|
160
|
+
|
|
161
|
+
// Reconcile with the live remote before enforcing anything: the ancestry
|
|
162
|
+
// that decides a fast-forward is the live remote's, not the mirror's copy's
|
|
163
|
+
// (see the docstring above). ls-remote answers "is the branch published at
|
|
164
|
+
// all" and "what does GitHub hold" in one call.
|
|
165
|
+
const liveListed = await exec(["git", "-C", mirror, "ls-remote", "origin", ref], { env });
|
|
166
|
+
if (liveListed.code !== 0) {
|
|
167
|
+
return { ok: false, stderr: scrubUserinfo(liveListed.stderr.trim() || `git ls-remote origin ${ref} exited ${String(liveListed.code)}`) };
|
|
168
|
+
}
|
|
169
|
+
const live = liveListed.stdout
|
|
170
|
+
.split("\n")
|
|
171
|
+
.map((line) => line.trimEnd())
|
|
172
|
+
.find((line) => line.endsWith(`\t${ref}`))
|
|
173
|
+
?.split(/\s+/, 1)[0];
|
|
174
|
+
|
|
175
|
+
if (live !== undefined && live !== proposed) {
|
|
176
|
+
// The branch is published and the run proposes something different. Pull
|
|
177
|
+
// the live branch's history into the mirror once (which also keeps the
|
|
178
|
+
// tracked ref a reattach reads fresh), then prove the fast-forward.
|
|
179
|
+
const reconciled = await exec(["git", "-C", mirror, "fetch", "--no-tags", "origin", `+${ref}:${tracked}`], { env });
|
|
180
|
+
if (reconciled.code !== 0) {
|
|
181
|
+
return { ok: false, stderr: scrubUserinfo(reconciled.stderr.trim() || reconciled.stdout.trim() || `git fetch origin exited ${String(reconciled.code)}`) };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const isAncestor = await exec(["git", "-C", mirror, "merge-base", "--is-ancestor", live, proposed], { env });
|
|
185
|
+
if (isAncestor.code === 128) {
|
|
186
|
+
// git could not perform the check at all (an object it was asked to
|
|
187
|
+
// resolve is missing, not merely unrelated). That is a failed
|
|
188
|
+
// verification, not a divergence verdict: refuse with git's own words,
|
|
189
|
+
// still naming both SHAs so the report carries the mismatch.
|
|
190
|
+
return {
|
|
191
|
+
ok: false,
|
|
192
|
+
stderr: scrubUserinfo(
|
|
193
|
+
isAncestor.stderr.trim() ||
|
|
194
|
+
isAncestor.stdout.trim() ||
|
|
195
|
+
`git merge-base --is-ancestor ${live} ${proposed} exited ${String(isAncestor.code)}`,
|
|
196
|
+
),
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
if (isAncestor.code !== 0) {
|
|
200
|
+
// A real divergence: no fast-forward exists, and the mirror is left
|
|
201
|
+
// exactly where it was. Both SHAs are named so the mismatch is
|
|
202
|
+
// diagnosable instead of a bare "non-fast-forward".
|
|
203
|
+
return {
|
|
204
|
+
ok: false,
|
|
205
|
+
stderr:
|
|
206
|
+
`refusing non-fast-forward push of ${run.branch}: the live remote tip ${live} is not an ancestor of ` +
|
|
207
|
+
`the proposed head ${proposed}. Fetch the live branch and rebase or merge it before pushing again.`,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
139
210
|
}
|
|
140
211
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
212
|
+
if (fetched.code !== 0 && live === undefined) {
|
|
213
|
+
// The mirror refused the plain copy and there is no published branch to
|
|
214
|
+
// validate the run's head against: a rewritten branch that was never
|
|
215
|
+
// published must not rewrite the mirror either. Refuse with git's words.
|
|
216
|
+
return { ok: false, stderr: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`) };
|
|
217
|
+
}
|
|
218
|
+
if (fetched.code !== 0) {
|
|
219
|
+
// The proposed head is a legitimate fast-forward over the live remote (it
|
|
220
|
+
// equals the live head or the ancestor test above passed), so the only
|
|
221
|
+
// thing the plain copy refused on was the mirror's own stale copy.
|
|
222
|
+
// Refresh it; the push below still re-checks against the live remote.
|
|
223
|
+
const refreshed = await exec(["git", "-C", mirror, "fetch", "--no-tags", run.runRepoPath, `+${ref}:${ref}`], { env });
|
|
224
|
+
if (refreshed.code !== 0) {
|
|
225
|
+
return { ok: false, stderr: scrubUserinfo(refreshed.stderr.trim() || refreshed.stdout.trim() || `git fetch exited ${String(refreshed.code)}`) };
|
|
226
|
+
}
|
|
144
227
|
}
|
|
145
|
-
const sha = resolved.stdout.trim();
|
|
146
228
|
|
|
147
229
|
const pushed = await exec(["git", "-C", mirror, "push", "origin", `${ref}:${ref}`], { env });
|
|
148
230
|
if (pushed.code !== 0) {
|
|
149
231
|
return { ok: false, stderr: scrubUserinfo(pushed.stderr.trim() || pushed.stdout.trim() || `git push exited ${String(pushed.code)}`) };
|
|
150
232
|
}
|
|
151
|
-
return { ok: true, sha };
|
|
233
|
+
return { ok: true, sha: proposed };
|
|
152
234
|
}
|
|
153
235
|
|
|
154
236
|
/**
|
package/src/lifecycle.ts
CHANGED
|
@@ -364,9 +364,19 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
|
|
|
364
364
|
* Same ownership rule as {@link stopDaemon}: when the unit owns the live pid,
|
|
365
365
|
* `systemctl restart` is the *only* path — a failed manager call is terminal,
|
|
366
366
|
* never a fallthrough to raw signals. An *unanswered* ownership query is also
|
|
367
|
-
* terminal: "dbus blipped" is not "no unit".
|
|
368
|
-
*
|
|
369
|
-
* a
|
|
367
|
+
* terminal: "dbus blipped" is not "no unit". A *failed* installed unit is the
|
|
368
|
+
* third non-fallthrough: it must be restored through the manager
|
|
369
|
+
* (`reset-failed` + `start`), because a detached CLI daemon beside a failed
|
|
370
|
+
* unit is exactly the "restarted, but the fleet is still down" lie this
|
|
371
|
+
* module exists to prevent (#376). Falling back to stop+start is reserved for
|
|
372
|
+
* a confirmed hand-started daemon (no systemd, inactive unit, or a unit whose
|
|
373
|
+
* MainPID is someone else).
|
|
374
|
+
*
|
|
375
|
+
* Success is reported only after the service manager is re-probed and proven
|
|
376
|
+
* to own the daemon: the unit's MainPID must equal the daemon record pid AND
|
|
377
|
+
* `/healthz` must answer. The manager call returning is not that proof — for
|
|
378
|
+
* `Type=simple` the unit is active as soon as the process is forked, before
|
|
379
|
+
* the daemon has bound its port.
|
|
370
380
|
*
|
|
371
381
|
* Returns the record of the process that is now answering `/healthz`.
|
|
372
382
|
*/
|
|
@@ -379,6 +389,16 @@ export async function restartDaemon(
|
|
|
379
389
|
throw new Error(ownershipUnknown("restart", ownership.reason));
|
|
380
390
|
}
|
|
381
391
|
|
|
392
|
+
if (ownership.kind === "failed") {
|
|
393
|
+
// An installed unit in failed state: reset the failure record and let
|
|
394
|
+
// systemd start the service. Never fall through to stopDaemon() +
|
|
395
|
+
// startDaemon() — that leaves the unit failed while handing the operator
|
|
396
|
+
// a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
|
|
397
|
+
// Any manager refusal is terminal; so is unproven ownership afterwards.
|
|
398
|
+
const record = await restoreFailedUnit(o.timeoutMs);
|
|
399
|
+
return { previous, record, via: "systemctl" };
|
|
400
|
+
}
|
|
401
|
+
|
|
382
402
|
const unitOwns =
|
|
383
403
|
ownership.kind === "active" && (previous === undefined || previous.pid === ownership.pid);
|
|
384
404
|
|
|
@@ -393,26 +413,95 @@ export async function restartDaemon(
|
|
|
393
413
|
// systemctl restart returns once the new MainPID is up; the pidfile is
|
|
394
414
|
// written by the daemon itself on boot, so wait for that rather than
|
|
395
415
|
// inventing a record from the unit alone.
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
const rec = livingDaemon();
|
|
399
|
-
if (rec !== undefined) {
|
|
400
|
-
const health = await healthCheck(rec.port);
|
|
401
|
-
if (health.ok) return { previous, record: rec, via: "systemctl" };
|
|
402
|
-
}
|
|
403
|
-
if (Date.now() >= deadline) break;
|
|
404
|
-
await sleep(READY_POLL_MS);
|
|
405
|
-
}
|
|
406
|
-
throw new Error(
|
|
407
|
-
`systemctl restart ${SYSTEMD_UNIT} returned, but the daemon never answered /healthz`,
|
|
408
|
-
);
|
|
416
|
+
const record = await waitForOwnedDaemon("restart", o.timeoutMs);
|
|
417
|
+
return { previous, record, via: "systemctl" };
|
|
409
418
|
}
|
|
410
419
|
|
|
420
|
+
// Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
|
|
421
|
+
// is somebody else. The detached CLI daemon is the only path left.
|
|
411
422
|
await stopDaemon({ timeoutMs: o.timeoutMs });
|
|
412
423
|
const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
|
|
413
424
|
return { previous, record, via: "cli" };
|
|
414
425
|
}
|
|
415
426
|
|
|
427
|
+
/**
|
|
428
|
+
* Restores a failed installed unit through the manager: `reset-failed` then
|
|
429
|
+
* `start`, and proves the result the same way any managed restart is proven
|
|
430
|
+
* (see {@link waitForOwnedDaemon}). Refusal or unproven ownership is terminal
|
|
431
|
+
* — never a fallthrough to the detached CLI daemon.
|
|
432
|
+
*/
|
|
433
|
+
async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
|
|
434
|
+
const reset = systemctl(["reset-failed", SYSTEMD_UNIT]);
|
|
435
|
+
if (!reset.ok) {
|
|
436
|
+
throw new Error(systemctlFailure("reset-failed", reset));
|
|
437
|
+
}
|
|
438
|
+
const started = systemctl(["start", SYSTEMD_UNIT]);
|
|
439
|
+
if (!started.ok) {
|
|
440
|
+
throw new Error(systemctlFailure("start", started));
|
|
441
|
+
}
|
|
442
|
+
return await waitForOwnedDaemon("start", timeoutMs);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Waits for the daemon a manager call was just made to boot, and refuses to
|
|
447
|
+
* report success until the service manager is proven to own it: the unit's
|
|
448
|
+
* MainPID must equal the daemon record pid AND `/healthz` must answer. A unit
|
|
449
|
+
* that went back to `failed` is a confirmed negative and fails fast — waiting
|
|
450
|
+
* cannot un-fail it. Any other unproven state fails at the deadline with a
|
|
451
|
+
* diagnostic that names the mismatch, never a bare "not ready".
|
|
452
|
+
*/
|
|
453
|
+
async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number): Promise<DaemonRecord> {
|
|
454
|
+
const via = `systemctl ${verb} ${SYSTEMD_UNIT} returned`;
|
|
455
|
+
const inspect = `systemctl status ${SYSTEMD_UNIT}`;
|
|
456
|
+
const deadline = Date.now() + (timeoutMs ?? READY_TIMEOUT_MS);
|
|
457
|
+
for (;;) {
|
|
458
|
+
const rec = livingDaemon();
|
|
459
|
+
const ownership = probeUnit();
|
|
460
|
+
if (rec !== undefined && ownership.kind === "active" && ownership.pid === rec.pid) {
|
|
461
|
+
const health = await healthCheck(rec.port);
|
|
462
|
+
if (health.ok) return rec;
|
|
463
|
+
} else if (ownership.kind === "failed") {
|
|
464
|
+
// A confirmed negative: the start did not take and the unit is failed
|
|
465
|
+
// again. Waiting longer cannot un-fail it.
|
|
466
|
+
throw new Error(
|
|
467
|
+
`${via}, but the unit went back to failed — the daemon is NOT running; check \`${inspect}\``,
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
if (Date.now() >= deadline) break;
|
|
471
|
+
await sleep(READY_POLL_MS);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// Failed to prove ownership. Name what is wrong rather than a bare timeout,
|
|
475
|
+
// so the operator knows whether to fix the unit or hunt the foreign pid.
|
|
476
|
+
const rec = livingDaemon();
|
|
477
|
+
const ownership = probeUnit();
|
|
478
|
+
if (rec === undefined) {
|
|
479
|
+
throw new Error(
|
|
480
|
+
`${via}, but no live daemon record appeared — the daemon did not come back; check \`${inspect}\``,
|
|
481
|
+
);
|
|
482
|
+
}
|
|
483
|
+
if (ownership.kind === "active") {
|
|
484
|
+
if (ownership.pid === rec.pid) {
|
|
485
|
+
throw new Error(
|
|
486
|
+
`${via}, but the daemon never answered /healthz on :${rec.port} — ` +
|
|
487
|
+
`the service manager owns pid ${rec.pid}, but it is not serving; ` +
|
|
488
|
+
`check \`${inspect}\` and ${rec.logFile}`,
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
throw new Error(
|
|
492
|
+
`${via}, but the unit's MainPID (${ownership.pid}) does not match the daemon record pid (${rec.pid}) — ` +
|
|
493
|
+
`the reported pid is NOT owned by the service manager; check \`${inspect}\``,
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
if (ownership.kind === "unknown") {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`${via}, but ownership can no longer be determined (${ownership.reason}) — ` +
|
|
499
|
+
`cannot confirm the daemon is managed; check \`${inspect}\``,
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
throw new Error(`${via}, but the unit is not active; check \`${inspect}\``);
|
|
503
|
+
}
|
|
504
|
+
|
|
416
505
|
/**
|
|
417
506
|
* What we know about {@link SYSTEMD_UNIT}.
|
|
418
507
|
*
|
|
@@ -420,14 +509,22 @@ export async function restartDaemon(
|
|
|
420
509
|
* states (`activating`, `deactivating`, `reloading`, `reactivating`) — the
|
|
421
510
|
* pid is still systemd-owned, so a raw SIGTERM would bounce under
|
|
422
511
|
* `Restart=on-failure`.
|
|
423
|
-
* - `
|
|
424
|
-
* no
|
|
512
|
+
* - `failed` — the unit is installed and its last activation failed
|
|
513
|
+
* (ActiveState=failed, no MainPID). systemd still owns this unit's
|
|
514
|
+
* lifecycle, so restoring the daemon must go through the manager
|
|
515
|
+
* (`reset-failed` + `start`); classifying a failed unit as "not supervised
|
|
516
|
+
* here" is how `restart` launched an unmanaged daemon next to a unit that
|
|
517
|
+
* stayed failed (#376).
|
|
518
|
+
* - `inactive` — confirmed not running (MainPID absent/0, ActiveState not
|
|
519
|
+
* `failed`), unit absent, *or* no `systemctl` binary on this host. Safe to
|
|
520
|
+
* treat as "not supervised here".
|
|
425
521
|
* - `unknown` — the manager exists (or we cannot tell it does not) but the
|
|
426
522
|
* query failed: dbus blip, permission, timeout. Must not be collapsed into
|
|
427
523
|
* `inactive` — that is how a unit-owned daemon gets a raw SIGTERM.
|
|
428
524
|
*/
|
|
429
525
|
export type UnitOwnership =
|
|
430
526
|
| { kind: "active"; pid: number }
|
|
527
|
+
| { kind: "failed" }
|
|
431
528
|
| { kind: "inactive" }
|
|
432
529
|
| { kind: "unknown"; reason: string };
|
|
433
530
|
|
|
@@ -454,26 +551,39 @@ export function probeUnit(unit = SYSTEMD_UNIT): UnitOwnership {
|
|
|
454
551
|
// live pid (> 1) means systemd still owns that process — including during
|
|
455
552
|
// `activating` / `deactivating` / `reloading`. Filtering on ActiveState here
|
|
456
553
|
// used to label those transitional states "inactive" and hand the pid to a
|
|
457
|
-
// raw SIGTERM, which is exactly the Restart=on-failure bounce.
|
|
554
|
+
// raw SIGTERM, which is exactly the Restart=on-failure bounce. The one
|
|
555
|
+
// place ActiveState decides is the `failed` classification below: with no
|
|
556
|
+
// MainPID there is no pid to hand to anything, but the unit has not gone
|
|
557
|
+
// away either.
|
|
458
558
|
const lines = ran.stdout
|
|
459
559
|
.split("\n")
|
|
460
560
|
.map((l) => l.trim())
|
|
461
561
|
.filter((l) => l.length > 0);
|
|
462
562
|
let pid: number | undefined;
|
|
563
|
+
let activeState: string | undefined;
|
|
463
564
|
for (const line of lines) {
|
|
464
565
|
if (/^\d+$/.test(line)) {
|
|
465
566
|
const n = Number(line);
|
|
466
567
|
if (Number.isInteger(n) && n > 1) pid = n;
|
|
568
|
+
} else {
|
|
569
|
+
activeState = line;
|
|
467
570
|
}
|
|
468
571
|
}
|
|
469
|
-
if (pid
|
|
470
|
-
|
|
572
|
+
if (pid !== undefined) return { kind: "active", pid };
|
|
573
|
+
// A failed unit is NOT the confirmed negative `inactive` claims to be: the
|
|
574
|
+
// unit is installed and the manager still owns its lifecycle, so restoring
|
|
575
|
+
// the daemon has to go through systemd (`reset-failed` + `start`). Reading
|
|
576
|
+
// `failed` as "not supervised here" is precisely how #376 ended with a
|
|
577
|
+
// healthy-looking but unmanaged pid next to a unit that stayed failed.
|
|
578
|
+
if (activeState === "failed") return { kind: "failed" };
|
|
579
|
+
return { kind: "inactive" };
|
|
471
580
|
}
|
|
472
581
|
|
|
473
582
|
/**
|
|
474
583
|
* The MainPID of an *active* {@link SYSTEMD_UNIT}, or `undefined` when the
|
|
475
|
-
* unit is confirmed inactive
|
|
476
|
-
* Prefer {@link probeUnit} when the caller must distinguish
|
|
584
|
+
* unit is confirmed inactive, failed, or absent, or when ownership could not
|
|
585
|
+
* be determined. Prefer {@link probeUnit} when the caller must distinguish
|
|
586
|
+
* those two.
|
|
477
587
|
*/
|
|
478
588
|
export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
|
|
479
589
|
const ownership = probeUnit(unit);
|
|
@@ -490,8 +600,9 @@ export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
|
|
|
490
600
|
* - `stop` — unit is active and owns `pid` (or there is no pidfile and the
|
|
491
601
|
* unit is the only candidate). Caller MUST go through systemctl; a failed
|
|
492
602
|
* manager call is terminal.
|
|
493
|
-
* - `not-ours` — confirmed inactive/absent unit, no systemd binary, or
|
|
494
|
-
* whose MainPID is someone else. Caller may SIGTERM its own pidfile
|
|
603
|
+
* - `not-ours` — confirmed inactive/failed/absent unit, no systemd binary, or
|
|
604
|
+
* a unit whose MainPID is someone else. Caller may SIGTERM its own pidfile
|
|
605
|
+
* process (a failed unit owns nothing, so signalling its record is safe).
|
|
495
606
|
* - `unknown` — ownership query failed. Caller MUST NOT signal.
|
|
496
607
|
*/
|
|
497
608
|
type SystemdStopDecision =
|
|
@@ -502,7 +613,10 @@ type SystemdStopDecision =
|
|
|
502
613
|
function decideSystemdStop(pid: number | undefined): SystemdStopDecision {
|
|
503
614
|
const ownership = probeUnit();
|
|
504
615
|
if (ownership.kind === "unknown") return { kind: "unknown", reason: ownership.reason };
|
|
505
|
-
|
|
616
|
+
// A failed unit owns no process (MainPID 0 — systemd has already reaped or
|
|
617
|
+
// lost it), so a live record pid is a confirmed "not ours", exactly like an
|
|
618
|
+
// inactive unit. Restoring the unit is restart's job, not stop's.
|
|
619
|
+
if (ownership.kind === "inactive" || ownership.kind === "failed") return { kind: "not-ours" };
|
|
506
620
|
if (pid !== undefined && ownership.pid !== pid) return { kind: "not-ours" };
|
|
507
621
|
return { kind: "stop", mainPid: ownership.pid };
|
|
508
622
|
}
|
|
@@ -533,11 +647,17 @@ async function runSystemdStop(pid: number, timeoutMs?: number): Promise<void> {
|
|
|
533
647
|
}
|
|
534
648
|
}
|
|
535
649
|
|
|
536
|
-
function systemctlFailure(
|
|
650
|
+
function systemctlFailure(
|
|
651
|
+
verb: "stop" | "restart" | "reset-failed" | "start",
|
|
652
|
+
ran: SystemctlResult,
|
|
653
|
+
): string {
|
|
537
654
|
const detail = (ran.stderr.trim() || ran.stdout.trim() || "no output").split("\n")[0] ?? "no output";
|
|
655
|
+
const guard =
|
|
656
|
+
verb === "stop" || verb === "restart"
|
|
657
|
+
? `refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce)`
|
|
658
|
+
: `the unit was NOT restored, so the daemon would be unmanaged`;
|
|
538
659
|
return (
|
|
539
|
-
`systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ` +
|
|
540
|
-
`refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce); ` +
|
|
660
|
+
`systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ${guard}; ` +
|
|
541
661
|
`fix the unit or run \`systemctl ${verb} ${SYSTEMD_UNIT}\` yourself`
|
|
542
662
|
);
|
|
543
663
|
}
|
package/src/omp.ts
CHANGED
|
@@ -398,6 +398,15 @@ const DRAIN_GRACE_MS = 2_000;
|
|
|
398
398
|
/** How long a disposed child gets to exit before it is signalled. */
|
|
399
399
|
const DISPOSE_GRACE_MS = 5_000;
|
|
400
400
|
|
|
401
|
+
/**
|
|
402
|
+
* Thrown by {@link createSession} when the pre-spawn admission gate closes: a
|
|
403
|
+
* daemon stop landed while the session socket was binding, so the child was
|
|
404
|
+
* never spawned. The caller maps this to a stopped run, not a failed one —
|
|
405
|
+
* a shutdown must not charge an attempt against a worker it refused to start
|
|
406
|
+
* (#374).
|
|
407
|
+
*/
|
|
408
|
+
export class SessionAdmissionClosedError extends Error {}
|
|
409
|
+
|
|
401
410
|
export interface CreateSessionOptions {
|
|
402
411
|
cwd: string;
|
|
403
412
|
sessionDir?: string;
|
|
@@ -430,6 +439,16 @@ export interface CreateSessionOptions {
|
|
|
430
439
|
* its verb channel to this pid, and a channel accepts nothing until it is bound.
|
|
431
440
|
*/
|
|
432
441
|
onSpawn?: (pid: number) => void;
|
|
442
|
+
/**
|
|
443
|
+
* Pre-spawn admission gate (#374). Consulted once, immediately after the
|
|
444
|
+
* socket bind await and immediately before `Bun.spawn` — the last window a
|
|
445
|
+
* daemon stop can land in before a child exists. A gate that returns false
|
|
446
|
+
* closes the listener, removes the socket, and throws
|
|
447
|
+
* {@link SessionAdmissionClosedError} instead of spawning; the run then
|
|
448
|
+
* settles as stopped rather than adding a worker the shutdown would have to
|
|
449
|
+
* wait for. Absent, the spawn always proceeds.
|
|
450
|
+
*/
|
|
451
|
+
maySpawn?: () => boolean;
|
|
433
452
|
/** Child stderr, line by line. Defaults to the process's own stderr. */
|
|
434
453
|
onChildLog?: (line: string) => void;
|
|
435
454
|
startupTimeoutMs?: number;
|
|
@@ -505,6 +524,21 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
|
|
|
505
524
|
resolve();
|
|
506
525
|
});
|
|
507
526
|
});
|
|
527
|
+
// Pre-spawn admission recheck (#374): SIGTERM/SIGINT can land during the
|
|
528
|
+
// listen await above — after the daemon's own pre-launch checks and before
|
|
529
|
+
// this child exists. A closed gate means no child: close the listener,
|
|
530
|
+
// remove the socket (and, when this side created the directory, the
|
|
531
|
+
// directory itself — the same cleanup the failure paths below run), and
|
|
532
|
+
// signal the caller to settle the run as stopped instead of spawning a
|
|
533
|
+
// worker the shutdown would then have to wait for.
|
|
534
|
+
if (opts.maySpawn?.() === false) {
|
|
535
|
+
server.close();
|
|
536
|
+
rmSync(socketPath, { force: true });
|
|
537
|
+
if (owned) rmSync(socketDir, { recursive: true, force: true });
|
|
538
|
+
throw new SessionAdmissionClosedError(
|
|
539
|
+
"daemon shutdown began while the session socket was binding",
|
|
540
|
+
);
|
|
541
|
+
}
|
|
508
542
|
// The socket is the run's own channel, and the daemon's own uid is the only
|
|
509
543
|
// one that speaks on it.
|
|
510
544
|
chmodSync(socketPath, 0o600);
|
package/src/setup-host.ts
CHANGED
|
@@ -314,21 +314,49 @@ function tickSearchRoots(project: ProjectConfig): string[] {
|
|
|
314
314
|
* have been a deliberate per-project choice — it *is* the collision — so it is
|
|
315
315
|
* rewritten; anything else is operator intent and survives untouched.
|
|
316
316
|
*/
|
|
317
|
-
|
|
318
|
-
|
|
317
|
+
/**
|
|
318
|
+
* This project's tick config, wherever it actually lives. A config stamped for
|
|
319
|
+
* another project is that project's file: the search roots overlap, and
|
|
320
|
+
* restamping it here would hand this project's identity to the other fleet's cwd.
|
|
321
|
+
*/
|
|
322
|
+
function findProjectTick(
|
|
323
|
+
project: ProjectConfig,
|
|
324
|
+
): { root: string; path: string; config: TickConfig } | undefined {
|
|
319
325
|
for (const root of tickSearchRoots(project)) {
|
|
320
326
|
const result = readTickConfig(root);
|
|
321
327
|
if (result.kind === "invalid") {
|
|
322
328
|
throw new Error(`tick config invalid at ${result.path}: ${result.problem}; fix or remove it before setup`);
|
|
323
329
|
}
|
|
324
|
-
// A config stamped for another project is that project's file: the search
|
|
325
|
-
// roots overlap, and restamping it here would hand this project's identity
|
|
326
|
-
// to the other fleet's cwd.
|
|
327
330
|
if (result.kind === "ok" && tickConfigMatchesProject(result.config, project.name)) {
|
|
328
|
-
|
|
329
|
-
break;
|
|
331
|
+
return { root, path: result.path, config: result.config };
|
|
330
332
|
}
|
|
331
333
|
}
|
|
334
|
+
return undefined;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* The directory this project's fleet pane runs in, which is **not** always
|
|
339
|
+
* `workspaceRoot`.
|
|
340
|
+
*
|
|
341
|
+
* A fleet that predates per-project roots keeps its `.conductor-tick.json` in the
|
|
342
|
+
* state dir, and that is the cwd its pane and `FLEET_CWD` already point at. Naming
|
|
343
|
+
* `workspaceRoot` instead sends `recover.sh` somewhere with no tick config, where
|
|
344
|
+
* `tick_agent_name` falls back to `fleet` and no longer matches the renamed pane —
|
|
345
|
+
* silently costing a live project its recovery. Falls back to `workspaceRoot`,
|
|
346
|
+
* which is where a brand-new project's config is written.
|
|
347
|
+
*/
|
|
348
|
+
export function tickCwdForProject(project: ProjectConfig): string {
|
|
349
|
+
try {
|
|
350
|
+
return findProjectTick(project)?.root ?? project.workspaceRoot;
|
|
351
|
+
} catch {
|
|
352
|
+
// An invalid config is setup's problem to report, not the handoff's.
|
|
353
|
+
return project.workspaceRoot;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function planTick(project: ProjectConfig, telegramStateDir: string): PlannedWrite<TickConfig> {
|
|
358
|
+
const found = findProjectTick(project);
|
|
359
|
+
const existing = found === undefined ? undefined : { path: found.path, config: found.config };
|
|
332
360
|
|
|
333
361
|
const armedFile = join(stateDir(), `armed-${project.name}`);
|
|
334
362
|
const path = existing?.path ?? join(project.workspaceRoot, TICK_CONFIG_FILE);
|
package/src/setup-wizard.ts
CHANGED
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
totalConfiguredWorkers,
|
|
38
38
|
runSetupSmoke,
|
|
39
39
|
SYSTEMD_UNIT_DIR,
|
|
40
|
+
tickCwdForProject,
|
|
40
41
|
writeHostRuntime,
|
|
41
42
|
} from "./setup-host.ts";
|
|
42
43
|
import { runGraphInstall, runHostInstall } from "./setup-install.ts";
|
|
@@ -1639,18 +1640,22 @@ export async function collectSetup(
|
|
|
1639
1640
|
*/
|
|
1640
1641
|
export function formatHerdrHandoff(project: ProjectConfig, cfg: ConductorConfig): string {
|
|
1641
1642
|
const session = process.env["HERDR_SESSION"] ?? "conductor";
|
|
1642
|
-
|
|
1643
|
+
// Every entry is the cwd that owns that project's tick config — the directory its
|
|
1644
|
+
// pane runs in — not `workspaceRoot`. They differ on any fleet whose config
|
|
1645
|
+
// predates per-project roots, and recovery keys on the tick config.
|
|
1646
|
+
const paneCwd = tickCwdForProject(project);
|
|
1647
|
+
const cwds = cfg.projects.map((p) => tickCwdForProject(p)).join(":");
|
|
1643
1648
|
return [
|
|
1644
1649
|
"herdr handoff (CLI cannot do these — exact herdr argv):",
|
|
1645
|
-
` 1. In session "${session}", create a workspace for project "${project.name}" at its
|
|
1646
|
-
` herdr --session ${session} workspace create --cwd ${
|
|
1650
|
+
` 1. In session "${session}", create a workspace for project "${project.name}" at its fleet cwd:`,
|
|
1651
|
+
` herdr --session ${session} workspace create --cwd ${paneCwd} --label ${project.name} --no-focus`,
|
|
1647
1652
|
" Read root_pane.pane_id from the JSON reply (jq -r '((.result // .).root_pane.pane_id) // empty').",
|
|
1648
1653
|
` 2. Start omp in that empty pane. The agent NAME must match tick config agentName ("${project.name}"):`,
|
|
1649
1654
|
` herdr --session ${session} agent start ${project.name} --kind omp --pane <pane-id>`,
|
|
1650
1655
|
" The pane must be at a shell prompt with no agent on it. Never agent-start into a live orchestrator pane.",
|
|
1651
1656
|
` 3. Point herdr-conductor recovery at every fleet cwd (colon-separated; see #320):`,
|
|
1652
1657
|
` FLEET_CWDS=${cwds}`,
|
|
1653
|
-
` Legacy single-fleet hosts can keep FLEET_CWD=${
|
|
1658
|
+
` Legacy single-fleet hosts can keep FLEET_CWD=${paneCwd} until multi-fleet recovery lands.`,
|
|
1654
1659
|
" 4. Reload the daemon so it serves every configured project:",
|
|
1655
1660
|
" omp-conductor restart --now",
|
|
1656
1661
|
" (printed, not auto-run, when workers are live or a neighbour was just added)",
|
package/src/types.ts
CHANGED
package/src/upgrade.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
|
|
3
3
|
import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
|
|
4
|
-
import { setPaused, statusSnapshot } from "./daemon.ts";
|
|
4
|
+
import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
|
|
5
5
|
import {
|
|
6
6
|
DEFAULT_HERDR_SESSION,
|
|
7
7
|
fleetLayers,
|
|
@@ -52,7 +52,21 @@ export interface UpgradeDeps {
|
|
|
52
52
|
snapshot(project?: string): { liveWorkers: number };
|
|
53
53
|
layers(project?: string): FleetLayers;
|
|
54
54
|
brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
|
|
55
|
-
|
|
55
|
+
/**
|
|
56
|
+
* The daemon the transaction targets, when one is running. `generation`
|
|
57
|
+
* identifies the exact instance (pid + start time) so a later restart cannot
|
|
58
|
+
* stop an instance created after the request began (#377).
|
|
59
|
+
*/
|
|
60
|
+
daemonIdentity(): { running: boolean; project?: string; generation?: string };
|
|
61
|
+
/**
|
|
62
|
+
* The durable pause sentinel this fleet currently holds, as an *instance*,
|
|
63
|
+
* read from one selected file (the per-project sentinel first): who set it,
|
|
64
|
+
* why, and when that sentinel was created. Two pauses with identical
|
|
65
|
+
* provenance are still different instances if `since` differs, so a drain
|
|
66
|
+
* can tell its own sentinel from one a later actor re-created after a
|
|
67
|
+
* resume lifted it — even when a legacy global sentinel coexists (#377).
|
|
68
|
+
*/
|
|
69
|
+
pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
|
|
56
70
|
setPaused(value: boolean, project?: string): void;
|
|
57
71
|
restartDaemon(): Promise<void>;
|
|
58
72
|
sleep(ms: number): Promise<void>;
|
|
@@ -91,8 +105,14 @@ export const DEFAULT_DEPS: UpgradeDeps = {
|
|
|
91
105
|
},
|
|
92
106
|
daemonIdentity: () => {
|
|
93
107
|
const daemon = livingDaemon();
|
|
94
|
-
|
|
108
|
+
if (daemon === undefined) return { running: false };
|
|
109
|
+
return {
|
|
110
|
+
running: true,
|
|
111
|
+
project: daemon.project,
|
|
112
|
+
generation: `${daemon.pid}@${daemon.startedAt}`,
|
|
113
|
+
};
|
|
95
114
|
},
|
|
115
|
+
pauseState: (project) => pauseInstance(project),
|
|
96
116
|
setPaused: (v, project) =>
|
|
97
117
|
setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
|
|
98
118
|
restartDaemon: async () => {
|
|
@@ -248,9 +268,20 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
|
|
|
248
268
|
];
|
|
249
269
|
}
|
|
250
270
|
|
|
251
|
-
async function waitForDrain(
|
|
271
|
+
async function waitForDrain(
|
|
272
|
+
deps: UpgradeDeps,
|
|
273
|
+
project?: string,
|
|
274
|
+
deadlineAt?: number,
|
|
275
|
+
stale?: () => string | undefined,
|
|
276
|
+
): Promise<void> {
|
|
252
277
|
let last = -1;
|
|
253
278
|
while (true) {
|
|
279
|
+
// A transaction that is no longer the fleet's current intent exits before
|
|
280
|
+
// it waits on anyone else's workers: a resume lifted its pause or the
|
|
281
|
+
// daemon it began with was replaced, so there is nothing left to drain
|
|
282
|
+
// and nothing it may stop (#377).
|
|
283
|
+
const cancelled = stale?.();
|
|
284
|
+
if (cancelled !== undefined) throw new Error(cancelled);
|
|
254
285
|
const workers = deps.snapshot(project).liveWorkers;
|
|
255
286
|
if (workers === 0) return;
|
|
256
287
|
if (deadlineAt !== undefined && Date.now() >= deadlineAt) {
|
|
@@ -264,10 +295,71 @@ async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: nu
|
|
|
264
295
|
}
|
|
265
296
|
}
|
|
266
297
|
|
|
298
|
+
/**
|
|
299
|
+
* What a draining restart captured when it began: the restart-owned pause and
|
|
300
|
+
* the daemon instance it is entitled to stop. Every later fence check compares
|
|
301
|
+
* live state against this snapshot, so the destructive restart call only ever
|
|
302
|
+
* acts on the world the request began in (#377).
|
|
303
|
+
*/
|
|
304
|
+
interface RestartBegun {
|
|
305
|
+
/** The durable pause sentinel this request relies on, proved readable as an instance. */
|
|
306
|
+
pauseToken: { source: string; reason?: string; since: number };
|
|
307
|
+
/** The daemon instance the restart targets, with its generation identity. */
|
|
308
|
+
daemon: ReturnType<UpgradeDeps["daemonIdentity"]>;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Why a draining restart can no longer act, or `undefined` when it still owns
|
|
313
|
+
* the pause and the daemon generation it began with. The pause check is the
|
|
314
|
+
* *restart-owned sentinel instance* — `drainAndRestart` proves ownership up
|
|
315
|
+
* front, so a token that cannot be re-read (sentinel removed, unreadable, or
|
|
316
|
+
* re-created — even under the same source and reason) means this request's own
|
|
317
|
+
* pause no longer exists and it must cancel. The generation check is the
|
|
318
|
+
* later-daemon fence: a daemon created after this request began — by
|
|
319
|
+
* `restart --now`, a crash-restart, or an operator start — must never be
|
|
320
|
+
* stopped by it.
|
|
321
|
+
*/
|
|
322
|
+
function restartFenceProblem(
|
|
323
|
+
deps: UpgradeDeps,
|
|
324
|
+
project: string | undefined,
|
|
325
|
+
begun: RestartBegun,
|
|
326
|
+
): string | undefined {
|
|
327
|
+
if (!deps.layers(project).paused) {
|
|
328
|
+
return "restart cancelled: dispatch was resumed while the drain was in progress — nothing was restarted";
|
|
329
|
+
}
|
|
330
|
+
const owned = deps.pauseState(project);
|
|
331
|
+
if (
|
|
332
|
+
owned === undefined ||
|
|
333
|
+
owned.since !== begun.pauseToken.since ||
|
|
334
|
+
owned.source !== begun.pauseToken.source ||
|
|
335
|
+
owned.reason !== begun.pauseToken.reason
|
|
336
|
+
) {
|
|
337
|
+
return "restart cancelled: the restart-owned pause was lifted and replaced while the drain was in progress — nothing was restarted";
|
|
338
|
+
}
|
|
339
|
+
const now = deps.daemonIdentity();
|
|
340
|
+
if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
|
|
341
|
+
return "restart cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running";
|
|
342
|
+
}
|
|
343
|
+
return undefined;
|
|
344
|
+
}
|
|
345
|
+
|
|
267
346
|
/**
|
|
268
347
|
* Pause, drain, restart, restore — the trusted restart transaction, minus the
|
|
269
348
|
* install/verify steps of {@link upgradeConductor}.
|
|
270
349
|
*
|
|
350
|
+
* Generation-scoped (#377): the transaction captures the restart-owned pause
|
|
351
|
+
* and the daemon generation when it begins and revalidates both on every drain
|
|
352
|
+
* poll and once more immediately before the destructive restart call. A
|
|
353
|
+
* `resume`, a `restart --now`, or any other action that lifts the pause or
|
|
354
|
+
* replaces the daemon while the drain waits makes the transaction stale: it
|
|
355
|
+
* exits nonzero without stopping anything, so a timed-out or disconnected
|
|
356
|
+
* restart caller cannot leave a delayed stop aimed at a later daemon
|
|
357
|
+
* generation. The stale transaction restores nothing and leaves whatever
|
|
358
|
+
* pause the fleet now has exactly as it found it. Ownership is proven up
|
|
359
|
+
* front: a sentinel that cannot be read as an instance at transaction start
|
|
360
|
+
* aborts before any drain or restart, because a lock the fence cannot compare
|
|
361
|
+
* against is a lock the fence cannot protect.
|
|
362
|
+
*
|
|
271
363
|
* Mirrors the upgrade's fail-closed posture: on ANY throw after the pause the
|
|
272
364
|
* fleet stays paused (no resume in a catch) and the error rethrows, so a
|
|
273
365
|
* systemd-owned restart that fails or a drain that outlives `timeoutMs` cannot
|
|
@@ -280,7 +372,26 @@ export async function drainAndRestart(
|
|
|
280
372
|
): Promise<void> {
|
|
281
373
|
const initial = deps.layers(o.project);
|
|
282
374
|
if (!initial.paused) deps.setPaused(true, o.project);
|
|
283
|
-
|
|
375
|
+
// The lock must be provable NOW. A pause that cannot be read as an instance
|
|
376
|
+
// — sentinel malformed or unreadable — would make the whole fence fail open
|
|
377
|
+
// (nothing to compare against), so the transaction refuses before waiting
|
|
378
|
+
// on anything it cannot act upon either way.
|
|
379
|
+
const pauseToken = deps.pauseState(o.project);
|
|
380
|
+
if (pauseToken === undefined) {
|
|
381
|
+
throw new Error(
|
|
382
|
+
"restart cancelled: cannot prove the restart-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted",
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
const begun: RestartBegun = {
|
|
386
|
+
pauseToken,
|
|
387
|
+
daemon: deps.daemonIdentity(),
|
|
388
|
+
};
|
|
389
|
+
const stale = () => restartFenceProblem(deps, o.project, begun);
|
|
390
|
+
await waitForDrain(deps, o.project, Date.now() + o.timeoutMs, stale);
|
|
391
|
+
// The drain completed; the world may have moved on while it did. Re-prove
|
|
392
|
+
// the pause and the generation an instant before the destructive call.
|
|
393
|
+
const cancelled = stale();
|
|
394
|
+
if (cancelled !== undefined) throw new Error(cancelled);
|
|
284
395
|
await deps.restartDaemon();
|
|
285
396
|
if (!initial.paused) deps.setPaused(false, o.project);
|
|
286
397
|
}
|
package/src/worker.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* sliding into a merge queue.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { createSession, disposeSession } from "./omp.ts";
|
|
13
|
+
import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
|
|
14
14
|
import type { ReleaseBlockContext } from "./release-policy.ts";
|
|
15
15
|
import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
|
|
16
16
|
|
|
@@ -123,6 +123,13 @@ export interface WorkerOpts {
|
|
|
123
123
|
* exists; absent = no pause surface (tests, one-shot callers).
|
|
124
124
|
*/
|
|
125
125
|
onPauseControl?: (control: WorkerPauseControl) => void;
|
|
126
|
+
/**
|
|
127
|
+
* Pre-spawn admission gate, forwarded to `createSession` (#374): the daemon
|
|
128
|
+
* closes it the moment its stop begins, so a stop that lands while the
|
|
129
|
+
* session socket is binding refuses the spawn and the run settles `stopped`
|
|
130
|
+
* instead of `failed`. Absent, the spawn always proceeds.
|
|
131
|
+
*/
|
|
132
|
+
maySpawn?: () => boolean;
|
|
126
133
|
}
|
|
127
134
|
|
|
128
135
|
/**
|
|
@@ -253,20 +260,39 @@ export async function runWorker(
|
|
|
253
260
|
const now = deps.now ?? Date.now;
|
|
254
261
|
const schedule = deps.schedule ?? scheduleWallClock;
|
|
255
262
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
263
|
+
let session: AgentSessionLike;
|
|
264
|
+
try {
|
|
265
|
+
session = await deps.createSession({
|
|
266
|
+
cwd: o.cwd,
|
|
267
|
+
...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
|
|
268
|
+
...(o.model === undefined ? {} : { model: o.model }),
|
|
269
|
+
// Prevention half of #24: as a worker, structured file tools cannot leave
|
|
270
|
+
// this worktree, and no release grant can ever reach this session (#122).
|
|
271
|
+
role: "worker",
|
|
272
|
+
...(o.releaseGrants === undefined ? {} : { releaseGrants: o.releaseGrants }),
|
|
273
|
+
...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
|
|
274
|
+
...(o.onSpawn === undefined ? {} : { onSpawn: o.onSpawn }),
|
|
275
|
+
...(o.socketPath === undefined ? {} : { socketPath: o.socketPath }),
|
|
276
|
+
...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
|
|
277
|
+
...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
|
|
278
|
+
...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
|
|
279
|
+
});
|
|
280
|
+
} catch (err) {
|
|
281
|
+
// The pre-spawn gate closed (#374): a daemon stop landed while the session
|
|
282
|
+
// socket was binding, so no child was ever created. That is a stop, not a
|
|
283
|
+
// failure — reporting `failed` here would charge an attempt against a
|
|
284
|
+
// worker the shutdown refused to start. `onSpawn` was never called.
|
|
285
|
+
if (err instanceof SessionAdmissionClosedError) {
|
|
286
|
+
return {
|
|
287
|
+
state: "stopped",
|
|
288
|
+
turns: 0,
|
|
289
|
+
spendUsd: 0,
|
|
290
|
+
report: "",
|
|
291
|
+
stoppedReason: "daemon shutdown began before the worker session started",
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
270
296
|
|
|
271
297
|
// Before the first turn, not after the last: a caller that only learns the
|
|
272
298
|
// transcript path from the result learns it once the run it wanted to watch
|