omp-conductor 0.15.2 → 0.15.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "omp-conductor",
3
- "version": "0.15.2",
3
+ "version": "0.15.4",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
package/src/cli.ts CHANGED
@@ -155,7 +155,9 @@ usage:
155
155
  that hits --timeout restarts nothing and leaves dispatch paused. On boot
156
156
  the new process salvages dirty live worktrees before orphaning those
157
157
  rows — see README "Deploying a new package onto a busy fleet". Goes
158
- through systemctl when the unit owns the live pid.
158
+ through systemctl when the unit owns the live pid; a failed
159
+ installed unit is reset and restarted through systemd, never
160
+ replaced by an unmanaged daemon.
159
161
  status layered fleet report: dispatch (running|paused|stopped), ticks and
160
162
  next due time, pane, herdr, Telegram bot/API health, daemon, caps
161
163
  and active runs.
@@ -682,7 +684,10 @@ try {
682
684
  // Inherit the running daemon's port and project: a restart that quietly
683
685
  // moved to the default port would leave every existing health check
684
686
  // pointing at nothing. When the unit owns the live pid, restartDaemon
685
- // goes through systemctl so the replacement stays supervised.
687
+ // goes through systemctl so the replacement stays supervised; a failed
688
+ // installed unit is reset and started through systemd the same way and
689
+ // is never replaced by an unmanaged daemon. Success is only reported
690
+ // after the manager is proven to own the reported pid.
686
691
  const { previous, record, via } = await restartDaemon({
687
692
  port: portFlag(argv),
688
693
  project: projectFlag,
package/src/daemon.ts CHANGED
@@ -171,11 +171,25 @@ export interface DaemonOpts {
171
171
  * model or caps — `handleIssue` destructures them at dispatch time. The rest
172
172
  * are resolved once at startup so a tick never re-reads config mid-flight and
173
173
  * changes its own limits underneath itself. */
174
+ export interface DrainSignal {
175
+ draining: boolean;
176
+ }
177
+
174
178
  interface Deps {
175
179
  project: ProjectConfig;
176
180
  caps: Caps;
177
181
  tracker: Tracker;
178
182
  store: Store;
183
+ /**
184
+ * Set true the moment a daemon stop (SIGTERM/SIGINT) begins. The run loop
185
+ * only sees `stopping` between whole ticks, so a pass that was already in
186
+ * flight when the stop landed must re-check this flag itself — before it
187
+ * claims and before it launches — or it creates exactly the work the
188
+ * shutdown is about to wait for and then lose to the stop timeout (#374).
189
+ * Optional only so tests that never exercise shutdown can omit it;
190
+ * `runDaemon` always wires the real one.
191
+ */
192
+ drain?: DrainSignal;
179
193
  /** False after a live config reload fails; autonomous delivery then holds
180
194
  * fail-closed until a later tick validates the config again. */
181
195
  deliveryPolicyValid?: boolean;
@@ -444,6 +458,40 @@ export function pauseProvenance(
444
458
  }
445
459
  }
446
460
 
461
+ /**
462
+ * One pause sentinel read as a single identity: who set it, why, and the
463
+ * creation instant, all from the SAME file that was selected. Unlike pairing
464
+ * {@link pauseProvenance} with {@link pausedAt} — which can describe
465
+ * different files when a project pause coexists with the legacy global
466
+ * sentinel, letting a stale global timestamp mask a recreated project pause —
467
+ * this reads provenance and timestamp from one sentinel, so a caller can prove
468
+ * "the pause I set still exists" instead of "some pause with the same labels
469
+ * still exists" (#377). Per-project sentinel wins, like {@link pauseProvenance}.
470
+ */
471
+ export function pauseInstance(
472
+ project?: string,
473
+ ): { source: string; reason?: string; since: number } | undefined {
474
+ const paths =
475
+ project === undefined
476
+ ? [pausedPath()]
477
+ : [pausedPath(project), pausedPath()];
478
+ const path = paths.find((candidate) => existsSync(candidate));
479
+ if (path === undefined) return undefined;
480
+ try {
481
+ const [line1, line2] = readFileSync(path, "utf8").split("\n");
482
+ const since = Date.parse(line1?.trim() ?? "");
483
+ if (!Number.isFinite(since)) return undefined;
484
+ if (line2 === undefined) return undefined;
485
+ const match = /^source=(\S+)(?: reason="(.*)")?$/.exec(line2.trim());
486
+ if (match === null) return undefined;
487
+ const source = match[1]!;
488
+ const reason = match[2];
489
+ return { source, since, ...(reason === undefined ? {} : { reason }) };
490
+ } catch {
491
+ return undefined;
492
+ }
493
+ }
494
+
447
495
  export function setPaused(
448
496
  v: boolean,
449
497
  why?: { source: string; reason?: string },
@@ -1309,7 +1357,65 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1309
1357
  return true;
1310
1358
  };
1311
1359
 
1360
+ /**
1361
+ * The daemon-wide stop fence can land after the claim but before `runWorker`
1362
+ * exposes its controller — the same window `settleStopBeforeSession` closes
1363
+ * for a per-run operator stop, entered from the shutdown path instead of the
1364
+ * pause control. Settle identically (close the claim, never launch) so the
1365
+ * drain waits only for workers admitted before the stop request, and the
1366
+ * shutdown cannot extend its own workload (#374).
1367
+ */
1368
+ const settleDrainBeforeSession = async (): Promise<boolean> => {
1369
+ if (d.drain?.draining !== true || run === undefined || workerSessionInstalled) return false;
1370
+ turnLimit?.close();
1371
+ turnLimit = undefined;
1372
+ const settlement =
1373
+ worktreePath === undefined
1374
+ ? undefined
1375
+ : await settleWorktree({
1376
+ issue,
1377
+ attempt,
1378
+ ending: "daemon shutdown began while the run was being prepared",
1379
+ worktree: worktreePath,
1380
+ branch,
1381
+ publish,
1382
+ tree: "remove",
1383
+ mirrorPath,
1384
+ });
1385
+ recordOperatorStop(store, {
1386
+ project: project.name,
1387
+ issue,
1388
+ runId: run.id,
1389
+ inProgress,
1390
+ reason: "daemon shutdown began after the claim",
1391
+ patch: {
1392
+ endedAt: Date.now(),
1393
+ turns: run.turns,
1394
+ spendUsd: run.spendUsd,
1395
+ worktree: worktreePath ?? run.worktree,
1396
+ report: [
1397
+ "Daemon shutdown began while this run was still being prepared; the worker",
1398
+ "never started and the claim was closed so the shutdown could complete.",
1399
+ ...(settlement?.lines ?? []),
1400
+ ].join("\n"),
1401
+ ...settlement?.patch,
1402
+ },
1403
+ });
1404
+ log(`#${issue} not launched: daemon shutdown began after the claim`);
1405
+ return true;
1406
+ };
1407
+
1312
1408
  try {
1409
+ // The claim-side of the stop fence (#374): the run row is the boundary the
1410
+ // shutdown drain waits on, so the claim itself refuses once the daemon is
1411
+ // draining. The tick re-checks before the dispatch tail; this covers the
1412
+ // same fence for any caller that reaches `handleIssue` without one, and
1413
+ // makes "no post-stop run rows" a property of the claim, not of its caller.
1414
+ if (d.drain?.draining === true) {
1415
+ log(`#${issue} not claimed: daemon is draining`);
1416
+ return;
1417
+ }
1418
+
1313
1419
  // Claim on the STORE first, before anything that can fail. The run row —
1314
1420
  // not the label — is the crash-safe guard against double dispatch: rows
1315
1421
  // are local, written before any network call, and the startup orphan
@@ -1351,6 +1457,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1351
1457
  turnLimit = d.turnLimits.open(project.name, issue, runId, maxTurns);
1352
1458
  workerControl = d.workerControls.open(project.name, issue, runId);
1353
1459
  if (await settleStopBeforeSession()) return;
1460
+ if (await settleDrainBeforeSession()) return;
1354
1461
 
1355
1462
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
1356
1463
  // an existing path, so a retry — or a tree kept from a failed attempt — has
@@ -1360,6 +1467,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1360
1467
  // would cost a second network fetch per attempt.
1361
1468
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
1362
1469
  if (await settleStopBeforeSession()) return;
1470
+ if (await settleDrainBeforeSession()) return;
1363
1471
  const provisioned = await addRunRepo(
1364
1472
  r.repo,
1365
1473
  project.mirrorRoot,
@@ -1370,6 +1478,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1370
1478
  worktreePath = provisioned.path;
1371
1479
  runRepo = { repo: r.repo, runRepoPath: worktreePath, branch };
1372
1480
  if (await settleStopBeforeSession()) return;
1481
+ if (await settleDrainBeforeSession()) return;
1373
1482
 
1374
1483
  // The SDK names the transcript itself, so the daemon supplies the parent
1375
1484
  // directory and learns the real path back from the result. Inventing one
@@ -1409,6 +1518,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1409
1518
  { ...(d.verbPeerReader === undefined ? {} : { peerReader: d.verbPeerReader }) },
1410
1519
  );
1411
1520
  if (await settleStopBeforeSession()) return;
1521
+ if (await settleDrainBeforeSession()) return;
1412
1522
 
1413
1523
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
1414
1524
 
@@ -1425,6 +1535,7 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1425
1535
  : {}),
1426
1536
  });
1427
1537
  if (await settleStopBeforeSession()) return;
1538
+ if (await settleDrainBeforeSession()) return;
1428
1539
 
1429
1540
  const repoSlug = githubRepo(r.repo.cloneUrl);
1430
1541
 
@@ -1470,6 +1581,12 @@ export async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<
1470
1581
  // and a path written at completion is a path nobody can follow live. The
1471
1582
  // completion-time update below writes the same value again, harmlessly.
1472
1583
  onSessionFile: (f) => store.updateRun(runId, { sessionFile: f }),
1584
+ // The last fence (#374): every pre-launch settle check above has
1585
+ // passed, but the stop can still land while the session socket is
1586
+ // binding inside `createSession`. This gate is re-checked there,
1587
+ // immediately before the child spawn, and closes the run as stopped
1588
+ // instead of launching a worker the shutdown would have to wait for.
1589
+ maySpawn: () => d.drain?.draining !== true,
1473
1590
  }, d.workerDeps);
1474
1591
  } finally {
1475
1592
  // This is the authoritative settlement edge for `extend`: close before
@@ -3273,6 +3390,21 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
3273
3390
  }
3274
3391
 
3275
3392
  const pass = await admitCandidates(d, routed, slots);
3393
+
3394
+ // The stop fence (#374): the run loop only re-reads `stopping` between
3395
+ // whole ticks, so a pass already in flight when SIGTERM/SIGINT landed must
3396
+ // re-check the drain signal here — after candidate admission, before any
3397
+ // claim or launch — or the shutdown path admits exactly the work it is
3398
+ // about to wait for and then loses to the stop timeout. The candidates keep
3399
+ // their queue labels; the next daemon start re-dispatches them.
3400
+ if (d.drain?.draining === true) {
3401
+ recordDispatch(0, [
3402
+ ...routingHolds,
3403
+ ...pass.holds,
3404
+ ...pass.admitted.map((a) => ({ issue: a.r.issue.number, reason: "shutting-down" as const })),
3405
+ ]);
3406
+ return;
3407
+ }
3276
3408
  recordDispatch(pass.admitted.length, [...routingHolds, ...pass.holds]);
3277
3409
 
3278
3410
  if (pass.admitted.length === 0) return;
@@ -4553,6 +4685,14 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4553
4685
  const alive = livingDaemon();
4554
4686
  const runtimes: ProjectRuntime[] = [];
4555
4687
 
4688
+ // The stop fence (#374): one signal shared by every project's tick.
4689
+ // `stopping` gates the run loop between whole ticks; `drain.draining` is
4690
+ // the same event visible *inside* a tick already in flight, so a pass that
4691
+ // was mid-admission when the stop landed re-checks it before claiming or
4692
+ // launching. Created here, before the loop, so `stop` below and every
4693
+ // project's `Deps` reference the same object.
4694
+ const drain: DrainSignal = { draining: false };
4695
+
4556
4696
  log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`);
4557
4697
  log(`package integrity baseline: ${integrity.baseline.size} files under ${import.meta.dir}`);
4558
4698
 
@@ -4675,6 +4815,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4675
4815
  caps,
4676
4816
  tracker,
4677
4817
  store,
4818
+ drain,
4678
4819
  deliveryPolicyValid: false,
4679
4820
  usage,
4680
4821
  escalate: (event) => escalator.escalate(event),
@@ -4754,6 +4895,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
4754
4895
  const stop = (): void => {
4755
4896
  if (stopping) return;
4756
4897
  stopping = true;
4898
+ // Close the admission fence before the loop is told, so a tick already in
4899
+ // flight — parked at an await when the signal landed — sees `draining`
4900
+ // when it resumes and cannot create the work this shutdown is about to
4901
+ // wait for (#374).
4902
+ drain.draining = true;
4757
4903
  log("shutting down after active workers finish");
4758
4904
  wake?.();
4759
4905
  };
package/src/gitops.ts CHANGED
@@ -112,17 +112,30 @@ export function repoSlugFor(repo: RepoTarget): string {
112
112
  * hop that touches the network happens here, so the dispatcher observes the
113
113
  * exact sha it recorded.
114
114
  *
115
- * Both hops are fast-forward only and there is no force path anywhere in this
116
- * package. The fetch refspec deliberately carries **no leading `+`** that
117
- * single character is the difference between "advance the mirror's copy of this
118
- * branch" and "make the mirror's copy whatever the run says it is", and a run
119
- * repo is the one place in this system that model-executed code fully controls.
120
- * The push carries no `--force` and no `--force-with-lease`: a lease is still a
121
- * force, and a rejected push is a decision for a human, not a retry.
115
+ * The run-repo hop carries **no leading `+`** that single character is the
116
+ * difference between "advance the mirror's copy of this branch" and "make the
117
+ * mirror's copy whatever the run says it is", and a run repo is the one place
118
+ * in this system that model-executed code fully controls. The push carries no
119
+ * `--force` and no `--force-with-lease`: a lease is still a force, and a
120
+ * rejected push is a decision for a human, not a retry.
121
+ *
122
+ * The one exception is narrower than it looks. The mirror's copy of a run
123
+ * branch can be stale in a way that is not the run's fault:
124
+ * `conductor_pr_update_branch` moves — and on a rewritten history, replaces —
125
+ * the remote branch server-side after a run's last push, so a continuation
126
+ * that incorporated the reviewed baseline can propose a head the *live remote*
127
+ * is an ancestor of while the mirror's old copy is not. That is a legitimate
128
+ * fast-forward, not a divergence, so when the plain copy is refused this push
129
+ * reconciles the mirror's remote-tracking ref of the run branch with the live
130
+ * remote and asks git whether the live head is an ancestor of the proposed
131
+ * head. Only that proof lets the mirror's copy be refreshed, and the push hop
132
+ * still re-checks the ancestor relation against the live remote itself.
133
+ * Everything else is refused carrying the exact proposed and live SHAs so a
134
+ * real divergence is diagnosable.
122
135
  *
123
136
  * A rejection — non-fast-forward, protected ref, unauthorised — comes back as a
124
- * value carrying git's stderr verbatim, because the caller settles the run
125
- * `failed` on it and a paraphrase is worthless in that report.
137
+ * value carrying the reason, because the caller settles the run `failed` on it
138
+ * and a paraphrase is worthless in that report.
126
139
  */
127
140
  export async function pushRunBranch(
128
141
  project: Pick<ProjectConfig, "mirrorRoot">,
@@ -131,24 +144,93 @@ export async function pushRunBranch(
131
144
  ): Promise<PushOutcome> {
132
145
  const mirror = mirrorPath(project, run.repo);
133
146
  const ref = `refs/heads/${run.branch}`;
147
+ const tracked = `refs/remotes/origin/${run.branch}`;
134
148
  const env = credentialedEnv();
135
149
 
150
+ // Copy the run's branch into the mirror, fast-forward only.
136
151
  const fetched = await exec(["git", "-C", mirror, "fetch", "--no-tags", run.runRepoPath, `${ref}:${ref}`], { env });
137
- if (fetched.code !== 0) {
138
- return { ok: false, stderr: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`) };
152
+
153
+ // The proposed head is read from the run repo itself: when the copy above
154
+ // was refused, the mirror's copy of the branch is exactly the stale one.
155
+ const proposedRun = await exec(["git", "-C", run.runRepoPath, "rev-parse", ref], { env });
156
+ if (proposedRun.code !== 0) {
157
+ return { ok: false, stderr: scrubUserinfo(proposedRun.stderr.trim() || `git rev-parse ${ref} exited ${String(proposedRun.code)}`) };
158
+ }
159
+ const proposed = proposedRun.stdout.trim();
160
+
161
+ // Reconcile with the live remote before enforcing anything: the ancestry
162
+ // that decides a fast-forward is the live remote's, not the mirror's copy's
163
+ // (see the docstring above). ls-remote answers "is the branch published at
164
+ // all" and "what does GitHub hold" in one call.
165
+ const liveListed = await exec(["git", "-C", mirror, "ls-remote", "origin", ref], { env });
166
+ if (liveListed.code !== 0) {
167
+ return { ok: false, stderr: scrubUserinfo(liveListed.stderr.trim() || `git ls-remote origin ${ref} exited ${String(liveListed.code)}`) };
168
+ }
169
+ const live = liveListed.stdout
170
+ .split("\n")
171
+ .map((line) => line.trimEnd())
172
+ .find((line) => line.endsWith(`\t${ref}`))
173
+ ?.split(/\s+/, 1)[0];
174
+
175
+ if (live !== undefined && live !== proposed) {
176
+ // The branch is published and the run proposes something different. Pull
177
+ // the live branch's history into the mirror once (which also keeps the
178
+ // tracked ref a reattach reads fresh), then prove the fast-forward.
179
+ const reconciled = await exec(["git", "-C", mirror, "fetch", "--no-tags", "origin", `+${ref}:${tracked}`], { env });
180
+ if (reconciled.code !== 0) {
181
+ return { ok: false, stderr: scrubUserinfo(reconciled.stderr.trim() || reconciled.stdout.trim() || `git fetch origin exited ${String(reconciled.code)}`) };
182
+ }
183
+
184
+ const isAncestor = await exec(["git", "-C", mirror, "merge-base", "--is-ancestor", live, proposed], { env });
185
+ if (isAncestor.code === 128) {
186
+ // git could not perform the check at all (an object it was asked to
187
+ // resolve is missing, not merely unrelated). That is a failed
188
+ // verification, not a divergence verdict: refuse with git's own words,
189
+ // still naming both SHAs so the report carries the mismatch.
190
+ return {
191
+ ok: false,
192
+ stderr: scrubUserinfo(
193
+ isAncestor.stderr.trim() ||
194
+ isAncestor.stdout.trim() ||
195
+ `git merge-base --is-ancestor ${live} ${proposed} exited ${String(isAncestor.code)}`,
196
+ ),
197
+ };
198
+ }
199
+ if (isAncestor.code !== 0) {
200
+ // A real divergence: no fast-forward exists, and the mirror is left
201
+ // exactly where it was. Both SHAs are named so the mismatch is
202
+ // diagnosable instead of a bare "non-fast-forward".
203
+ return {
204
+ ok: false,
205
+ stderr:
206
+ `refusing non-fast-forward push of ${run.branch}: the live remote tip ${live} is not an ancestor of ` +
207
+ `the proposed head ${proposed}. Fetch the live branch and rebase or merge it before pushing again.`,
208
+ };
209
+ }
139
210
  }
140
211
 
141
- const resolved = await exec(["git", "-C", mirror, "rev-parse", ref], { env });
142
- if (resolved.code !== 0) {
143
- return { ok: false, stderr: scrubUserinfo(resolved.stderr.trim() || `git rev-parse ${ref} exited ${String(resolved.code)}`) };
212
+ if (fetched.code !== 0 && live === undefined) {
213
+ // The mirror refused the plain copy and there is no published branch to
214
+ // validate the run's head against: a rewritten branch that was never
215
+ // published must not rewrite the mirror either. Refuse with git's words.
216
+ return { ok: false, stderr: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`) };
217
+ }
218
+ if (fetched.code !== 0) {
219
+ // The proposed head is a legitimate fast-forward over the live remote (it
220
+ // equals the live head or the ancestor test above passed), so the only
221
+ // thing the plain copy refused on was the mirror's own stale copy.
222
+ // Refresh it; the push below still re-checks against the live remote.
223
+ const refreshed = await exec(["git", "-C", mirror, "fetch", "--no-tags", run.runRepoPath, `+${ref}:${ref}`], { env });
224
+ if (refreshed.code !== 0) {
225
+ return { ok: false, stderr: scrubUserinfo(refreshed.stderr.trim() || refreshed.stdout.trim() || `git fetch exited ${String(refreshed.code)}`) };
226
+ }
144
227
  }
145
- const sha = resolved.stdout.trim();
146
228
 
147
229
  const pushed = await exec(["git", "-C", mirror, "push", "origin", `${ref}:${ref}`], { env });
148
230
  if (pushed.code !== 0) {
149
231
  return { ok: false, stderr: scrubUserinfo(pushed.stderr.trim() || pushed.stdout.trim() || `git push exited ${String(pushed.code)}`) };
150
232
  }
151
- return { ok: true, sha };
233
+ return { ok: true, sha: proposed };
152
234
  }
153
235
 
154
236
  /**
package/src/lifecycle.ts CHANGED
@@ -364,9 +364,19 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
364
364
  * Same ownership rule as {@link stopDaemon}: when the unit owns the live pid,
365
365
  * `systemctl restart` is the *only* path — a failed manager call is terminal,
366
366
  * never a fallthrough to raw signals. An *unanswered* ownership query is also
367
- * terminal: "dbus blipped" is not "no unit". Falling back to stop+start is
368
- * reserved for a confirmed hand-started daemon (no systemd, inactive unit, or
369
- * a unit whose MainPID is someone else).
367
+ * terminal: "dbus blipped" is not "no unit". A *failed* installed unit is the
368
+ * third non-fallthrough: it must be restored through the manager
369
+ * (`reset-failed` + `start`), because a detached CLI daemon beside a failed
370
+ * unit is exactly the "restarted, but the fleet is still down" lie this
371
+ * module exists to prevent (#376). Falling back to stop+start is reserved for
372
+ * a confirmed hand-started daemon (no systemd, inactive unit, or a unit whose
373
+ * MainPID is someone else).
374
+ *
375
+ * Success is reported only after the service manager is re-probed and proven
376
+ * to own the daemon: the unit's MainPID must equal the daemon record pid AND
377
+ * `/healthz` must answer. The manager call returning is not that proof — for
378
+ * `Type=simple` the unit is active as soon as the process is forked, before
379
+ * the daemon has bound its port.
370
380
  *
371
381
  * Returns the record of the process that is now answering `/healthz`.
372
382
  */
@@ -379,6 +389,16 @@ export async function restartDaemon(
379
389
  throw new Error(ownershipUnknown("restart", ownership.reason));
380
390
  }
381
391
 
392
+ if (ownership.kind === "failed") {
393
+ // An installed unit in failed state: reset the failure record and let
394
+ // systemd start the service. Never fall through to stopDaemon() +
395
+ // startDaemon() — that leaves the unit failed while handing the operator
396
+ // a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
397
+ // Any manager refusal is terminal; so is unproven ownership afterwards.
398
+ const record = await restoreFailedUnit(o.timeoutMs);
399
+ return { previous, record, via: "systemctl" };
400
+ }
401
+
382
402
  const unitOwns =
383
403
  ownership.kind === "active" && (previous === undefined || previous.pid === ownership.pid);
384
404
 
@@ -393,26 +413,95 @@ export async function restartDaemon(
393
413
  // systemctl restart returns once the new MainPID is up; the pidfile is
394
414
  // written by the daemon itself on boot, so wait for that rather than
395
415
  // inventing a record from the unit alone.
396
- const deadline = Date.now() + READY_TIMEOUT_MS;
397
- for (;;) {
398
- const rec = livingDaemon();
399
- if (rec !== undefined) {
400
- const health = await healthCheck(rec.port);
401
- if (health.ok) return { previous, record: rec, via: "systemctl" };
402
- }
403
- if (Date.now() >= deadline) break;
404
- await sleep(READY_POLL_MS);
405
- }
406
- throw new Error(
407
- `systemctl restart ${SYSTEMD_UNIT} returned, but the daemon never answered /healthz`,
408
- );
416
+ const record = await waitForOwnedDaemon("restart", o.timeoutMs);
417
+ return { previous, record, via: "systemctl" };
409
418
  }
410
419
 
420
+ // Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
421
+ // is somebody else. The detached CLI daemon is the only path left.
411
422
  await stopDaemon({ timeoutMs: o.timeoutMs });
412
423
  const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
413
424
  return { previous, record, via: "cli" };
414
425
  }
415
426
 
427
+ /**
428
+ * Restores a failed installed unit through the manager: `reset-failed` then
429
+ * `start`, and proves the result the same way any managed restart is proven
430
+ * (see {@link waitForOwnedDaemon}). Refusal or unproven ownership is terminal
431
+ * — never a fallthrough to the detached CLI daemon.
432
+ */
433
+ async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
434
+ const reset = systemctl(["reset-failed", SYSTEMD_UNIT]);
435
+ if (!reset.ok) {
436
+ throw new Error(systemctlFailure("reset-failed", reset));
437
+ }
438
+ const started = systemctl(["start", SYSTEMD_UNIT]);
439
+ if (!started.ok) {
440
+ throw new Error(systemctlFailure("start", started));
441
+ }
442
+ return await waitForOwnedDaemon("start", timeoutMs);
443
+ }
444
+
445
+ /**
446
+ * Waits for the daemon a manager call was just made to boot, and refuses to
447
+ * report success until the service manager is proven to own it: the unit's
448
+ * MainPID must equal the daemon record pid AND `/healthz` must answer. A unit
449
+ * that went back to `failed` is a confirmed negative and fails fast — waiting
450
+ * cannot un-fail it. Any other unproven state fails at the deadline with a
451
+ * diagnostic that names the mismatch, never a bare "not ready".
452
+ */
453
+ async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number): Promise<DaemonRecord> {
454
+ const via = `systemctl ${verb} ${SYSTEMD_UNIT} returned`;
455
+ const inspect = `systemctl status ${SYSTEMD_UNIT}`;
456
+ const deadline = Date.now() + (timeoutMs ?? READY_TIMEOUT_MS);
457
+ for (;;) {
458
+ const rec = livingDaemon();
459
+ const ownership = probeUnit();
460
+ if (rec !== undefined && ownership.kind === "active" && ownership.pid === rec.pid) {
461
+ const health = await healthCheck(rec.port);
462
+ if (health.ok) return rec;
463
+ } else if (ownership.kind === "failed") {
464
+ // A confirmed negative: the start did not take and the unit is failed
465
+ // again. Waiting longer cannot un-fail it.
466
+ throw new Error(
467
+ `${via}, but the unit went back to failed — the daemon is NOT running; check \`${inspect}\``,
468
+ );
469
+ }
470
+ if (Date.now() >= deadline) break;
471
+ await sleep(READY_POLL_MS);
472
+ }
473
+
474
+ // Failed to prove ownership. Name what is wrong rather than a bare timeout,
475
+ // so the operator knows whether to fix the unit or hunt the foreign pid.
476
+ const rec = livingDaemon();
477
+ const ownership = probeUnit();
478
+ if (rec === undefined) {
479
+ throw new Error(
480
+ `${via}, but no live daemon record appeared — the daemon did not come back; check \`${inspect}\``,
481
+ );
482
+ }
483
+ if (ownership.kind === "active") {
484
+ if (ownership.pid === rec.pid) {
485
+ throw new Error(
486
+ `${via}, but the daemon never answered /healthz on :${rec.port} — ` +
487
+ `the service manager owns pid ${rec.pid}, but it is not serving; ` +
488
+ `check \`${inspect}\` and ${rec.logFile}`,
489
+ );
490
+ }
491
+ throw new Error(
492
+ `${via}, but the unit's MainPID (${ownership.pid}) does not match the daemon record pid (${rec.pid}) — ` +
493
+ `the reported pid is NOT owned by the service manager; check \`${inspect}\``,
494
+ );
495
+ }
496
+ if (ownership.kind === "unknown") {
497
+ throw new Error(
498
+ `${via}, but ownership can no longer be determined (${ownership.reason}) — ` +
499
+ `cannot confirm the daemon is managed; check \`${inspect}\``,
500
+ );
501
+ }
502
+ throw new Error(`${via}, but the unit is not active; check \`${inspect}\``);
503
+ }
504
+
416
505
  /**
417
506
  * What we know about {@link SYSTEMD_UNIT}.
418
507
  *
@@ -420,14 +509,22 @@ export async function restartDaemon(
420
509
  * states (`activating`, `deactivating`, `reloading`, `reactivating`) — the
421
510
  * pid is still systemd-owned, so a raw SIGTERM would bounce under
422
511
  * `Restart=on-failure`.
423
- * - `inactive` — confirmed not running (MainPID absent/0), unit absent, *or*
424
- * no `systemctl` binary on this host. Safe to treat as "not supervised here".
512
+ * - `failed` — the unit is installed and its last activation failed
513
+ * (ActiveState=failed, no MainPID). systemd still owns this unit's
514
+ * lifecycle, so restoring the daemon must go through the manager
515
+ * (`reset-failed` + `start`); classifying a failed unit as "not supervised
516
+ * here" is how `restart` launched an unmanaged daemon next to a unit that
517
+ * stayed failed (#376).
518
+ * - `inactive` — confirmed not running (MainPID absent/0, ActiveState not
519
+ * `failed`), unit absent, *or* no `systemctl` binary on this host. Safe to
520
+ * treat as "not supervised here".
425
521
  * - `unknown` — the manager exists (or we cannot tell it does not) but the
426
522
  * query failed: dbus blip, permission, timeout. Must not be collapsed into
427
523
  * `inactive` — that is how a unit-owned daemon gets a raw SIGTERM.
428
524
  */
429
525
  export type UnitOwnership =
430
526
  | { kind: "active"; pid: number }
527
+ | { kind: "failed" }
431
528
  | { kind: "inactive" }
432
529
  | { kind: "unknown"; reason: string };
433
530
 
@@ -454,26 +551,39 @@ export function probeUnit(unit = SYSTEMD_UNIT): UnitOwnership {
454
551
  // live pid (> 1) means systemd still owns that process — including during
455
552
  // `activating` / `deactivating` / `reloading`. Filtering on ActiveState here
456
553
  // used to label those transitional states "inactive" and hand the pid to a
457
- // raw SIGTERM, which is exactly the Restart=on-failure bounce.
554
+ // raw SIGTERM, which is exactly the Restart=on-failure bounce. The one
555
+ // place ActiveState decides is the `failed` classification below: with no
556
+ // MainPID there is no pid to hand to anything, but the unit has not gone
557
+ // away either.
458
558
  const lines = ran.stdout
459
559
  .split("\n")
460
560
  .map((l) => l.trim())
461
561
  .filter((l) => l.length > 0);
462
562
  let pid: number | undefined;
563
+ let activeState: string | undefined;
463
564
  for (const line of lines) {
464
565
  if (/^\d+$/.test(line)) {
465
566
  const n = Number(line);
466
567
  if (Number.isInteger(n) && n > 1) pid = n;
568
+ } else {
569
+ activeState = line;
467
570
  }
468
571
  }
469
- if (pid === undefined) return { kind: "inactive" };
470
- return { kind: "active", pid };
572
+ if (pid !== undefined) return { kind: "active", pid };
573
+ // A failed unit is NOT the confirmed negative `inactive` claims to be: the
574
+ // unit is installed and the manager still owns its lifecycle, so restoring
575
+ // the daemon has to go through systemd (`reset-failed` + `start`). Reading
576
+ // `failed` as "not supervised here" is precisely how #376 ended with a
577
+ // healthy-looking but unmanaged pid next to a unit that stayed failed.
578
+ if (activeState === "failed") return { kind: "failed" };
579
+ return { kind: "inactive" };
471
580
  }
472
581
 
473
582
  /**
474
583
  * The MainPID of an *active* {@link SYSTEMD_UNIT}, or `undefined` when the
475
- * unit is confirmed inactive/absent or when ownership could not be determined.
476
- * Prefer {@link probeUnit} when the caller must distinguish those two.
584
+ * unit is confirmed inactive, failed, or absent, or when ownership could not
585
+ * be determined. Prefer {@link probeUnit} when the caller must distinguish
586
+ * those two.
477
587
  */
478
588
  export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
479
589
  const ownership = probeUnit(unit);
@@ -490,8 +600,9 @@ export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
490
600
  * - `stop` — unit is active and owns `pid` (or there is no pidfile and the
491
601
  * unit is the only candidate). Caller MUST go through systemctl; a failed
492
602
  * manager call is terminal.
493
- * - `not-ours` — confirmed inactive/absent unit, no systemd binary, or a unit
494
- * whose MainPID is someone else. Caller may SIGTERM its own pidfile process.
603
+ * - `not-ours` — confirmed inactive/failed/absent unit, no systemd binary, or
604
+ * a unit whose MainPID is someone else. Caller may SIGTERM its own pidfile
605
+ * process (a failed unit owns nothing, so signalling its record is safe).
495
606
  * - `unknown` — ownership query failed. Caller MUST NOT signal.
496
607
  */
497
608
  type SystemdStopDecision =
@@ -502,7 +613,10 @@ type SystemdStopDecision =
502
613
  function decideSystemdStop(pid: number | undefined): SystemdStopDecision {
503
614
  const ownership = probeUnit();
504
615
  if (ownership.kind === "unknown") return { kind: "unknown", reason: ownership.reason };
505
- if (ownership.kind === "inactive") return { kind: "not-ours" };
616
+ // A failed unit owns no process (MainPID 0 systemd has already reaped or
617
+ // lost it), so a live record pid is a confirmed "not ours", exactly like an
618
+ // inactive unit. Restoring the unit is restart's job, not stop's.
619
+ if (ownership.kind === "inactive" || ownership.kind === "failed") return { kind: "not-ours" };
506
620
  if (pid !== undefined && ownership.pid !== pid) return { kind: "not-ours" };
507
621
  return { kind: "stop", mainPid: ownership.pid };
508
622
  }
@@ -533,11 +647,17 @@ async function runSystemdStop(pid: number, timeoutMs?: number): Promise<void> {
533
647
  }
534
648
  }
535
649
 
536
- function systemctlFailure(verb: "stop" | "restart", ran: SystemctlResult): string {
650
+ function systemctlFailure(
651
+ verb: "stop" | "restart" | "reset-failed" | "start",
652
+ ran: SystemctlResult,
653
+ ): string {
537
654
  const detail = (ran.stderr.trim() || ran.stdout.trim() || "no output").split("\n")[0] ?? "no output";
655
+ const guard =
656
+ verb === "stop" || verb === "restart"
657
+ ? `refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce)`
658
+ : `the unit was NOT restored, so the daemon would be unmanaged`;
538
659
  return (
539
- `systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ` +
540
- `refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce); ` +
660
+ `systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ${guard}; ` +
541
661
  `fix the unit or run \`systemctl ${verb} ${SYSTEMD_UNIT}\` yourself`
542
662
  );
543
663
  }
package/src/omp.ts CHANGED
@@ -398,6 +398,15 @@ const DRAIN_GRACE_MS = 2_000;
398
398
  /** How long a disposed child gets to exit before it is signalled. */
399
399
  const DISPOSE_GRACE_MS = 5_000;
400
400
 
401
+ /**
402
+ * Thrown by {@link createSession} when the pre-spawn admission gate closes: a
403
+ * daemon stop landed while the session socket was binding, so the child was
404
+ * never spawned. The caller maps this to a stopped run, not a failed one —
405
+ * a shutdown must not charge an attempt against a worker it refused to start
406
+ * (#374).
407
+ */
408
+ export class SessionAdmissionClosedError extends Error {}
409
+
401
410
  export interface CreateSessionOptions {
402
411
  cwd: string;
403
412
  sessionDir?: string;
@@ -430,6 +439,16 @@ export interface CreateSessionOptions {
430
439
  * its verb channel to this pid, and a channel accepts nothing until it is bound.
431
440
  */
432
441
  onSpawn?: (pid: number) => void;
442
+ /**
443
+ * Pre-spawn admission gate (#374). Consulted once, immediately after the
444
+ * socket bind await and immediately before `Bun.spawn` — the last window a
445
+ * daemon stop can land in before a child exists. A gate that returns false
446
+ * closes the listener, removes the socket, and throws
447
+ * {@link SessionAdmissionClosedError} instead of spawning; the run then
448
+ * settles as stopped rather than adding a worker the shutdown would have to
449
+ * wait for. Absent, the spawn always proceeds.
450
+ */
451
+ maySpawn?: () => boolean;
433
452
  /** Child stderr, line by line. Defaults to the process's own stderr. */
434
453
  onChildLog?: (line: string) => void;
435
454
  startupTimeoutMs?: number;
@@ -505,6 +524,21 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
505
524
  resolve();
506
525
  });
507
526
  });
527
+ // Pre-spawn admission recheck (#374): SIGTERM/SIGINT can land during the
528
+ // listen await above — after the daemon's own pre-launch checks and before
529
+ // this child exists. A closed gate means no child: close the listener,
530
+ // remove the socket (and, when this side created the directory, the
531
+ // directory itself — the same cleanup the failure paths below run), and
532
+ // signal the caller to settle the run as stopped instead of spawning a
533
+ // worker the shutdown would then have to wait for.
534
+ if (opts.maySpawn?.() === false) {
535
+ server.close();
536
+ rmSync(socketPath, { force: true });
537
+ if (owned) rmSync(socketDir, { recursive: true, force: true });
538
+ throw new SessionAdmissionClosedError(
539
+ "daemon shutdown began while the session socket was binding",
540
+ );
541
+ }
508
542
  // The socket is the run's own channel, and the daemon's own uid is the only
509
543
  // one that speaks on it.
510
544
  chmodSync(socketPath, 0o600);
package/src/setup-host.ts CHANGED
@@ -314,21 +314,49 @@ function tickSearchRoots(project: ProjectConfig): string[] {
314
314
  * have been a deliberate per-project choice — it *is* the collision — so it is
315
315
  * rewritten; anything else is operator intent and survives untouched.
316
316
  */
317
- function planTick(project: ProjectConfig, telegramStateDir: string): PlannedWrite<TickConfig> {
318
- let existing: { path: string; config: TickConfig } | undefined;
317
+ /**
318
+ * This project's tick config, wherever it actually lives. A config stamped for
319
+ * another project is that project's file: the search roots overlap, and
320
+ * restamping it here would hand this project's identity to the other fleet's cwd.
321
+ */
322
+ function findProjectTick(
323
+ project: ProjectConfig,
324
+ ): { root: string; path: string; config: TickConfig } | undefined {
319
325
  for (const root of tickSearchRoots(project)) {
320
326
  const result = readTickConfig(root);
321
327
  if (result.kind === "invalid") {
322
328
  throw new Error(`tick config invalid at ${result.path}: ${result.problem}; fix or remove it before setup`);
323
329
  }
324
- // A config stamped for another project is that project's file: the search
325
- // roots overlap, and restamping it here would hand this project's identity
326
- // to the other fleet's cwd.
327
330
  if (result.kind === "ok" && tickConfigMatchesProject(result.config, project.name)) {
328
- existing = { path: result.path, config: result.config };
329
- break;
331
+ return { root, path: result.path, config: result.config };
330
332
  }
331
333
  }
334
+ return undefined;
335
+ }
336
+
337
+ /**
338
+ * The directory this project's fleet pane runs in, which is **not** always
339
+ * `workspaceRoot`.
340
+ *
341
+ * A fleet that predates per-project roots keeps its `.conductor-tick.json` in the
342
+ * state dir, and that is the cwd its pane and `FLEET_CWD` already point at. Naming
343
+ * `workspaceRoot` instead sends `recover.sh` somewhere with no tick config, where
344
+ * `tick_agent_name` falls back to `fleet` and no longer matches the renamed pane —
345
+ * silently costing a live project its recovery. Falls back to `workspaceRoot`,
346
+ * which is where a brand-new project's config is written.
347
+ */
348
+ export function tickCwdForProject(project: ProjectConfig): string {
349
+ try {
350
+ return findProjectTick(project)?.root ?? project.workspaceRoot;
351
+ } catch {
352
+ // An invalid config is setup's problem to report, not the handoff's.
353
+ return project.workspaceRoot;
354
+ }
355
+ }
356
+
357
+ function planTick(project: ProjectConfig, telegramStateDir: string): PlannedWrite<TickConfig> {
358
+ const found = findProjectTick(project);
359
+ const existing = found === undefined ? undefined : { path: found.path, config: found.config };
332
360
 
333
361
  const armedFile = join(stateDir(), `armed-${project.name}`);
334
362
  const path = existing?.path ?? join(project.workspaceRoot, TICK_CONFIG_FILE);
@@ -37,6 +37,7 @@ import {
37
37
  totalConfiguredWorkers,
38
38
  runSetupSmoke,
39
39
  SYSTEMD_UNIT_DIR,
40
+ tickCwdForProject,
40
41
  writeHostRuntime,
41
42
  } from "./setup-host.ts";
42
43
  import { runGraphInstall, runHostInstall } from "./setup-install.ts";
@@ -1639,18 +1640,22 @@ export async function collectSetup(
1639
1640
  */
1640
1641
  export function formatHerdrHandoff(project: ProjectConfig, cfg: ConductorConfig): string {
1641
1642
  const session = process.env["HERDR_SESSION"] ?? "conductor";
1642
- const cwds = cfg.projects.map((p) => p.workspaceRoot).join(":");
1643
+ // Every entry is the cwd that owns that project's tick config — the directory its
1644
+ // pane runs in — not `workspaceRoot`. They differ on any fleet whose config
1645
+ // predates per-project roots, and recovery keys on the tick config.
1646
+ const paneCwd = tickCwdForProject(project);
1647
+ const cwds = cfg.projects.map((p) => tickCwdForProject(p)).join(":");
1643
1648
  return [
1644
1649
  "herdr handoff (CLI cannot do these — exact herdr argv):",
1645
- ` 1. In session "${session}", create a workspace for project "${project.name}" at its workspaceRoot:`,
1646
- ` herdr --session ${session} workspace create --cwd ${project.workspaceRoot} --label ${project.name} --no-focus`,
1650
+ ` 1. In session "${session}", create a workspace for project "${project.name}" at its fleet cwd:`,
1651
+ ` herdr --session ${session} workspace create --cwd ${paneCwd} --label ${project.name} --no-focus`,
1647
1652
  " Read root_pane.pane_id from the JSON reply (jq -r '((.result // .).root_pane.pane_id) // empty').",
1648
1653
  ` 2. Start omp in that empty pane. The agent NAME must match tick config agentName ("${project.name}"):`,
1649
1654
  ` herdr --session ${session} agent start ${project.name} --kind omp --pane <pane-id>`,
1650
1655
  " The pane must be at a shell prompt with no agent on it. Never agent-start into a live orchestrator pane.",
1651
1656
  ` 3. Point herdr-conductor recovery at every fleet cwd (colon-separated; see #320):`,
1652
1657
  ` FLEET_CWDS=${cwds}`,
1653
- ` Legacy single-fleet hosts can keep FLEET_CWD=${project.workspaceRoot} until multi-fleet recovery lands.`,
1658
+ ` Legacy single-fleet hosts can keep FLEET_CWD=${paneCwd} until multi-fleet recovery lands.`,
1654
1659
  " 4. Reload the daemon so it serves every configured project:",
1655
1660
  " omp-conductor restart --now",
1656
1661
  " (printed, not auto-run, when workers are live or a neighbour was just added)",
package/src/types.ts CHANGED
@@ -1154,6 +1154,7 @@ export type AdmissionHoldReason =
1154
1154
  | "unsalvaged-wip"
1155
1155
  | "daily-spend-cap"
1156
1156
  | "plan-usage-cap"
1157
+ | "shutting-down"
1157
1158
  | "unroutable:no-repo-label"
1158
1159
  | "unroutable:multiple-repo-labels"
1159
1160
  | "unroutable:unknown-repo";
package/src/upgrade.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
2
  import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
3
3
  import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
4
- import { setPaused, statusSnapshot } from "./daemon.ts";
4
+ import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
5
5
  import {
6
6
  DEFAULT_HERDR_SESSION,
7
7
  fleetLayers,
@@ -52,7 +52,21 @@ export interface UpgradeDeps {
52
52
  snapshot(project?: string): { liveWorkers: number };
53
53
  layers(project?: string): FleetLayers;
54
54
  brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
55
- daemonIdentity(): { running: boolean; project?: string };
55
+ /**
56
+ * The daemon the transaction targets, when one is running. `generation`
57
+ * identifies the exact instance (pid + start time) so a later restart cannot
58
+ * stop an instance created after the request began (#377).
59
+ */
60
+ daemonIdentity(): { running: boolean; project?: string; generation?: string };
61
+ /**
62
+ * The durable pause sentinel this fleet currently holds, as an *instance*,
63
+ * read from one selected file (the per-project sentinel first): who set it,
64
+ * why, and when that sentinel was created. Two pauses with identical
65
+ * provenance are still different instances if `since` differs, so a drain
66
+ * can tell its own sentinel from one a later actor re-created after a
67
+ * resume lifted it — even when a legacy global sentinel coexists (#377).
68
+ */
69
+ pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
56
70
  setPaused(value: boolean, project?: string): void;
57
71
  restartDaemon(): Promise<void>;
58
72
  sleep(ms: number): Promise<void>;
@@ -91,8 +105,14 @@ export const DEFAULT_DEPS: UpgradeDeps = {
91
105
  },
92
106
  daemonIdentity: () => {
93
107
  const daemon = livingDaemon();
94
- return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
108
+ if (daemon === undefined) return { running: false };
109
+ return {
110
+ running: true,
111
+ project: daemon.project,
112
+ generation: `${daemon.pid}@${daemon.startedAt}`,
113
+ };
95
114
  },
115
+ pauseState: (project) => pauseInstance(project),
96
116
  setPaused: (v, project) =>
97
117
  setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
98
118
  restartDaemon: async () => {
@@ -248,9 +268,20 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
248
268
  ];
249
269
  }
250
270
 
251
- async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: number): Promise<void> {
271
+ async function waitForDrain(
272
+ deps: UpgradeDeps,
273
+ project?: string,
274
+ deadlineAt?: number,
275
+ stale?: () => string | undefined,
276
+ ): Promise<void> {
252
277
  let last = -1;
253
278
  while (true) {
279
+ // A transaction that is no longer the fleet's current intent exits before
280
+ // it waits on anyone else's workers: a resume lifted its pause or the
281
+ // daemon it began with was replaced, so there is nothing left to drain
282
+ // and nothing it may stop (#377).
283
+ const cancelled = stale?.();
284
+ if (cancelled !== undefined) throw new Error(cancelled);
254
285
  const workers = deps.snapshot(project).liveWorkers;
255
286
  if (workers === 0) return;
256
287
  if (deadlineAt !== undefined && Date.now() >= deadlineAt) {
@@ -264,10 +295,71 @@ async function waitForDrain(deps: UpgradeDeps, project?: string, deadlineAt?: nu
264
295
  }
265
296
  }
266
297
 
298
+ /**
299
+ * What a draining restart captured when it began: the restart-owned pause and
300
+ * the daemon instance it is entitled to stop. Every later fence check compares
301
+ * live state against this snapshot, so the destructive restart call only ever
302
+ * acts on the world the request began in (#377).
303
+ */
304
+ interface RestartBegun {
305
+ /** The durable pause sentinel this request relies on, proved readable as an instance. */
306
+ pauseToken: { source: string; reason?: string; since: number };
307
+ /** The daemon instance the restart targets, with its generation identity. */
308
+ daemon: ReturnType<UpgradeDeps["daemonIdentity"]>;
309
+ }
310
+
311
+ /**
312
+ * Why a draining restart can no longer act, or `undefined` when it still owns
313
+ * the pause and the daemon generation it began with. The pause check is the
314
+ * *restart-owned sentinel instance* — `drainAndRestart` proves ownership up
315
+ * front, so a token that cannot be re-read (sentinel removed, unreadable, or
316
+ * re-created — even under the same source and reason) means this request's own
317
+ * pause no longer exists and it must cancel. The generation check is the
318
+ * later-daemon fence: a daemon created after this request began — by
319
+ * `restart --now`, a crash-restart, or an operator start — must never be
320
+ * stopped by it.
321
+ */
322
+ function restartFenceProblem(
323
+ deps: UpgradeDeps,
324
+ project: string | undefined,
325
+ begun: RestartBegun,
326
+ ): string | undefined {
327
+ if (!deps.layers(project).paused) {
328
+ return "restart cancelled: dispatch was resumed while the drain was in progress — nothing was restarted";
329
+ }
330
+ const owned = deps.pauseState(project);
331
+ if (
332
+ owned === undefined ||
333
+ owned.since !== begun.pauseToken.since ||
334
+ owned.source !== begun.pauseToken.source ||
335
+ owned.reason !== begun.pauseToken.reason
336
+ ) {
337
+ return "restart cancelled: the restart-owned pause was lifted and replaced while the drain was in progress — nothing was restarted";
338
+ }
339
+ const now = deps.daemonIdentity();
340
+ if (now.running !== begun.daemon.running || (begun.daemon.running && now.generation !== begun.daemon.generation)) {
341
+ return "restart cancelled: the daemon instance was replaced while the drain was in progress — the newer daemon was left running";
342
+ }
343
+ return undefined;
344
+ }
345
+
267
346
  /**
268
347
  * Pause, drain, restart, restore — the trusted restart transaction, minus the
269
348
  * install/verify steps of {@link upgradeConductor}.
270
349
  *
350
+ * Generation-scoped (#377): the transaction captures the restart-owned pause
351
+ * and the daemon generation when it begins and revalidates both on every drain
352
+ * poll and once more immediately before the destructive restart call. A
353
+ * `resume`, a `restart --now`, or any other action that lifts the pause or
354
+ * replaces the daemon while the drain waits makes the transaction stale: it
355
+ * exits nonzero without stopping anything, so a timed-out or disconnected
356
+ * restart caller cannot leave a delayed stop aimed at a later daemon
357
+ * generation. The stale transaction restores nothing and leaves whatever
358
+ * pause the fleet now has exactly as it found it. Ownership is proven up
359
+ * front: a sentinel that cannot be read as an instance at transaction start
360
+ * aborts before any drain or restart, because a lock the fence cannot compare
361
+ * against is a lock the fence cannot protect.
362
+ *
271
363
  * Mirrors the upgrade's fail-closed posture: on ANY throw after the pause the
272
364
  * fleet stays paused (no resume in a catch) and the error rethrows, so a
273
365
  * systemd-owned restart that fails or a drain that outlives `timeoutMs` cannot
@@ -280,7 +372,26 @@ export async function drainAndRestart(
280
372
  ): Promise<void> {
281
373
  const initial = deps.layers(o.project);
282
374
  if (!initial.paused) deps.setPaused(true, o.project);
283
- await waitForDrain(deps, o.project, Date.now() + o.timeoutMs);
375
+ // The lock must be provable NOW. A pause that cannot be read as an instance
376
+ // — sentinel malformed or unreadable — would make the whole fence fail open
377
+ // (nothing to compare against), so the transaction refuses before waiting
378
+ // on anything it cannot act upon either way.
379
+ const pauseToken = deps.pauseState(o.project);
380
+ if (pauseToken === undefined) {
381
+ throw new Error(
382
+ "restart cancelled: cannot prove the restart-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted",
383
+ );
384
+ }
385
+ const begun: RestartBegun = {
386
+ pauseToken,
387
+ daemon: deps.daemonIdentity(),
388
+ };
389
+ const stale = () => restartFenceProblem(deps, o.project, begun);
390
+ await waitForDrain(deps, o.project, Date.now() + o.timeoutMs, stale);
391
+ // The drain completed; the world may have moved on while it did. Re-prove
392
+ // the pause and the generation an instant before the destructive call.
393
+ const cancelled = stale();
394
+ if (cancelled !== undefined) throw new Error(cancelled);
284
395
  await deps.restartDaemon();
285
396
  if (!initial.paused) deps.setPaused(false, o.project);
286
397
  }
package/src/worker.ts CHANGED
@@ -10,7 +10,7 @@
10
10
  * sliding into a merge queue.
11
11
  */
12
12
 
13
- import { createSession, disposeSession } from "./omp.ts";
13
+ import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
14
14
  import type { ReleaseBlockContext } from "./release-policy.ts";
15
15
  import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
16
16
 
@@ -123,6 +123,13 @@ export interface WorkerOpts {
123
123
  * exists; absent = no pause surface (tests, one-shot callers).
124
124
  */
125
125
  onPauseControl?: (control: WorkerPauseControl) => void;
126
+ /**
127
+ * Pre-spawn admission gate, forwarded to `createSession` (#374): the daemon
128
+ * closes it the moment its stop begins, so a stop that lands while the
129
+ * session socket is binding refuses the spawn and the run settles `stopped`
130
+ * instead of `failed`. Absent, the spawn always proceeds.
131
+ */
132
+ maySpawn?: () => boolean;
126
133
  }
127
134
 
128
135
  /**
@@ -253,20 +260,39 @@ export async function runWorker(
253
260
  const now = deps.now ?? Date.now;
254
261
  const schedule = deps.schedule ?? scheduleWallClock;
255
262
 
256
- const session = await deps.createSession({
257
- cwd: o.cwd,
258
- ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
259
- ...(o.model === undefined ? {} : { model: o.model }),
260
- // Prevention half of #24: as a worker, structured file tools cannot leave
261
- // this worktree, and no release grant can ever reach this session (#122).
262
- role: "worker",
263
- ...(o.releaseGrants === undefined ? {} : { releaseGrants: o.releaseGrants }),
264
- ...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
265
- ...(o.onSpawn === undefined ? {} : { onSpawn: o.onSpawn }),
266
- ...(o.socketPath === undefined ? {} : { socketPath: o.socketPath }),
267
- ...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
268
- ...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
269
- });
263
+ let session: AgentSessionLike;
264
+ try {
265
+ session = await deps.createSession({
266
+ cwd: o.cwd,
267
+ ...(o.sessionDir === undefined ? {} : { sessionDir: o.sessionDir }),
268
+ ...(o.model === undefined ? {} : { model: o.model }),
269
+ // Prevention half of #24: as a worker, structured file tools cannot leave
270
+ // this worktree, and no release grant can ever reach this session (#122).
271
+ role: "worker",
272
+ ...(o.releaseGrants === undefined ? {} : { releaseGrants: o.releaseGrants }),
273
+ ...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
274
+ ...(o.onSpawn === undefined ? {} : { onSpawn: o.onSpawn }),
275
+ ...(o.socketPath === undefined ? {} : { socketPath: o.socketPath }),
276
+ ...(o.verbSocketPath === undefined ? {} : { verbSocketPath: o.verbSocketPath }),
277
+ ...(o.onChildLog === undefined ? {} : { onChildLog: o.onChildLog }),
278
+ ...(o.maySpawn === undefined ? {} : { maySpawn: o.maySpawn }),
279
+ });
280
+ } catch (err) {
281
+ // The pre-spawn gate closed (#374): a daemon stop landed while the session
282
+ // socket was binding, so no child was ever created. That is a stop, not a
283
+ // failure — reporting `failed` here would charge an attempt against a
284
+ // worker the shutdown refused to start. `onSpawn` was never called.
285
+ if (err instanceof SessionAdmissionClosedError) {
286
+ return {
287
+ state: "stopped",
288
+ turns: 0,
289
+ spendUsd: 0,
290
+ report: "",
291
+ stoppedReason: "daemon shutdown began before the worker session started",
292
+ };
293
+ }
294
+ throw err;
295
+ }
270
296
 
271
297
  // Before the first turn, not after the last: a caller that only learns the
272
298
  // transcript path from the result learns it once the run it wanted to watch