omp-conductor 0.3.7 → 0.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -11,7 +11,7 @@ import { createHash } from "node:crypto";
11
11
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
12
  import { dirname, join, relative } from "node:path";
13
13
  import { configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
14
- import { createEscalator } from "./escalate.ts";
14
+ import { createEscalator, escalationIssueRef } from "./escalate.ts";
15
15
  import { graphHint } from "./graph.ts";
16
16
  import { livingDaemon } from "./lifecycle.ts";
17
17
  import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
@@ -369,7 +369,7 @@ async function safeEscalate(d: Pick<Deps, "escalate">, e: Escalation): Promise<b
369
369
  await d.escalate(e);
370
370
  return true;
371
371
  } catch (err) {
372
- log(`escalation for #${e.issue} could not be delivered: ${errText(err)}`);
372
+ log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
373
373
  return false;
374
374
  }
375
375
  }
@@ -393,13 +393,22 @@ export function salvageLines(outcome: SalvageOutcome, worktree: string): string[
393
393
  ];
394
394
  }
395
395
 
396
- return [
396
+ const where =
397
397
  `WIP committed to ${outcome.branch} @ ${outcome.sha}` +
398
- (outcome.pushed
399
- ? " and pushed — the work outlives this worktree"
400
- : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`),
401
- kept,
402
- ];
398
+ (outcome.pushed
399
+ ? " and pushed — the work outlives this worktree"
400
+ : ` but NOT pushed (${outcome.pushError ?? "no reason given"}) — it lives only in this host's mirror`);
401
+ // Manifest belongs in the escalation too: opening the commit is how the
402
+ // orchestrator talked itself into scrubbing a worker tree (#38).
403
+ const n = outcome.files.length;
404
+ const count = `${n} file${n === 1 ? "" : "s"}`;
405
+ const manifest =
406
+ outcome.newPaths.length === 0
407
+ ? `${count} (all modifications to tracked paths)`
408
+ : `${count}; new: ${outcome.newPaths.slice(0, 12).join(", ")}${
409
+ outcome.newPaths.length > 12 ? `, … +${outcome.newPaths.length - 12} more` : ""
410
+ }`;
411
+ return [where, manifest, kept];
403
412
  }
404
413
 
405
414
  /**
@@ -444,10 +453,35 @@ export async function buildBrief(
444
453
  r: Routed,
445
454
  branch: string,
446
455
  worktree: string,
456
+ opts: { continuation?: boolean; defaultBranch?: string } = {},
447
457
  ): Promise<string> {
448
458
  // Read per dispatch rather than caching: editing the brief then takes effect
449
459
  // on the next issue instead of needing a daemon restart.
450
460
  const template = await Bun.file(BRIEF_TEMPLATE_PATH).text();
461
+ const defaultBranch = opts.defaultBranch ?? r.repo.defaultBranch;
462
+ const continuation =
463
+ opts.continuation === true
464
+ ? [
465
+ "",
466
+ "## Continuation — do not start from zero",
467
+ "",
468
+ `You are **resuming** issue #${r.issue.number}. Branch \`${branch}\` already exists`,
469
+ "and was reattached with prior commits (and possibly a salvaged WIP tip).",
470
+ "Before writing anything:",
471
+ "",
472
+ "```bash",
473
+ "git log --oneline origin/" + defaultBranch + "..HEAD",
474
+ "git diff --stat origin/" + defaultBranch + "...HEAD",
475
+ "git status --porcelain",
476
+ "```",
477
+ "",
478
+ "Read that history. **Do not recreate work that already exists.** Finish",
479
+ "what remains against the same acceptance criteria. If a prior attempt",
480
+ "left a `wip(#…): … auto-salvaged` commit, treat it as your starting",
481
+ "point, not as trash to rewrite from scratch.",
482
+ "",
483
+ ].join("\n")
484
+ : "";
451
485
  return renderBrief(template, {
452
486
  ISSUE_NUMBER: String(r.issue.number),
453
487
  ISSUE_TITLE: r.issue.title,
@@ -461,6 +495,7 @@ export async function buildBrief(
461
495
  // placeholder sits flush against the next list item in the template, so an
462
496
  // unconfigured render leaves no blank line where a hint would have gone.
463
497
  GRAPH_HINT: graphHint(r.repo),
498
+ CONTINUATION: continuation,
464
499
  });
465
500
  }
466
501
 
@@ -513,14 +548,14 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
513
548
  // would cost a second network fetch per attempt.
514
549
  const mirrorPath = mirrorPathFor(r.repo, project.mirrorRoot);
515
550
  await removeWorktree(mirrorPath, worktreePathFor(project.workspaceRoot, issue));
516
-
517
- worktreePath = await addWorktree(
551
+ const provisioned = await addWorktree(
518
552
  r.repo,
519
553
  project.mirrorRoot,
520
554
  project.workspaceRoot,
521
555
  issue,
522
556
  branch,
523
557
  );
558
+ worktreePath = provisioned.path;
524
559
 
525
560
  // The SDK names the transcript itself, so the daemon supplies the parent
526
561
  // directory and learns the real path back from the result. Inventing one
@@ -529,10 +564,16 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
529
564
  mkdirSync(sessionDir, { recursive: true });
530
565
  store.updateRun(runId, { worktree: worktreePath, state: "running" });
531
566
 
532
- log(`#${issue} attempt ${attempt} → ${r.repo.name} ${branch}`);
567
+ log(
568
+ `#${issue} attempt ${attempt} → ${r.repo.name} ${branch}` +
569
+ (provisioned.reattached ? " (continuation: reattached existing branch)" : ""),
570
+ );
533
571
 
534
572
  const result = await runWorker({
535
- brief: await buildBrief(project, r, branch, worktreePath),
573
+ brief: await buildBrief(project, r, branch, worktreePath, {
574
+ continuation: provisioned.reattached,
575
+ defaultBranch: r.repo.defaultBranch,
576
+ }),
536
577
  cwd: worktreePath,
537
578
  caps,
538
579
  sessionDir,
@@ -572,30 +613,61 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
572
613
  detail: [`${r.issue.title}`, r.issue.url, "", result.report].join("\n"),
573
614
  });
574
615
  } else if (result.state === "failed" || result.state === "killed") {
575
- await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
576
- // Before the escalation is composed, so it can say where the work went —
577
- // and long before the next attempt provisions over this tree.
616
+ // Turns-cap with attempts left: salvage, put the issue back on the queue,
617
+ // and skip the failed label so the next tick reclaims as a continuation
618
+ // instead of burning a human triage cycle (#50 / #21).
619
+ const continueTurns =
620
+ result.killedBy === "turns" && attempt < caps.maxAttemptsPerIssue;
578
621
  const salvaged = await salvage(issue, attempt, endedBy(result.killedBy), worktreePath);
579
- await safeEscalate(d, {
580
- tier: 1,
581
- project: project.name,
582
- issue,
583
- runId,
584
- // The dedup key includes the summary, so the attempt number is what
585
- // lets a genuine second failure page again while a tick that keeps
586
- // seeing the same dead issue stays quiet.
587
- summary: result.killedBy
588
- ? `#${issue} was killed on attempt ${attempt} by the ${result.killedBy} cap`
589
- : `#${issue} failed on attempt ${attempt}`,
590
- detail: [
591
- `${r.issue.title}`,
592
- r.issue.url,
593
- ...salvaged,
594
- `Session: ${result.sessionFile ?? "(no transcript)"}`,
595
- "",
596
- result.report,
597
- ].join("\n"),
598
- });
622
+
623
+ if (continueTurns) {
624
+ await tracker.removeLabel(issue, inProgress);
625
+ await tracker.addLabel(issue, project.queueLabel);
626
+ log(
627
+ `#${issue} turns-cap on attempt ${attempt}/${caps.maxAttemptsPerIssue} ` +
628
+ `salvaged and re-queued for continuation`,
629
+ );
630
+ await safeEscalate(d, {
631
+ tier: 1,
632
+ project: project.name,
633
+ issue,
634
+ runId,
635
+ summary: `#${issue} hit the turns cap on attempt ${attempt} — auto-requeued for continuation`,
636
+ detail: [
637
+ `${r.issue.title}`,
638
+ r.issue.url,
639
+ ...salvaged,
640
+ `Session: ${result.sessionFile ?? "(no transcript)"}`,
641
+ "",
642
+ "The queue label is back on; the next tick should reattach the branch",
643
+ "and open a continuation brief. No failed label was applied.",
644
+ "",
645
+ result.report,
646
+ ].join("\n"),
647
+ });
648
+ } else {
649
+ await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
650
+ await safeEscalate(d, {
651
+ tier: 1,
652
+ project: project.name,
653
+ issue,
654
+ runId,
655
+ // The dedup key includes the summary, so the attempt number is what
656
+ // lets a genuine second failure page again while a tick that keeps
657
+ // seeing the same dead issue stays quiet.
658
+ summary: result.killedBy
659
+ ? `#${issue} was killed on attempt ${attempt} by the ${result.killedBy} cap`
660
+ : `#${issue} failed on attempt ${attempt}`,
661
+ detail: [
662
+ `${r.issue.title}`,
663
+ r.issue.url,
664
+ ...salvaged,
665
+ `Session: ${result.sessionFile ?? "(no transcript)"}`,
666
+ "",
667
+ result.report,
668
+ ].join("\n"),
669
+ });
670
+ }
599
671
  } else {
600
672
  // pushed-green: the PR belongs to a human now. The in-progress label
601
673
  // stays on until the merge closes the issue, which is also what keeps
@@ -923,9 +995,10 @@ async function tick(d: Deps): Promise<void> {
923
995
 
924
996
  // Spend is the one cap that stops the fleet instead of merely deferring work.
925
997
  // A loop that is burning money has to halt itself; waiting for a human to
926
- // notice tomorrow is how a runaway becomes expensive.
998
+ // notice tomorrow is how a runaway becomes expensive. `null` means the
999
+ // operator opted out — turns and wall-clock still brake every run (#46).
927
1000
  const spent = store.spendSince(project.name, since);
928
- if (spent >= caps.dailySpendUsd) {
1001
+ if (caps.dailySpendUsd !== null && spent >= caps.dailySpendUsd) {
929
1002
  setPaused(true);
930
1003
  await safeEscalate(d, {
931
1004
  tier: 2,
@@ -1011,7 +1084,9 @@ export function formatStatus(s: StatusSnapshot): string {
1011
1084
  "caps",
1012
1085
  ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`,
1013
1086
  ` issues today ${s.runsToday}`,
1014
- ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
1087
+ s.caps.dailySpendUsd === null
1088
+ ? ` spend today $${s.spendTodayUsd.toFixed(2)} (no daily cap)`
1089
+ : ` spend today $${s.spendTodayUsd.toFixed(2)} / $${s.caps.dailySpendUsd.toFixed(2)}`,
1015
1090
  ` worker max turns ${s.caps.workerMaxTurns}`,
1016
1091
  ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
1017
1092
  ` attempts per issue ${s.caps.maxAttemptsPerIssue}`,
@@ -1029,6 +1104,15 @@ export function formatStatus(s: StatusSnapshot): string {
1029
1104
  );
1030
1105
  }
1031
1106
  }
1107
+ // Deploy hint: a restart while workers are live orphans them (salvage runs
1108
+ // first — #35). Prefer pause + drain to zero live workers when you can wait.
1109
+ if (s.liveWorkers > 0) {
1110
+ lines.push(
1111
+ "",
1112
+ `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` +
1113
+ `pause and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`,
1114
+ );
1115
+ }
1032
1116
  return lines.join("\n");
1033
1117
  }
1034
1118
 
@@ -1085,18 +1169,18 @@ export function armConductor(): void {
1085
1169
  }
1086
1170
 
1087
1171
  /**
1088
- * Settles the runs a previous daemon process left in flight.
1172
+ * Settles `claimed`/`running` rows left by a dead daemon process and, before
1173
+ * marking each one `orphaned`, salvages any dirty worktree.
1089
1174
  *
1090
- * A `claimed` or `running` row is a promise that a worker exists in *some*
1091
- * process. This is called from a freshly started daemon, so when no other
1092
- * daemon is alive every such row is a worker that died with the previous
1093
- * process. Left "active", those rows deadlock admission forever: the slot
1094
- * count reads full while nothing runs, and the fleet looks busy doing nothing
1095
- * (found live, after a host restart killed two workers mid-run).
1175
+ * Found live after a host restart killed two workers mid-run, and again on
1176
+ * every package deploy that restarted while workers were live (#35): without
1177
+ * the salvage call the next attempt's `worktree remove --force` destroyed
1178
+ * uncommitted edits that had no other copy. Cap-kills already salvaged (#27);
1179
+ * this is the same call site for the restart path.
1096
1180
  *
1097
1181
  * Only the rows change. The issue keeps its in-progress label — that label is
1098
1182
  * the crash guard against double-dispatch, and deciding what a dead worker's
1099
- * remains are worth (an open PR? unpushed commits? a dirty tree?) is the
1183
+ * remains are worth (an open PR? a salvaged sha? a clean tree?) is the
1100
1184
  * orchestrator's drain-duty judgement, not something to automate here. The
1101
1185
  * rows also keep counting toward `maxAttemptsPerIssue`, so a loop of deaths
1102
1186
  * still escalates instead of retrying forever.
@@ -1107,12 +1191,21 @@ export function armConductor(): void {
1107
1191
  * them is {@link settlePushedGreen}, on the tick, by asking the tracker what
1108
1192
  * became of the PR — the one question a restart cannot answer by inference.
1109
1193
  */
1110
- export function reconcileOrphanedRuns(store: Store, project: string): RunRecord[] {
1194
+ export async function reconcileOrphanedRuns(
1195
+ store: Store,
1196
+ project: string,
1197
+ ): Promise<RunRecord[]> {
1111
1198
  // Live runs only: `pushed-green` holds no process, so it cannot be orphaned by
1112
1199
  // a process dying — it is finished work waiting on a human merge.
1113
1200
  const stale = store.liveRuns(project);
1114
1201
  const endedAt = Date.now();
1115
1202
  for (const r of stale) {
1203
+ // Salvage before the row flips: the worktree path is on the record, and
1204
+ // salvageWip is a no-op for a missing/clean tree. Reason string matches
1205
+ // the cap-kill wording so triage reads the same either way.
1206
+ if (r.worktree !== "") {
1207
+ await salvage(r.issue, r.attempt, "a daemon restart", r.worktree);
1208
+ }
1116
1209
  store.updateRun(r.id, { state: "orphaned", endedAt });
1117
1210
  }
1118
1211
  return stale;
@@ -1141,7 +1234,7 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1141
1234
  // daemon must not orphan that daemon's real, live workers).
1142
1235
  const alive = livingDaemon();
1143
1236
  if (alive === undefined || alive.pid === process.pid) {
1144
- for (const r of reconcileOrphanedRuns(store, project.name)) {
1237
+ for (const r of await reconcileOrphanedRuns(store, project.name)) {
1145
1238
  log(
1146
1239
  `#${r.issue} orphaned by a previous daemon (attempt ${r.attempt}, was ${r.state}, worktree ${r.worktree}) — ` +
1147
1240
  `slot freed; the ${project.stateLabels.inProgress} label stays until the orchestrator triages what the worker left`,
@@ -1283,5 +1376,11 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
1283
1376
  await orchestrator?.dispose();
1284
1377
  store.close();
1285
1378
  log("stopped");
1379
+ // A handled SIGTERM still leaves some runtimes with a non-zero default
1380
+ // (historically 128+signal). Under systemd `Restart=on-failure` that looks
1381
+ // like a crash and the unit comes straight back — the exact failure mode
1382
+ // `omp-conductor stop` hit on the reference fleet. Force success so a
1383
+ // graceful drain is not a restart.
1384
+ process.exitCode = 0;
1286
1385
  }
1287
1386
  }
package/src/escalate.ts CHANGED
@@ -45,11 +45,21 @@ export interface Escalator {
45
45
  * send, turning a cosmetic problem into a lost escalation. Plain text is also
46
46
  * valid Markdown, so the same string renders fine as an issue comment.
47
47
  */
48
+ /**
49
+ * Fleet-scoped pages (integrity tripwire, spend-cap halt, stall) use issue `0`
50
+ * as a sentinel — there is no tracker issue. Rendering that as `#0` made
51
+ * Telegram pages and daemon logs look like a bug (#52). Keep the sentinel in
52
+ * the typed field; only the human-facing label changes here.
53
+ */
54
+ export function escalationIssueRef(issue: number): string {
55
+ return issue === 0 ? "fleet" : `#${issue}`;
56
+ }
57
+
48
58
  export function formatEscalation(e: Escalation, project: string): string {
49
59
  const lines = [
50
60
  `omp-conductor · tier ${e.tier} escalation`,
51
61
  `project: ${project}`,
52
- `issue: #${e.issue}`,
62
+ `issue: ${escalationIssueRef(e.issue)}`,
53
63
  `summary: ${e.summary}`,
54
64
  ];
55
65
  if (e.detail) lines.push(`detail: ${e.detail}`);
@@ -99,18 +109,21 @@ export function createEscalator(
99
109
  const onTurnFailed = async (cause: unknown): Promise<void> => {
100
110
  if (!p.escalation.fallbackToIssueComment) {
101
111
  warn(
102
- `tier 1 escalation on issue #${e.issue} was accepted by the orchestrator but its ` +
112
+ `tier 1 escalation on ${escalationIssueRef(e.issue)} was accepted by the orchestrator but its ` +
103
113
  `turn failed (${errText(cause)}), and no fallback is configured — left unmarked ` +
104
114
  `so the next tick retries`,
105
115
  );
106
116
  return;
107
117
  }
108
118
  try {
119
+ if (e.issue === 0) {
120
+ throw new Error("fleet-scoped escalation has no issue to comment on");
121
+ }
109
122
  await tracker.comment(e.issue, text);
110
123
  store.markNotified(key);
111
124
  } catch (err) {
112
125
  warn(
113
- `tier 1 escalation on issue #${e.issue} failed after acceptance ` +
126
+ `tier 1 escalation on ${escalationIssueRef(e.issue)} failed after acceptance ` +
114
127
  `(${errText(cause)}) and its issue-comment fallback failed too ` +
115
128
  `(${errText(err)}) — left unmarked so the next tick retries`,
116
129
  );
@@ -154,10 +167,19 @@ export function createEscalator(
154
167
  if (!p.escalation.fallbackToIssueComment) {
155
168
  throw new Error(
156
169
  `no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
157
- `escalation on issue #${e.issue} (${e.summary}) could not be delivered — ` +
170
+ `escalation on ${escalationIssueRef(e.issue)} (${e.summary}) could not be delivered — ` +
158
171
  `set escalation.telegramChatId or escalation.fallbackToIssueComment`,
159
172
  );
160
173
  }
174
+ // Fleet pages (issue 0) have nowhere to comment. Falling through here used
175
+ // to call `issues/0/comments` and surface as a confusing "#0" delivery
176
+ // failure after Telegram had already been tried (#52).
177
+ if (e.issue === 0) {
178
+ throw new Error(
179
+ `fleet-scoped tier ${e.tier} escalation (${e.summary}) has no issue to comment on — ` +
180
+ `configure escalation.telegramChatId so integrity/spend/stall pages can reach an operator`,
181
+ );
182
+ }
161
183
  await tracker.comment(e.issue, text);
162
184
  store.markNotified(key);
163
185
  },
package/src/lifecycle.ts CHANGED
@@ -10,16 +10,30 @@
10
10
  * operator real time are `start` refusing against a ghost and `status`
11
11
  * reporting a daemon that died hours ago.
12
12
  *
13
+ * When the live pid is the MainPID of the `omp-conductor.service` unit,
14
+ * `stop`/`restart` go through `systemctl` rather than a raw `SIGTERM`. A
15
+ * raw signal against a unit with `Restart=on-failure` is read as a crash and
16
+ * the unit comes straight back — the bug that made `omp-conductor stop` look
17
+ * like a no-op on a systemd-managed host.
18
+ *
13
19
  * Deliberately free of every other module in this package: nothing here opens
14
20
  * the store, loads the config or talks to `gh`, so `stop` and `status` keep
15
21
  * working when the config is the very thing that is broken.
16
22
  */
17
23
 
18
- import { spawn } from "node:child_process";
24
+ import { spawn, spawnSync } from "node:child_process";
19
25
  import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
20
26
  import { homedir } from "node:os";
21
27
  import { join } from "node:path";
22
28
 
29
+ /**
30
+ * The systemd unit name operators are expected to install for a supervised
31
+ * daemon. Hardcoded rather than configured: one name on the host is the
32
+ * whole point of a unit, and a wrong name would silently fall back to the
33
+ * SIGTERM path and reintroduce the restart loop this module exists to avoid.
34
+ */
35
+ export const SYSTEMD_UNIT = "omp-conductor.service";
36
+
23
37
  /**
24
38
  * Mirrors `DEFAULT_PORT` in ./daemon.ts. Duplicated rather than imported so
25
39
  * this module stays free of the dispatcher's dependency tree; exported so the
@@ -271,23 +285,67 @@ export async function startDaemon(o: { port?: number; project?: string } = {}):
271
285
  }
272
286
 
273
287
  /**
274
- * Stops the daemon: `SIGTERM`, wait, then `SIGKILL`.
288
+ * How the last stop actually landed. Callers print this so an operator can
289
+ * tell a supervised stop from a bare SIGTERM without reading the journal.
290
+ */
291
+ export type StopResult =
292
+ | { kind: "stopped"; pid: number; via: "systemctl" | "signal" }
293
+ | { kind: "not-running" };
294
+
295
+ /**
296
+ * Stops the daemon.
275
297
  *
276
- * `SIGTERM` asks the loop to finish the tick it is on, and a tick that has a
277
- * worker in flight can run for the worker's whole wall clock. The grace period
278
- * is therefore a deadline, not a promise of a clean drain.
279
- * // ponytail: no way to say "stop when the current worker lands". If that
280
- * // matters, take a `--timeout` on the CLI verb, or poll `activeRuns` before
281
- * // escalating.
298
+ * Prefer `systemctl stop omp-conductor.service` when that unit's MainPID is
299
+ * the live daemon: systemd then owns the stop and will not schedule a restart
300
+ * for the exit it just requested. A raw `SIGTERM` against a unit with
301
+ * `Restart=on-failure` is what made `omp-conductor stop` look like a no-op on
302
+ * the reference fleet exit 143 was a "failure", and the unit came back five
303
+ * seconds later.
304
+ *
305
+ * Falls back to `SIGTERM` then `SIGKILL` when there is no unit, the unit is
306
+ * not running, or its MainPID is somebody else (a hand-started daemon next to
307
+ * a stopped unit). The grace period is a deadline, not a clean drain: a tick
308
+ * with a worker in flight can run for that worker's whole wall clock.
282
309
  */
283
- export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<"stopped" | "not-running"> {
310
+ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopResult> {
284
311
  const rec = livingDaemon();
285
312
  if (rec === undefined) {
286
313
  // `livingDaemon` already cleared a stale file; this covers the unparseable
287
- // one it refused to read.
314
+ // one it refused to read. Still ask systemd: a unit can be running with
315
+ // no pidfile (boot race, wiped runtime dir) and stop must still land.
316
+ const unitPid = systemdMainPid();
317
+ if (unitPid !== undefined && (await stopViaSystemd(undefined))) {
318
+ const deadline = Date.now() + (o.timeoutMs ?? STOP_TIMEOUT_MS);
319
+ while (isAlive(unitPid) && Date.now() < deadline) await sleep(100);
320
+ if (isAlive(unitPid)) {
321
+ throw new Error(
322
+ `daemon pid ${unitPid} still alive after systemctl stop ${SYSTEMD_UNIT} — ` +
323
+ `check \`systemctl status ${SYSTEMD_UNIT}\``,
324
+ );
325
+ }
326
+ clearRecord();
327
+ return { kind: "stopped", pid: unitPid, via: "systemctl" };
328
+ }
288
329
  clearRecord();
289
- return "not-running";
330
+ return { kind: "not-running" };
290
331
  }
332
+
333
+ if (await stopViaSystemd(rec.pid)) {
334
+ // Wait out the unit: systemctl stop is synchronous for Type=simple, but
335
+ // a slow drain still holds the old pid briefly and the next start would
336
+ // refuse against it.
337
+ const deadline = Date.now() + (o.timeoutMs ?? STOP_TIMEOUT_MS);
338
+ while (isAlive(rec.pid) && Date.now() < deadline) await sleep(100);
339
+ if (isAlive(rec.pid)) {
340
+ throw new Error(
341
+ `daemon pid ${rec.pid} still alive after systemctl stop ${SYSTEMD_UNIT} — ` +
342
+ `check \`systemctl status ${SYSTEMD_UNIT}\``,
343
+ );
344
+ }
345
+ clearRecord();
346
+ return { kind: "stopped", pid: rec.pid, via: "systemctl" };
347
+ }
348
+
291
349
  const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
292
350
  if (!gone) {
293
351
  // The record stays: something is still holding that pid, and forgetting
@@ -295,13 +353,146 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<"stopp
295
353
  throw new Error(`daemon pid ${rec.pid} survived SIGTERM and SIGKILL — it may belong to another user`);
296
354
  }
297
355
  clearRecord();
298
- return "stopped";
356
+ return { kind: "stopped", pid: rec.pid, via: "signal" };
357
+ }
358
+
359
+ /**
360
+ * Restarts the daemon.
361
+ *
362
+ * Same ownership rule as {@link stopDaemon}: when the unit owns the live pid,
363
+ * `systemctl restart` keeps systemd in charge of the replacement process so
364
+ * the new MainPID is still the unit's. Falling back to stop+start for a
365
+ * hand-started daemon would leave the unit dead and the new process
366
+ * unsupervised — fine for a laptop, wrong for the host that installed a unit
367
+ * specifically so a crash comes back.
368
+ *
369
+ * Returns the record of the process that is now answering `/healthz`.
370
+ */
371
+ export async function restartDaemon(
372
+ o: { port?: number; project?: string; timeoutMs?: number } = {},
373
+ ): Promise<{ previous: DaemonRecord | undefined; record: DaemonRecord; via: "systemctl" | "cli" }> {
374
+ const previous = livingDaemon();
375
+ const unitPid = systemdMainPid();
376
+ const unitOwns =
377
+ unitPid !== undefined && (previous === undefined || previous.pid === unitPid);
378
+
379
+ if (unitOwns) {
380
+ const ran = systemctl(["restart", SYSTEMD_UNIT]);
381
+ if (ran.ok) {
382
+ // systemctl restart returns once the new MainPID is up; the pidfile is
383
+ // written by the daemon itself on boot, so wait for that rather than
384
+ // inventing a record from the unit alone.
385
+ const deadline = Date.now() + READY_TIMEOUT_MS;
386
+ for (;;) {
387
+ const rec = livingDaemon();
388
+ if (rec !== undefined) {
389
+ const health = await healthCheck(rec.port);
390
+ if (health.ok) return { previous, record: rec, via: "systemctl" };
391
+ }
392
+ if (Date.now() >= deadline) break;
393
+ await sleep(READY_POLL_MS);
394
+ }
395
+ throw new Error(
396
+ `systemctl restart ${SYSTEMD_UNIT} returned, but the daemon never answered /healthz`,
397
+ );
398
+ }
399
+ // Unit exists and owns the pid but systemctl refused (permissions, dbus
400
+ // down). Fall through to the signal path rather than stranding the
401
+ // operator with "restart failed" and a still-running daemon they cannot
402
+ // reach through the unit.
403
+ }
404
+
405
+ await stopDaemon({ timeoutMs: o.timeoutMs });
406
+ const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
407
+ return { previous, record, via: "cli" };
408
+ }
409
+
410
+ /**
411
+ * The MainPID of {@link SYSTEMD_UNIT}, or `undefined` when systemd is absent,
412
+ * the unit is unknown, or it is not running. Never throws: a missing binary
413
+ * or a dbus blip is "no unit", and stop falls back to SIGTERM.
414
+ */
415
+ export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
416
+ const ran = systemctl(["show", unit, "--property=MainPID", "--property=ActiveState", "--value"]);
417
+ if (!ran.ok) return undefined;
418
+ // `systemctl show --value` prints one property per line, MainPID then
419
+ // ActiveState, in the order requested. Tolerate either order and blank
420
+ // lines so a future systemctl rearrange does not silently disable the path.
421
+ const lines = ran.stdout
422
+ .split("\n")
423
+ .map((l) => l.trim())
424
+ .filter((l) => l.length > 0);
425
+ let pid: number | undefined;
426
+ let active: string | undefined;
427
+ for (const line of lines) {
428
+ if (/^\d+$/.test(line)) {
429
+ const n = Number(line);
430
+ if (Number.isInteger(n) && n > 1) pid = n;
431
+ } else {
432
+ active = line;
433
+ }
434
+ }
435
+ if (active !== undefined && active !== "active" && active !== "reactivating") return undefined;
436
+ return pid;
299
437
  }
300
438
 
301
439
  // ---------------------------------------------------------------------------
302
440
  // internals
303
441
  // ---------------------------------------------------------------------------
304
442
 
443
+ /**
444
+ * Ask systemd to stop the unit, but only when it actually owns `pid`.
445
+ *
446
+ * `pid === undefined` means "no pidfile" — still stop the unit if it is
447
+ * active, because that is the only process that could be the daemon. A unit
448
+ * whose MainPID is some other process is left alone: stopping it would take
449
+ * down a neighbour, and the signal path handles *our* pid.
450
+ */
451
+ async function stopViaSystemd(pid: number | undefined): Promise<boolean> {
452
+ const main = systemdMainPid();
453
+ if (main === undefined) return false;
454
+ if (pid !== undefined && main !== pid) return false;
455
+ const ran = systemctl(["stop", SYSTEMD_UNIT]);
456
+ return ran.ok;
457
+ }
458
+
459
+ /**
460
+ * Run one `systemctl` invocation. Captures output and never throws: absence
461
+ * of the binary, a missing unit, or a permission error are all "not ok", and
462
+ * the caller decides whether to fall back.
463
+ *
464
+ * The default shells out. Tests replace it with {@link setSystemctlForTest}
465
+ * so the ownership decision is exercised without a real systemd.
466
+ */
467
+ export type SystemctlFn = (args: string[]) => { ok: boolean; stdout: string; stderr: string };
468
+
469
+ function defaultSystemctl(args: string[]): { ok: boolean; stdout: string; stderr: string } {
470
+ try {
471
+ const res = spawnSync("systemctl", args, {
472
+ encoding: "utf8",
473
+ // A hung dbus is not worth blocking stop on; the signal path is right there.
474
+ timeout: 15_000,
475
+ env: process.env,
476
+ });
477
+ if (res.error) return { ok: false, stdout: "", stderr: res.error.message };
478
+ return {
479
+ ok: res.status === 0,
480
+ stdout: res.stdout ?? "",
481
+ stderr: res.stderr ?? "",
482
+ };
483
+ } catch (err) {
484
+ return { ok: false, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
485
+ }
486
+ }
487
+
488
+ let systemctl: SystemctlFn = defaultSystemctl;
489
+
490
+ /** Test-only: replace the `systemctl` runner. Pass `undefined` to restore. */
491
+ export function setSystemctlForTest(fn: SystemctlFn | undefined): void {
492
+ systemctl = fn ?? defaultSystemctl;
493
+ }
494
+
495
+
305
496
  function sleep(ms: number): Promise<void> {
306
497
  return new Promise<void>((resolve) => {
307
498
  const t = setTimeout(resolve, ms);
package/src/omp.ts CHANGED
@@ -162,6 +162,11 @@ export async function createSession(opts: {
162
162
  // sharing it fails to start. The daemon runs `maxConcurrentWorkers`
163
163
  // (2 by default) workers at once, which makes this the normal path.
164
164
  agentRegistry: new mod.AgentRegistry(),
165
+ // Default true in the SDK, but pass it explicitly so a harness change
166
+ // cannot silently strip MCP from workers. Discovery walks cwd + user
167
+ // agentDir (~/.omp/agent/mcp.json) — without this, workers grep-only and
168
+ // burn the turns cap on discovery (#29).
169
+ enableMCP: true,
165
170
  });
166
171
  const raw = asRawSession(created);
167
172
  // Surfaced rather than swallowed: this is how a quiet downgrade to a weaker