omp-conductor 0.15.12 → 0.15.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/doctor.ts CHANGED
@@ -27,18 +27,38 @@
27
27
 
28
28
  import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
29
29
  import { spawnSync } from "node:child_process";
30
+ import { homedir } from "node:os";
30
31
  import { join } from "node:path";
31
32
  import type { Stats } from "node:fs";
32
33
  import { Database } from "bun:sqlite";
33
34
 
34
35
  import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
35
- import { DEFAULT_HERDR_UNIT, probeTelegramHealth, telegramStateDir, type TelegramHealth } from "./fleet.ts";
36
- import { planHostRuntime, STAGED_SERVICE_NAME, SYSTEMD_UNIT_DIR, totalConfiguredWorkers } from "./setup-host.ts";
36
+ import {
37
+ DEFAULT_HERDR_UNIT,
38
+ probeTelegramHealth,
39
+ resolveHerdrSessionWithBridge,
40
+ telegramStateDir,
41
+ type TelegramHealth,
42
+ } from "./fleet.ts";
43
+ import {
44
+ herdrConductorPluginConfigDir,
45
+ planHostRuntime,
46
+ STAGED_SERVICE_NAME,
47
+ SYSTEMD_UNIT_DIR,
48
+ tickCwdForProject,
49
+ totalConfiguredWorkers,
50
+ } from "./setup-host.ts";
37
51
  import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
38
52
  import { dbPath, LIVE_STATES } from "./store.ts";
39
53
  import { telegramReportSend, type ReportSend } from "./reports.ts";
40
54
  import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
41
55
  import { repoSlugFor } from "./gitops.ts";
56
+ import {
57
+ DEFAULT_FLEET_AGENT_NAME,
58
+ parseHerdrAgents,
59
+ readTickConfig,
60
+ type HerdrAgentList,
61
+ } from "./orchestrator-tick.ts";
42
62
  import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
43
63
 
44
64
  /**
@@ -148,6 +168,20 @@ export interface DoctorDeps {
148
168
  telegramSend?: (project: ProjectConfig) => ReportSend;
149
169
  /** The canonical units to compare the installed units against. */
150
170
  canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
171
+ /** Whether herdr is installed on this host (a `herdr` on PATH). */
172
+ herdrInstalled?: () => boolean;
173
+ /** Live `herdr --session <s> agent list`, parsed through the tick's own
174
+ * parser (the same "one JSON line on stdout" contract recover.sh reads). */
175
+ herdrAgents?: (session: string) => HerdrAgentList;
176
+ /** The fleet session name, for the rename fix line. */
177
+ herdrSession?: () => string;
178
+ /** The live herdr config at `~/.config/herdr/config.toml`, or undefined. */
179
+ herdrConfig?: () => string | undefined;
180
+ /** The live herdr-conductor plugin `config.env`, or undefined. */
181
+ herdrEnv?: () => string | undefined;
182
+ /** The fleet agent name the tick config of one project names, or undefined
183
+ * when there is no (readable) tick — the expected live herdr pane identity. */
184
+ tickAgentName?: (project: ProjectConfig) => string | undefined;
151
185
  /** Clock, so a run is deterministic in tests. */
152
186
  now?: () => number;
153
187
  /** The one opt-in side effect: send one self-identified Telegram probe. */
@@ -790,7 +824,190 @@ function spendProbe(rows: RunSpendRow[], limit: number): Finding {
790
824
  return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
791
825
  }
792
826
 
793
- // ------------------------------------------------------------- composition
827
+ /** Live `herdr --session <session> agent list` through the project's own
828
+ * parser — the published "one JSON line on stdout" interface, never the
829
+ * socket. Same bound as the tick's own query so a hung herdr cannot hang
830
+ * doctor. */
831
+ function defaultHerdrAgents(session: string): HerdrAgentList {
832
+ try {
833
+ const run = spawnSync("herdr", ["--session", session, "agent", "list"], {
834
+ encoding: "utf8",
835
+ timeout: 3000,
836
+ stdio: ["ignore", "pipe", "pipe"],
837
+ });
838
+ if (run.error !== undefined) return { kind: "unavailable", problem: run.error.message };
839
+ if (run.status !== 0) {
840
+ return {
841
+ kind: "unavailable",
842
+ problem: `herdr agent list exited ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}`,
843
+ };
844
+ }
845
+ return parseHerdrAgents(run.stdout ?? "");
846
+ } catch (err) {
847
+ return { kind: "unavailable", problem: err instanceof Error ? err.message : String(err) };
848
+ }
849
+ }
850
+
851
+ /** Live herdr config, defaulting to the fleet account's own path. */
852
+ function defaultHerdrConfig(): string | undefined {
853
+ try {
854
+ return readFileSync(join(homedir(), ".config", "herdr", "config.toml"), "utf8");
855
+ } catch {
856
+ return undefined;
857
+ }
858
+ }
859
+
860
+ /** Live herdr-conductor plugin `config.env`, same path setup provisions. */
861
+ function defaultHerdrEnv(): string | undefined {
862
+ try {
863
+ return readFileSync(join(herdrConductorPluginConfigDir(homedir()), "config.env"), "utf8");
864
+ } catch {
865
+ return undefined;
866
+ }
867
+ }
868
+
869
+ /**
870
+ * #541 check 1 — live herdr agent name equals the tick config's agentName.
871
+ *
872
+ * The paneOwnership decline the fleet actually hit names the exact fix (`herdr
873
+ * agent rename <pane> <name>`); doctor checks the same premise the tick does
874
+ * (is a pane registered under the configured agent name?) and reports drift
875
+ * with the one-line remedy.
876
+ */
877
+ function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding {
878
+ if (!probes.herdrInstalled()) {
879
+ return passFinding("herdr-agent-name", `[${p.name}] herdr not installed — nothing to check`);
880
+ }
881
+ const want = probes.tickAgentName(p);
882
+ if (want === undefined) {
883
+ return passFinding("herdr-agent-name", `[${p.name}] no readable tick config — nothing to check`);
884
+ }
885
+ const session = probes.herdrSession();
886
+ const agents = probes.herdrAgents(session);
887
+ if (agents.kind !== "ok") {
888
+ return failFinding(
889
+ "herdr-agent-name",
890
+ `[${p.name}] could not read the live herdr agent list (${agents.problem}) — cannot verify the fleet agent name`,
891
+ `restore the herdr session (check \`herdr status\`), then re-run doctor`,
892
+ );
893
+ }
894
+ const holders = agents.agents.filter((a) => a.name === want);
895
+ if (holders.length > 0) {
896
+ return passFinding("herdr-agent-name", `[${p.name}] live pane(s) ${holders.map((a) => a.paneId).join(", ")} are agent "${want}"`);
897
+ }
898
+ // Drift: no pane carries the configured identity. Exactly one pane pinned to
899
+ // the shared default is the safe candidate from the decline turnover — name
900
+ // it when unambiguous, else give the generic command.
901
+ const stale = agents.agents.filter((a) => a.name === DEFAULT_FLEET_AGENT_NAME);
902
+ const pane = stale.length === 1 ? stale[0]!.paneId : "<pane>";
903
+ return failFinding(
904
+ "herdr-agent-name",
905
+ `[${p.name}] no live pane is agent "${want}" — every tick will be declined until the fleet pane carries this name (the 2026-08-15 restamp class)`,
906
+ `herdr --session ${session} agent rename ${pane} ${want}`,
907
+ );
908
+ }
909
+
910
+ /**
911
+ * #541 check 2 — `[session] resume_agents_on_restore` on the live herdr config.
912
+ *
913
+ * herdr's own README: the default `true` leaves a restored omp pane with a
914
+ * deferred resume plan and no live terminal, which is the configuration that
915
+ * pages instead of recovering. Absent ~ the unrecoverable default; an explicit
916
+ * `true` is a deliberate (desktop) choice and only warned.
917
+ */
918
+ function herdrResumeProbe(probes: Probes): Finding {
919
+ if (!probes.herdrInstalled()) {
920
+ return passFinding("herdr-resume", "herdr not installed — nothing to check");
921
+ }
922
+ const live = probes.herdrConfig();
923
+ if (live === undefined) {
924
+ return failFinding(
925
+ "herdr-resume",
926
+ "no herdr config at " + join(homedir(), ".config", "herdr", "config.toml") + " — a restored omp pane gets no live terminal (herdr's own README)",
927
+ "run \`omp-conductor setup host\` from the fleet account (writes [session] resume_agents_on_restore = false), or add it by hand",
928
+ );
929
+ }
930
+ let parsed: { session?: { resume_agents_on_restore?: unknown } };
931
+ try {
932
+ parsed = Bun.TOML.parse(live) as { session?: { resume_agents_on_restore?: unknown } };
933
+ } catch (err) {
934
+ return failFinding(
935
+ "herdr-resume",
936
+ `the live herdr config does not parse (${err instanceof Error ? err.message : String(err)}) — herdr itself refuses it`,
937
+ "run `herdr config check` to name the line, fix, then re-run doctor",
938
+ );
939
+ }
940
+ const value = parsed.session?.resume_agents_on_restore;
941
+ if (value === false) {
942
+ return passFinding("herdr-resume", "resume_agents_on_restore is false — restored panes come up as shells, not deferred plans");
943
+ }
944
+ if (value === true) {
945
+ return warnFinding(
946
+ "herdr-resume",
947
+ "resume_agents_on_restore is explicitly true — possibly a deliberate desktop config, but a headless fleet restores panes with no live terminal",
948
+ "set it to false for a headless host ([session] resume_agents_on_restore = false), or run `omp-conductor setup host`",
949
+ );
950
+ }
951
+ return failFinding(
952
+ "herdr-resume",
953
+ "resume_agents_on_restore is absent — herdr defaults it to true, the configuration its own README calls unrecoverable",
954
+ "run `omp-conductor setup host` from the fleet account (writes the key), or add [session] resume_agents_on_restore = false by hand",
955
+ );
956
+ }
957
+
958
+ /**
959
+ * #541 check 3 — herdr-conductor `config.env` FLEET_CWDS vs the configured
960
+ * projects. Absent, a virgin host recovers against legacy `/root/fleet` paths
961
+ * that do not exist: recovery still runs, only the page silently goes nowhere.
962
+ */
963
+ function herdrEnvProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding {
964
+ if (!probes.herdrInstalled()) {
965
+ return passFinding("herdr-config-env", "herdr not installed — nothing to check");
966
+ }
967
+ if (cfg === undefined) {
968
+ return passFinding("herdr-config-env", "config unreadable — nothing to compare");
969
+ }
970
+ const live = probes.herdrEnv();
971
+ if (live === undefined) {
972
+ return failFinding(
973
+ "herdr-config-env",
974
+ "no herdr-conductor config.env — recovery falls back to legacy /root-based single-tenant paths this host does not use, so pages silently skip",
975
+ "run `omp-conductor setup host` from the fleet account (writes config.env)",
976
+ );
977
+ }
978
+ const expected = new Set(cfg.projects.map((p) => tickCwdForProject(p)));
979
+ const actual = parseFleetCwds(live);
980
+ const coherent =
981
+ actual !== undefined && actual.size === expected.size && [...actual].every((cwd) => expected.has(cwd));
982
+ if (coherent) {
983
+ return passFinding("herdr-config-env", `config.env FLEET_CWDS matches the configured fleets (${[...expected].join(":")})`);
984
+ }
985
+ return failFinding(
986
+ "herdr-config-env",
987
+ `config.env FLEET_CWDS (${actual === undefined ? "unset" : [...actual].join(":")}) does not match the configured fleets (${[...expected].join(":")})`,
988
+ "re-run `omp-conductor setup host`, which derives FLEET_CWDS from the configured projects",
989
+ );
990
+ }
991
+
992
+ /** FLEET_CWDS (colon- or space-separated, per recover.sh) or undefined. */
993
+ function parseFleetCwds(text: string): Set<string> | undefined {
994
+ const value = parseEnvKey(text, "FLEET_CWDS");
995
+ if (value !== undefined) {
996
+ return new Set(value.split(/[\s:]+/).filter((w) => w.length > 0));
997
+ }
998
+ // Legacy single-fleet form, the same fallback recover.sh applies.
999
+ const legacy = parseEnvKey(text, "FLEET_CWD");
1000
+ return legacy === undefined ? undefined : new Set([legacy]);
1001
+ }
1002
+
1003
+ /** The first unquoted-or-quoted `KEY=value` on its own line, comments ignored. */
1004
+ function parseEnvKey(text: string, key: string): string | undefined {
1005
+ for (const line of text.split("\n")) {
1006
+ const match = new RegExp(`^[ \\t]*${key}[ \\t]*=[ \\t]*"?([^"\\n]*)"?[ \\t]*$`).exec(line);
1007
+ if (match !== null) return match[1];
1008
+ }
1009
+ return undefined;
1010
+ }
794
1011
 
795
1012
  /**
796
1013
  * Run every probe and assemble the stable report.
@@ -873,8 +1090,14 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
873
1090
  findings.push(
874
1091
  passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`),
875
1092
  );
1093
+ findings.push(
1094
+ passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`),
1095
+ );
876
1096
  } else {
877
- for (const p of projects) findings.push(await labelProbe(probes, p));
1097
+ for (const p of projects) {
1098
+ findings.push(await labelProbe(probes, p));
1099
+ findings.push(herdrAgentNameProbe(probes, p));
1100
+ }
878
1101
  }
879
1102
  // The installed-unit check is host-global: one shared daemon. Any project
880
1103
  // renders the same canonical units (the daemon/units carry no project), so
@@ -882,6 +1105,10 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
882
1105
  findings.push(unitProbe(probes, projects[0], cfg));
883
1106
  findings.push(recoveryProbe(probes));
884
1107
  findings.push(ownershipProbe(probes));
1108
+ // #541 seam checks, host-global: the live herdr config and the plugin's
1109
+ // config.env are single files on the host, not per-project facts.
1110
+ findings.push(herdrResumeProbe(probes));
1111
+ findings.push(herdrEnvProbe(probes, cfg));
885
1112
  if (projects.length === 0) {
886
1113
  findings.push(timezoneProbe(undefined));
887
1114
  findings.push(await telegramProbe(probes, undefined, checkedAt));
@@ -926,6 +1153,16 @@ export function defaultProbes(): Probes {
926
1153
  telegramHealth: (projectName) => probeTelegramHealth(projectName),
927
1154
  telegramSend: telegramReportSend,
928
1155
  canonicalUnits: defaultCanonicalUnits,
1156
+ herdrInstalled: () => Bun.which("herdr") !== null,
1157
+ herdrAgents: defaultHerdrAgents,
1158
+ herdrSession: () => resolveHerdrSessionWithBridge(),
1159
+ herdrConfig: defaultHerdrConfig,
1160
+ herdrEnv: defaultHerdrEnv,
1161
+ tickAgentName: (p) => {
1162
+ const tick = readTickConfig(tickCwdForProject(p));
1163
+ if (tick.kind !== "ok") return undefined;
1164
+ return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME;
1165
+ },
929
1166
  now: Date.now,
930
1167
  probeTelegram: false,
931
1168
  };
package/src/escalate.ts CHANGED
@@ -144,6 +144,7 @@ export function createEscalator(
144
144
  orchestrator?: OrchestratorHandle,
145
145
  now: () => number = Date.now,
146
146
  deliveryAllowed: () => boolean = () => true,
147
+ onTier1Diverted?: (e: Escalation) => void,
147
148
  ): Escalator {
148
149
  const currentProject = typeof source === "function" ? source : (): ProjectConfig => source;
149
150
  return {
@@ -278,6 +279,13 @@ export function createEscalator(
278
279
  // Tier 1 lands here with no orchestrator, or with one that would not take
279
280
  // the injection; tier 2 lands here when omp-telegram is not installed or
280
281
  // the project never configured a chat id.
282
+ //
283
+ // A tier-1 event reaching this point was diverted from the orchestrator to
284
+ // an issue comment. That is the exact condition the down incident counts:
285
+ // the daemon hooks this to accumulate the orchestrator-down incident's
286
+ // diverted tally (a reader that owns its own dedupe and no-ops for a
287
+ // healthy or external orchestrator).
288
+ if (e.tier === 1) onTier1Diverted?.(e);
281
289
  if (!p.escalation.fallbackToIssueComment) {
282
290
  throw new Error(
283
291
  `no escalation transport configured for project "${p.name}": tier ${e.tier} ` +
package/src/fleet.ts CHANGED
@@ -36,7 +36,7 @@ import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
36
36
  import { inspectBriefLayout } from "./brief-upgrade.ts";
37
37
  import { dbPath, openStore } from "./store.ts";
38
38
  import { renderBriefForProject } from "./setup.ts";
39
- import type { ProjectConfig, Store } from "./types.ts";
39
+ import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
40
40
  import { settlementFlagSummary } from "./diff-flags.ts";
41
41
  import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
42
42
  import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
@@ -52,6 +52,7 @@ import {
52
52
  statusSnapshot,
53
53
  type StatusSnapshot,
54
54
  } from "./daemon.ts";
55
+ import { formatOrchestratorDown } from "./orchestrator-down.ts";
55
56
  import {
56
57
  healthCheck,
57
58
  isAlive,
@@ -1410,6 +1411,31 @@ function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): s
1410
1411
  }
1411
1412
  }
1412
1413
 
1414
+ /**
1415
+ * The newest daemon stop/restart provenance as status lines (#378).
1416
+ *
1417
+ * Rendered under the daemon block whether the daemon is down or has since
1418
+ * been restarted — the whole point is that the debrief survives the restart.
1419
+ * An unattributed record says so explicitly: the external-signal fallback
1420
+ * records what the receiving daemon knew (that it was unattributed, when,
1421
+ * which projects it served with live counts) rather than guessing a caller.
1422
+ */
1423
+ function formatLastStop(lastStop: DaemonStop | undefined): string[] {
1424
+ if (lastStop === undefined) return [];
1425
+ const caller = lastStop.unattributed
1426
+ ? "unattributed — no mediated request (external signal)"
1427
+ : `pid ${lastStop.callerPid ?? "?"}` +
1428
+ (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
1429
+ (lastStop.role === undefined ? "" : ` (${lastStop.role})`);
1430
+ return [
1431
+ ` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`,
1432
+ ` caller ${caller}`,
1433
+ ` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
1434
+ ` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
1435
+ ` reason ${lastStop.reason}`,
1436
+ ];
1437
+ }
1438
+
1413
1439
  export function formatFleetStatus(
1414
1440
  s: StatusSnapshot,
1415
1441
  layers: FleetLayers,
@@ -1422,6 +1448,7 @@ export function formatFleetStatus(
1422
1448
  failureClasses: string | undefined = undefined,
1423
1449
  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1424
1450
  intake: string | undefined = undefined,
1451
+ lastStop: DaemonStop | undefined = undefined,
1425
1452
  ): string {
1426
1453
  const tickLine =
1427
1454
  layers.ticksDetail === undefined
@@ -1505,8 +1532,9 @@ export function formatFleetStatus(
1505
1532
  ...(intake === undefined ? [] : [intake]),
1506
1533
  ...(graphBlock === undefined ? [] : [graphBlock]),
1507
1534
  daemonBlock,
1535
+ ...formatLastStop(lastStop),
1508
1536
  "",
1509
- formatProjectBody(s, workerPhases),
1537
+ formatProjectBody(s, workerPhases, now),
1510
1538
  ].join("\n");
1511
1539
  }
1512
1540
 
@@ -1540,12 +1568,18 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
1540
1568
  function formatProjectBody(
1541
1569
  s: StatusSnapshot,
1542
1570
  workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
1571
+ now = Date.now(),
1543
1572
  ): string {
1544
1573
  const lines = [
1545
1574
  `project ${s.project}${s.paused ? " (PAUSED)" : ""}`,
1546
1575
  ...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]),
1547
1576
  `config ${s.configPath}`,
1548
1577
  `state ${s.stateDir}`,
1578
+ // The orchestrator-down degrade row: first-class in the body, present only
1579
+ // while the incident is open, so recovery drops it (#288).
1580
+ ...(s.orchestratorDown === undefined
1581
+ ? []
1582
+ : formatOrchestratorDown(s.orchestratorDown, now)),
1549
1583
  ...formatAvailabilityStatus(s),
1550
1584
  ...formatDigestScheduleStatus(s),
1551
1585
  "",
@@ -1675,6 +1709,18 @@ export async function renderStatus(projectName?: string): Promise<string> {
1675
1709
  const cached = codeGraphFromHealthz(healthBody, project.name);
1676
1710
  const codeGraph = cached ?? (await probeCodeGraph(project));
1677
1711
  const workerPhases = workerPhasesFromHealthz(healthBody, project.name);
1712
+ // The newest host-wide stop/restart provenance (#378). Read here — not in
1713
+ // `statusSnapshot`, which is synchronous and belongs to the daemon module —
1714
+ // and rendered identically from either project: the daemon_stops table is
1715
+ // deliberately not partitioned by project, because the daemon serves every
1716
+ // project and the uninvolved one must see who stopped it too.
1717
+ const store = openStore(dbPath());
1718
+ let lastStop: DaemonStop | undefined;
1719
+ try {
1720
+ lastStop = store.latestDaemonStop();
1721
+ } finally {
1722
+ store.close();
1723
+ }
1678
1724
  return formatFleetStatus(
1679
1725
  { ...s, planUsage, github },
1680
1726
  layers,
@@ -1687,6 +1733,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
1687
1733
  failureClassBlock(project.name),
1688
1734
  workerPhases,
1689
1735
  intakeStatusLine(project.name),
1736
+ lastStop,
1690
1737
  );
1691
1738
  }
1692
1739
 
package/src/lifecycle.ts CHANGED
@@ -25,6 +25,10 @@ import { spawn, spawnSync } from "node:child_process";
25
25
  import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
26
26
  import { homedir } from "node:os";
27
27
  import { join } from "node:path";
28
+ // Type-only: erased at runtime, so the "free of every other module" property
29
+ // below survives — this module still opens no store, loads no config and runs
30
+ // no `gh`, and `stop`/`status` keep working when the config is broken.
31
+ import type { DaemonStopDraft } from "./types.ts";
28
32
 
29
33
  /**
30
34
  * The systemd unit name operators are expected to install for a supervised
@@ -480,6 +484,58 @@ export async function startDaemon(
480
484
  );
481
485
  }
482
486
 
487
+ /**
488
+ * The delivery facts a mediated stop/restart records at the exact chokepoint:
489
+ * the request's own provenance, how the stop is about to be delivered, and the
490
+ * daemon pid it targets (#378). The recorder is injected by the CLI callers
491
+ * (which own the store); this module stays free of it.
492
+ */
493
+ export interface StopDelivery {
494
+ /** The request facts built by the CLI (commands/stop.ts, commands/restart.ts). */
495
+ provenance: DaemonStopDraft;
496
+ /** Whether the stop is about to go through `systemctl` or a raw signal. */
497
+ via: "systemctl" | "signal";
498
+ /** The daemon pid being stopped, when one was known. */
499
+ pid?: number;
500
+ }
501
+
502
+ /** The injected recorder invoked immediately before a stop/restart is signalled. */
503
+ export type StopDeliveryFn = (stop: StopDelivery) => void;
504
+
505
+ /**
506
+ * The honest provenance for a daemon that received a stop with no mediated
507
+ * request anywhere: unattributed by construction — Linux exposes no sender
508
+ * identity for a signal, so the record must say "unattributed" rather than
509
+ * guess one — plus what the receiving process actually knows: its own pid,
510
+ * the moment (stamped by the store), the runtime directory, and the projects
511
+ * it serves with their live-run counts.
512
+ *
513
+ * Storage belongs at the caller: the daemon's SIGINT/SIGTERM path writes the
514
+ * returned draft through the store before/while draining. This builds the
515
+ * facts, store-free, so a raw `kill`/out-of-band `systemctl stop` — no
516
+ * conductor CLI in the delivery path at all — still leaves a durable row the
517
+ * next `omp-conductor status` can show.
518
+ */
519
+ export function externalStopProvenance(o: {
520
+ /** The daemon pid that received the signal, when known. */
521
+ daemonPid?: number;
522
+ /** The daemon's runtime directory (host paths only — never secrets). */
523
+ runtimeDir: string;
524
+ /** Every served project with its live-run count at signal time. */
525
+ affected: { project: string; live: number }[];
526
+ reason?: string;
527
+ }): DaemonStopDraft {
528
+ return {
529
+ controlPath: "external signal",
530
+ scope: "global",
531
+ ...(o.daemonPid === undefined ? {} : { daemonPid: o.daemonPid }),
532
+ runtimeDir: o.runtimeDir,
533
+ affected: o.affected,
534
+ reason: o.reason ?? "external signal — no mediated stop request was recorded",
535
+ unattributed: true,
536
+ };
537
+ }
538
+
483
539
  /**
484
540
  * How the last stop actually landed. Callers print this so an operator can
485
541
  * tell a supervised stop from a bare SIGTERM without reading the journal.
@@ -510,7 +566,35 @@ export interface RestartResult {
510
566
  * a stopped unit). The grace period is a deadline, not a clean drain: a tick
511
567
  * with a worker in flight can run for that worker's whole wall clock.
512
568
  */
513
- export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopResult> {
569
+ export async function stopDaemon(
570
+ o: {
571
+ timeoutMs?: number;
572
+ /**
573
+ * The request facts for this stop, when a mediated caller carries them.
574
+ * With `record`, the durable row is written immediately before the signal
575
+ * is sent — the exact ordering "provenance precedes signalling" refers to
576
+ * — and only when a stop actually lands: a request that finds no daemon
577
+ * stops nothing and records nothing. Drain-style callers whose restart may
578
+ * never execute additionally record the request at entry themselves.
579
+ */
580
+ provenance?: DaemonStopDraft;
581
+ /** Invoked just before the daemon is signalled, with the delivery method. */
582
+ record?: StopDeliveryFn;
583
+ } = {},
584
+ ): Promise<StopResult> {
585
+ const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
586
+ if (o.provenance === undefined || o.record === undefined) return;
587
+ o.record({
588
+ provenance: {
589
+ ...o.provenance,
590
+ controlPath: `${o.provenance.controlPath} via ${via}`,
591
+ ...(pid === undefined ? {} : { daemonPid: pid }),
592
+ },
593
+ via,
594
+ pid,
595
+ });
596
+ };
597
+
514
598
  const rec = livingDaemon();
515
599
  if (rec === undefined) {
516
600
  // `livingDaemon` already cleared a stale file; this covers the unparseable
@@ -521,6 +605,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
521
605
  throw new Error(ownershipUnknown("stop", decision.reason));
522
606
  }
523
607
  if (decision.kind === "stop") {
608
+ recordStop("systemctl", decision.mainPid);
524
609
  await runSystemdStop(decision.mainPid, o.timeoutMs);
525
610
  clearRecord();
526
611
  return { kind: "stopped", pid: decision.mainPid, via: "systemctl" };
@@ -536,6 +621,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
536
621
  throw new Error(ownershipUnknown("stop", decision.reason));
537
622
  }
538
623
  if (decision.kind === "stop") {
624
+ recordStop("systemctl", rec.pid);
539
625
  await runSystemdStop(rec.pid, o.timeoutMs);
540
626
  clearRecord();
541
627
  return { kind: "stopped", pid: rec.pid, via: "systemctl" };
@@ -544,6 +630,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
544
630
  // decision.kind === "not-ours": confirmed no unit, inactive unit, no systemd
545
631
  // binary, or a unit whose MainPID is somebody else. Only a *confirmed*
546
632
  // negative is safe to signal.
633
+ recordStop("signal", rec.pid);
547
634
  const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
548
635
  if (!gone) {
549
636
  // The record stays: something is still holding that pid, and forgetting
@@ -577,8 +664,29 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
577
664
  * Returns the record of the process that is now answering `/healthz`.
578
665
  */
579
666
  export async function restartDaemon(
580
- o: { port?: number; project?: string; timeoutMs?: number } = {},
667
+ o: {
668
+ port?: number;
669
+ project?: string;
670
+ timeoutMs?: number;
671
+ /** Request facts and recorder, same semantics as {@link stopDaemon}: the
672
+ * durable row is written immediately before the restarting signal. */
673
+ provenance?: DaemonStopDraft;
674
+ record?: StopDeliveryFn;
675
+ } = {},
581
676
  ): Promise<RestartResult> {
677
+ const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
678
+ if (o.provenance === undefined || o.record === undefined) return;
679
+ o.record({
680
+ provenance: {
681
+ ...o.provenance,
682
+ controlPath: `${o.provenance.controlPath} via ${via}`,
683
+ ...(pid === undefined ? {} : { daemonPid: pid }),
684
+ },
685
+ via,
686
+ pid,
687
+ });
688
+ };
689
+
582
690
  const previous = livingDaemon();
583
691
  const ownership = probeUnit();
584
692
  if (ownership.kind === "unknown") {
@@ -591,6 +699,7 @@ export async function restartDaemon(
591
699
  // startDaemon() — that leaves the unit failed while handing the operator
592
700
  // a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
593
701
  // Any manager refusal is terminal; so is unproven ownership afterwards.
702
+ recordStop("systemctl", previous?.pid);
594
703
  const record = await restoreFailedUnit(o.timeoutMs, o.project);
595
704
  return { previous, record, via: "systemctl" };
596
705
  }
@@ -602,6 +711,7 @@ export async function restartDaemon(
602
711
  // Ownership is proven. A refused/timed-out restart must not fall through
603
712
  // to stopDaemon's signal path — that is the exact bounce this module exists
604
713
  // to prevent (SIGTERM → exit 143 → Restart=on-failure → new MainPID).
714
+ recordStop("systemctl", ownership.pid);
605
715
  const ran = systemctl(["restart", SYSTEMD_UNIT]);
606
716
  if (!ran.ok) {
607
717
  throw new Error(systemctlFailure("restart", ran));
@@ -615,6 +725,7 @@ export async function restartDaemon(
615
725
 
616
726
  // Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
617
727
  // is somebody else. The detached CLI daemon is the only path left.
728
+ recordStop("signal", previous?.pid);
618
729
  await stopDaemon({ timeoutMs: o.timeoutMs });
619
730
  const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
620
731
  return { previous, record, via: "cli" };
package/src/omp.ts CHANGED
@@ -44,6 +44,14 @@ export interface AgentSessionLike {
44
44
  * `"*"` for every event. The payload is the harness's own event union, which
45
45
  * this package cannot name without the peer dependency, so it arrives as
46
46
  * `unknown` and each caller narrows the two or three fields it reads.
47
+ *
48
+ * One event is not the harness's at all: `"session_exit"` fires when the
49
+ * underlying session *process* has terminated, carrying
50
+ * `{ type: "session_exit", code?: number | null }`. It is the single real
51
+ * terminal signal — a crashed or killed session is never confused with one
52
+ * that merely stopped streaming (`"agent_end"`). A supervisor may receive it
53
+ * after its own `dispose()`, and must decide what an exit means from its own
54
+ * state rather than assuming every exit is a crash.
47
55
  */
48
56
  on(event: string, cb: (e: unknown) => void): void;
49
57
  abort(): void;
@@ -714,8 +722,24 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
714
722
  pending.clear();
715
723
  };
716
724
 
725
+ // The one terminal event this proxy owns outright: the session process is
726
+ // gone, whether by crash, kill, or a clean dispose of its own. Delivered to
727
+ // subscribers *before* any failure is surfaced, so a supervisor learning of
728
+ // the death can also read its exit code off the same event. `disposing` is
729
+ // the proxy's own intent (its disposer asked the child to stop), not the
730
+ // subscriber's, so the event is emitted in both branches and each observer
731
+ // decides what an exit means from its own state.
732
+ const emitSessionExit = (code: number | null): void => {
733
+ const type = "session_exit";
734
+ const event = { type, code };
735
+ for (const cb of handlers.get(type) ?? []) cb(event);
736
+ for (const cb of handlers.get("*") ?? []) cb(event);
737
+ };
738
+
717
739
  void child.exited.then(async (code) => {
718
740
  onExit();
741
+ const exitCode = typeof code === "number" ? code : null;
742
+ emitSessionExit(exitCode);
719
743
  // A child that cannot start writes `start-error` over the socket, not the
720
744
  // pipes — and its exit can be dispatched before the accept of a connection
721
745
  // that already completed in the kernel. Let the socket settle before the