omp-conductor 0.20.1 → 0.20.3

package/src/doctor.ts

@@ -61,7 +61,9 @@ import {
   sessionDirForCwd,
   telegramStateDir,
 } from "./fleet.ts";
-import { pauseInstance } from "./pause.ts";
+import { errText } from "./log.ts";
+import { formatDownDuration } from "./orchestrator-down.ts";
+import { pauseInstance, pauseInstanceAt, pausedPath } from "./pause.ts";
 import type { TelegramHealth } from "./status-render.ts";
 import {
   claimIsLive,
@@ -71,6 +73,7 @@ import {
   killOf,
   readTelegramChannel,
   readTelegramDmOwner,
+  readTelegramMisroutes,
   readTelegramPollState,
   resolveClaimedSessionFile,
   resolveProjectClaim,
@@ -82,6 +85,7 @@ import {
   type TelegramChannelState,
   type TelegramDmOwnerState,
   type TelegramPollState,
+  type TelegramMisroute,
 } from "./escalate.ts";
 import {
   herdrConductorPluginConfigDir,
@@ -103,7 +107,7 @@ import {
   readTickConfig,
   type HerdrAgentList,
 } from "./orchestrator-tick.ts";
-import type { Caps, ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
+import type { Caps, ConductorConfig, Escalation, ProjectConfig, RepoTarget, RunState } from "./types.ts";
 import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts";
 import {
   DEFAULT_DEPS as UPGRADE_DEPS,
@@ -273,6 +277,13 @@ export interface DoctorDeps {
   /** Whether one pid is a live process. Injected so the abandoned-fence
    *  finding is testable without spawning anything. */
   pidLive?: (pid: number) => boolean;
+  /**
+   * Whether any conductor process is running an upgrade transaction right
+   * now (#1113) — the liveness fact an owner-less `upgrade` sentinel needs,
+   * because it records no pid of its own. Injected so the orphaned-upgrade
+   * fault is testable without scanning /proc.
+   */
+  upgradeLive?: () => boolean;
   /** Live `herdr --session <s> agent list`, parsed through the tick's own
    *  parser (the same "one JSON line on stdout" contract recover.sh reads). */
   herdrAgents?: (session: string) => HerdrAgentList;
@@ -286,6 +297,11 @@ export interface DoctorDeps {
    *  `unavailable` when the registry cannot be read or does not parse — the
    *  caller must not read that as "no claims" (#626). */
   claimedTopics?: () => ClaimedTopicsResult;
+  /** Recorded flat-chat deliveries this project's stale pin already caused
+   *  (#1094), read from conductor state. The topic-pin row renders them as
+   *  observed evidence until the pin is repaired — rows match the *current*
+   *  pin id, so re-pinning is what retires them. */
+  topicMisroutes?: () => readonly TelegramMisroute[];
   /** Whether the bridge's access.json runs per-session topic tidy on. */
   topicsTidy?: () => boolean;
   /** The typed bot.lock poll-ownership read, told apart so absent/malformed
@@ -1390,14 +1406,24 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
       `[${p.name}] no pinned topic id — sends go to the flat chat (live-claim substitution only follows a stale pin)`,
     );
   }
+  // #1094: sends that already rode the flat chat because of THIS pin turn the
+  // warnings below from hypothetical into observed. Rows are keyed to the pin
+  // id, so re-pinning retires them — nothing deletes them.
+  const misroutes = probes.topicMisroutes().filter(
+    (row) => row.project === p.name && row.staleTopicId === pinned,
+  );
+  const misrouteNote =
+    misroutes.length === 0
+      ? ""
+      : ` It has happened: ${misroutes.length} send(s) were delivered to the flat chat because of this pin, last at ${new Date(misroutes[misroutes.length - 1]!.at).toISOString()}`;
   const result = probes.claimedTopics();
   if (result.kind !== "ok") {
     return warnFinding(
       "topic-pin",
       `[${p.name}] pinned topic ${pinned} — ` +
         (result.kind === "missing"
-          ? `omp-telegram has no claim registry (threads.json in its state dir), so doctor cannot tell whether the pin is live or a claim answers to this project`
-          : `omp-telegram's claim registry is unreadable (${result.problem}), so doctor cannot tell whether the pin is live or a claim answers to this project`),
+          ? `omp-telegram has no claim registry (threads.json in its state dir), so doctor cannot tell whether the pin is live or a claim answers to this project${misrouteNote}`
+          : `omp-telegram's claim registry is unreadable (${result.problem}), so doctor cannot tell whether the pin is live or a claim answers to this project${misrouteNote}`),
       `check omp-telegram's claim registry (threads.json in its state dir) is readable and the bridge is running, then re-run doctor — until then sends keep the pinned topic and degrade to the flat chat on a missing thread (#318)`,
     );
   }
@@ -1423,7 +1449,7 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
     if (dead.length > 0) {
       return warnFinding(
         "topic-pin",
-        `[${p.name}] pinned topic ${pinned} — every claim in omp-telegram's registry is dead${deadNote}`,
+        `[${p.name}] pinned topic ${pinned} — every claim in omp-telegram's registry is dead${deadNote}${misrouteNote}`,
         `run /cleanup in Telegram so the bridge closes those topics and drops their rows itself, then re-run setup to re-pin escalation.telegramTopicId to a live claim — never delete rows from threads.json by hand, because the row is the only index to the remote topic (#987)`,
       );
     }
@@ -1437,7 +1463,7 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
   if (claims.some((claim) => claim.threadId === pinned)) {
     return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim${deadNote}`);
   }
-  const match = resolveProjectClaim(claims, p.name, probes.pidAlive);
+  const match = resolveProjectClaim(claims, p.name, probes.pidAlive, p.workspaceRoot);
   if (match.kind === "match") {
     return passFinding(
       "topic-pin",
@@ -1454,7 +1480,7 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
     return warnFinding(
       "topic-pin",
       `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, and several live claims answer to this project ` +
-        `(topics ${match.claimants.map((c) => c.threadId).join(", ")}) — substitution would be a coin toss, so sends keep the stale pin`,
+        `(topics ${match.claimants.map((c) => c.threadId).join(", ")}) — substitution would be a coin toss, so sends keep the stale pin${misrouteNote}`,
       `make exactly one live claim answer to this project: rename the other panes' herdr spaces or re-claim them one at a time, then re-run doctor — live-claim substitution follows the unique claim from there`,
     );
   }
@@ -1463,14 +1489,14 @@ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding {
       "topic-pin",
       `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, no live claim answers ` +
         `to this project, and this host runs topicsTidy — the pin is vestigial and can never be live again ` +
-        `(every pane exit closes its topic); sends keep the stale pin`,
+        `(every pane exit closes its topic); sends keep the stale pin${misrouteNote}`,
       `turn topicsTidy off, then re-run setup to re-pin escalation.telegramTopicId to a currently claimed topic`,
     );
   }
   return warnFinding(
     "topic-pin",
     `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims (${claims.map((c) => c.threadId).join(", ")}), ` +
-      `and no live claim answers to this project — sends keep the stale pin and degrade to the flat chat`,
+      `and no live claim answers to this project — sends keep the stale pin and degrade to the flat chat${misrouteNote}`,
     `update escalation.telegramTopicId to a currently claimed topic, or re-run setup to re-pin`,
   );
 }
@@ -1654,6 +1680,286 @@ function fenceProbe(probes: Probes, project: string | undefined): Finding {
   );
 }
 
+/**
+ * Whether any conductor process is running an upgrade transaction (#1113).
+ *
+ * The upgrade sentinel records no `owner=` — unlike a setup fence, whose pid
+ * is precise, the drain spans subprocesses and the detached installer is a
+ * transient unit — so the honest question is whether ANY conductor process is
+ * mid-upgrade at all. Read from `/proc` cmdlines, never inferred from journal
+ * state: a journal outlives its crash, which is exactly the ambiguity this
+ * check exists to cut through. The whole transaction family matches
+ * (`upgrade`, the detached `upgrade-install`, `upgrade-rollback`); unreadable
+ * entries are skipped, and a zombie's empty cmdline cannot match.
+ */
+function upgradeProcessLive(): boolean {
+  try {
+    for (const entry of readdirSync("/proc")) {
+      if (!/^\d+$/.test(entry)) continue;
+      let raw: string;
+      try {
+        raw = readFileSync(join("/proc", entry, "cmdline"), "utf8");
+      } catch {
+        continue; // vanished mid-scan, or not ours to read
+      }
+      const argv = raw.split("\0").filter((arg) => arg.length > 0);
+      const conductor = argv.some((arg) => /(^|\/)(omp-conductor|cli\.ts)$/.test(arg));
+      if (conductor && argv.some((arg) => arg === "upgrade" || arg.startsWith("upgrade-"))) return true;
+    }
+  } catch {
+    return false;
+  }
+  return false;
+}
+
+/**
+ * The pause-state judgement shared by the doctor row and the daemon watch
+ * (#1113), so the two surfaces cannot drift apart about which pauses still
+ * have a plausible owner. The rules, in the order they are applied:
+ *
+ *  - a recorded `owner=` is the precise fact whenever one exists (#938);
+ *  - `hold` is supported at any age — stopping the fleet deliberately is not
+ *    a fault, and must never nag;
+ *  - `upgrade` owns its drain by existing: a live transaction is fine, an
+ *    exited one is the one fault class, because no future event can ever
+ *    clear that sentinel;
+ *  - `spend-cap` clears itself when spend falls back below the cap;
+ *  - every other durable hold is somebody's decision, reported as such.
+ */
+export type HeldPause = {
+  kind: "held";
+  verdict:
+    | "operator-hold"
+    | "self-expiring"
+    | "durable"
+    | "live-owner"
+    | "live-upgrade"
+    | "dead-owner"
+    | "orphaned-upgrade";
+  source: string;
+  reason?: string;
+  since: number;
+  owner?: number;
+};
+
+export type PauseVerdict = { kind: "unpaused" } | HeldPause;
+
+export interface PauseLiveness {
+  upgradeLive(): boolean;
+  pidLive(pid: number): boolean;
+}
+
+export function classifyPause(
+  pause: { source: string; reason?: string; since: number; owner?: number } | undefined,
+  liveness: PauseLiveness,
+): PauseVerdict {
+  if (pause === undefined) return { kind: "unpaused" };
+  const base: Omit<HeldPause, "verdict"> = {
+    kind: "held",
+    source: pause.source,
+    ...(pause.reason === undefined ? {} : { reason: pause.reason }),
+    since: pause.since,
+    ...(pause.owner === undefined ? {} : { owner: pause.owner }),
+  };
+  if (pause.owner !== undefined) {
+    return { ...base, verdict: liveness.pidLive(pause.owner) ? "live-owner" : "dead-owner" };
+  }
+  if (pause.source === "hold") return { ...base, verdict: "operator-hold" };
+  if (pause.source === "upgrade") {
+    return { ...base, verdict: liveness.upgradeLive() ? "live-upgrade" : "orphaned-upgrade" };
+  }
+  if (pause.source === "spend-cap") return { ...base, verdict: "self-expiring" };
+  return { ...base, verdict: "durable" };
+}
+
+/** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */
+const PAUSE_PAGE_ISSUE = 0;
+
+/**
+ * The orphaned-upgrade page (#1113). Anchored on the sentinel's `since`, like
+ * every once-per-incident page here: the notification ledger dedupes on the
+ * summary, so a still-orphaned sentinel pages exactly once while each newly
+ * recreated sentinel — a fresh `since` — pages again.
+ */
+export function orphanedPauseEscalation(project: string, pause: HeldPause): Escalation {
+  const since = new Date(pause.since).toISOString();
+  return {
+    tier: 2,
+    category: "fleet-stopped",
+    project,
+    issue: PAUSE_PAGE_ISSUE,
+    summary: `Orphaned upgrade pause holds dispatch since ${since} — no upgrade process is running (${project})`,
+    detail: [
+      `Dispatch has been paused by the upgrade sentinel since ${since}` +
+        (pause.reason === undefined ? "" : ` — "${pause.reason}"`) +
+        ".",
+      "The upgrade transaction that wrote it has exited, so no future event will",
+      "ever clear it: dispatch stays stopped until someone resumes.",
+      "",
+      "Remedy: `omp-conductor resume --all`. Nothing auto-resumes — reporting is",
+      "the whole contract (#1113); resuming a pause nobody authorised stays yours.",
+    ].join("\n"),
+  };
+}
+
+/** One pause sentinel read: who set it, why, when — and any recorded owner. */
+type PauseInstance = { source: string; reason?: string; since: number; owner?: number };
+
+export interface OrphanedPauseDeps {
+  /** The project this daemon serves; the effective sentinel for it reads
+   *  through the same precedence the doctor row uses (project wins over the
+   *  legacy global file). */
+  project: string;
+  escalate(e: Escalation): Promise<void>;
+  log?(msg: string): void;
+  pause?: () => PauseInstance | undefined;
+  /** This project's own `paused-<project>` sentinel, told apart from the
+   *  legacy global one so a genuinely per-project orphan pages for its own
+   *  project while a shared one pages exactly once (#1113 review round 1). */
+  projectSentinel?: () => PauseInstance | undefined;
+  globalSentinel?: () => PauseInstance | undefined;
+  upgradeLive?: () => boolean;
+  pidLive?: (pid: number) => boolean;
+  /** Every configured project name — from the shared config.json every
+   *  daemon on this host reads, which is what makes the canonical owner
+   *  below one identity everywhere. Defaults to {@link loadConfig}. */
+  configuredProjects?: () => string[];
+}
+
+/**
+ * Best-effort page delivery, mirroring `safeEscalate`: a transport failure is
+ * logged, never thrown, and cannot take the tick down.
+ */
+async function deliverOrphanPage(
+  deps: OrphanedPauseDeps,
+  write: (msg: string) => void,
+  held: HeldPause,
+): Promise<void> {
+  let delivered = true;
+  try {
+    await deps.escalate(orphanedPauseEscalation(deps.project, held));
+  } catch (err) {
+    delivered = false;
+    write(`orphaned-pause escalation could not be delivered: ${errText(err)}`);
+  }
+  write(
+    `ERROR: an orphaned upgrade pause holds dispatch (since ${new Date(held.since).toISOString()})` +
+      (delivered ? " — paged once for this sentinel" : " — page unsent"),
+  );
+}
+
+/**
+ * The daemon half of #1113: raise the orphaned-upgrade case through the
+ * ordinary escalation path instead of waiting for a tick to read `status` —
+ * a stopped fleet is precisely the condition that cannot assume anything is
+ * watching. Called above the pause gate on every pass; nothing here ever
+ * resumes.
+ *
+ * "Once" needs an ownership boundary, because the ledger that dedupes pages
+ * is per-project store while the upgrade sentinel usually is not: it writes
+ * the legacy GLOBAL file, and every project's effective read falls back to
+ * it. So the two sentinel kinds are told apart before paging:
+ *
+ *  - this project's own `paused-<project>` orphan → this project pages as
+ *    itself, whatever else is configured;
+ *  - the legacy global orphan → exactly one daemon owns the incident: the
+ *    one serving the alphabetically-first configured project name — the same
+ *    name on every daemon, because they all read the same config.json. A
+ *    daemon that cannot read that config assumes it is alone and pages
+ *    anyway; staying silent on a failed read would be the exact silence
+ *    #1113 exists to end.
+ *
+ * Within either owner, the ledger dedupes on the since-anchored summary, so
+ * a still-orphaned sentinel pages once and a recreated one (fresh `since`)
+ * pages again.
+ */
+export async function reconcileOrphanedPause(deps: OrphanedPauseDeps): Promise<void> {
+  const write =
+    deps.log ??
+    ((msg: string) => process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`));
+  const liveness: PauseLiveness = {
+    upgradeLive: deps.upgradeLive ?? upgradeProcessLive,
+    pidLive: deps.pidLive ?? pidAlive,
+  };
+  const verdict = classifyPause((deps.pause ?? (() => pauseInstance(deps.project)))(), liveness);
+  if (verdict.kind !== "held" || verdict.verdict !== "orphaned-upgrade") return;
+
+  // A per-project sentinel of its own: this project owns that page outright.
+  const ownVerdict = classifyPause(
+    (deps.projectSentinel ?? (() => pauseInstanceAt(pausedPath(deps.project))))(),
+    liveness,
+  );
+  if (ownVerdict.kind === "held" && ownVerdict.verdict === "orphaned-upgrade") {
+    await deliverOrphanPage(deps, write, ownVerdict);
+    return;
+  }
+
+  // The shared global sentinel: gate on the canonical owner before paging.
+  let configured: string[] | undefined;
+  try {
+    configured = deps.configuredProjects?.() ?? loadConfig().projects.map((p) => p.name);
+  } catch {
+    configured = undefined;
+  }
+  const names = configured ?? [];
+  const owner = names.length === 0 ? deps.project : [...names].sort()[0];
+  if (deps.project !== owner) {
+    write(`orphaned upgrade pause holds dispatch — ${owner} owns the page for the global sentinel`);
+    return;
+  }
+  const globalVerdict = classifyPause(
+    (deps.globalSentinel ?? (() => pauseInstanceAt(pausedPath())))(),
+    liveness,
+  );
+  if (globalVerdict.kind === "held" && globalVerdict.verdict === "orphaned-upgrade") {
+    await deliverOrphanPage(deps, write, globalVerdict);
+  }
+}
+
+/**
+ * The dispatch-pause row itself (#1113), beside {@link fenceProbe}: the fence
+ * row judges a declared owning process, this one judges the pause — `pass`
+ * when unpaused, and when paused naming the source, the age, and whether
+ * anything still plausibly owns it. One fault class only: the orphaned
+ * upgrade sentinel, which can never clear itself. Nothing here resumes
+ * anything; reporting is the whole contract.
+ */
+function pauseProbe(probes: Probes, project: string | undefined): Finding {
+  const id = project === undefined ? "dispatch-pause" : `dispatch-pause:${project}`;
+  const v = classifyPause(probes.pauseFence(project), {
+    upgradeLive: probes.upgradeLive,
+    pidLive: probes.pidLive,
+  });
+  if (v.kind === "unpaused") return passFinding(id, "dispatch is not paused");
+  const why = v.reason === undefined ? "" : ` — "${v.reason}"`;
+  const age = formatDownDuration(Math.max(0, probes.now() - v.since));
+  const base = `dispatch paused by ${v.source}${why}, ${age} ago`;
+  switch (v.verdict) {
+    case "orphaned-upgrade":
+      return failFinding(
+        id,
+        `${base}, and no upgrade process is running — this pause can never clear itself`,
+        "run `omp-conductor resume --all` (nothing here resumes it)",
+      );
+    case "dead-owner":
+      return warnFinding(
+        id,
+        `${base} — its owning process (pid ${v.owner}) is gone`,
+        `confirm nothing is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``,
+      );
+    case "operator-hold":
+      return passFinding(id, `${base} — a deliberate hold, supported at any age`);
+    case "live-upgrade":
+      return passFinding(id, `${base} — the upgrade transaction is draining, and resumes dispatch when it finishes`);
+    case "live-owner":
+      return passFinding(id, `${base} — owned by live pid ${v.owner}`);
+    case "self-expiring":
+      return passFinding(id, `${base} — clears itself when spend falls below the cap`);
+    case "durable":
+      return passFinding(id, `${base} — no owning process; held until someone resumes it`);
+  }
+}
+
 function herdrResumeProbe(probes: Probes): Finding {
   if (!probes.herdrInstalled()) {
     return passFinding("herdr-resume", "herdr not installed — nothing to check");
@@ -2049,12 +2355,15 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
     findings.push(timezoneProbe(undefined));
     findings.push(await telegramProbe(probes, undefined, checkedAt));
     findings.push(fenceProbe(probes, undefined));
+    findings.push(pauseProbe(probes, undefined));
   } else {
     for (const p of projects) findings.push(timezoneProbe(p));
     for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt));
     // Per project, because the sentinel is: one fleet's fence must never be
     // reported against another's name.
     for (const p of projects) findings.push(fenceProbe(probes, p.name));
+    // The pause state itself (#1113), per project for the same reason.
+    for (const p of projects) findings.push(pauseProbe(probes, p.name));
   }
   // The billing class the spend row needs (#984). Declared only when EVERY
   // project requires a subscription credential: on a mixed fleet the zeros of
@@ -2156,6 +2465,7 @@ export function defaultProbes(): Probes {
     canonicalUnits: defaultCanonicalUnits,
     herdrInstalled: () => Bun.which("herdr") !== null,
     pauseFence: (project) => pauseInstance(project),
+    upgradeLive: upgradeProcessLive,
     workerHarness: defaultWorkerHarness,
     pidLive: (pid) => pidAlive(pid),
     herdrAgents: defaultHerdrAgents,
@@ -2163,6 +2473,7 @@ export function defaultProbes(): Probes {
     herdrConfig: defaultHerdrConfig,
     herdrEnv: defaultHerdrEnv,
     claimedTopics: () => claimedTelegramTopics(),
+    topicMisroutes: () => readTelegramMisroutes(),
     topicsTidy: () => telegramTopicsTidy(),
     pollState: () => readTelegramPollState(),
     dmOwner: () => readTelegramDmOwner(),