omp-conductor 0.15.9 → 0.15.11

package/src/doctor.ts

@@ -0,0 +1,795 @@
+/**
+ * `doctor` — mechanically check the deployment faults that have actually cost
+ * debugging sessions, before the fleet hits them (#287, legibility 1/5).
+ *
+ * Every production incident this reports on was environmental, silent, and
+ * cheap to detect: a systemd unit whose user/working-directory did not match
+ * the host (#152/#153 class), case-mismatched routing labels documented to
+ * fail silently (README Limitations), spend telemetry absent so the USD cap
+ * never fired, `gh` auth expiring under a live daemon. None of them had a
+ * check before the incident; this module is that check.
+ *
+ * Two properties are load-bearing and enforced here rather than documented:
+ *
+ *   - **Read-only by default.** The store is opened as a `readonly` sqlite
+ *     connection; nothing writes config, labels, systemd, or the database.
+ *     The single side effect anywhere is the self-identified Telegram probe
+ *     message, and only `--probe-telegram` enables it.
+ *   - **Every probe is injectable.** The runner composes probes through
+ *     {@link DoctorDeps}; the same seams the rest of the package uses — the
+ *     tracker's own `gh` transport, `probeTelegramHealth`, the report
+ *     outbox's send, the `renderDaemonService` runtime plan — are what a
+ *     default run wires, so `doctor` cannot disagree with the daemon about a
+ *     shared fact. Tests inject every probe, and no test touches the host's
+ *     systemd unit, daemon, Telegram, GitHub account, or production state
+ *     (the #399 boundary).
+ */
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { join } from "node:path";
+import type { Stats } from "node:fs";
+import { Database } from "bun:sqlite";
+
+import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir } from "./config.ts";
+import { DEFAULT_HERDR_UNIT, probeTelegramHealth, telegramStateDir, type TelegramHealth } from "./fleet.ts";
+import { planHostRuntime, STAGED_SERVICE_NAME, SYSTEMD_UNIT_DIR, totalConfiguredWorkers } from "./setup-host.ts";
+import { checkTokenScopes, type ScopeCheck } from "./setup.ts";
+import { dbPath, LIVE_STATES } from "./store.ts";
+import { telegramReportSend, type ReportSend } from "./reports.ts";
+import { fetchRateLimit, GhError, gh } from "./tracker/github.ts";
+import { repoSlugFor } from "./gitops.ts";
+import type { ConductorConfig, ProjectConfig, RepoTarget, RunState } from "./types.ts";
+
+/**
+ * `doctor`'s finding vocabulary. `pass`/`warn`/`fail` — a warning renders the
+ * same but never flips the exit code; exit is nonzero iff any finding fails.
+ */
+export type FindingStatus = "pass" | "warn" | "fail";
+export type ReportStatus = "pass" | "warn" | "fail";
+
+/** One checked fact: id (stable for CI), status, summary, and a one-line fix. */
+export interface Finding {
+  /** Stable identifier — the JSON shape is a CI contract, so ids never move. */
+  id: string;
+  status: FindingStatus;
+  summary: string;
+  /** The one-line remedy; absent when the finding passes. */
+  fix?: string;
+}
+
+/**
+ * The stable report shape `--json` prints and the human renderer reads.
+ *
+ * `checkedAt` is the UTC ISO instant the run started; `status` is `fail` iff
+ * any finding is `fail`, else `warn` iff any is `warn`, else `pass`.
+ */
+export interface DoctorReport {
+  project: string;
+  checkedAt: string;
+  status: ReportStatus;
+  findings: Finding[];
+}
+
+/** How many of the most recent completed runs are sampled for spend telemetry. */
+export const SPEND_SAMPLE_RUNS = 5;
+
+/** The past incidents this doctor flags, quoted in the finding so an operator
+ *  who never lived them knows which failure mode the check exists to prevent. */
+const INCIDENTS = {
+  systemd:
+    "the unit/user mismatch class that once killed workers at turn 0 with an empty agent.db (#152/#153)",
+  labels:
+    "label matching is exact and case-sensitive, and a case-mismatched label is documented to fail silently (README Limitations)",
+  spend:
+    "spend telemetry was once absent, so the USD cap never fired — $0.00 spend is not proof of no spend",
+  ghauth: "gh auth expired under a live daemon and every tracker call failed silently",
+} as const;
+
+// ------------------------------------------------------------------ dependencies
+
+export interface GhRepoRead {
+  ok: boolean;
+  detail?: string;
+}
+
+export interface GhLabelsRead {
+  ok: boolean;
+  labels?: string[];
+  detail?: string;
+}
+
+/** The canonical units `setup host`/`upgrade` would stage right now. */
+export interface CanonicalUnits {
+  daemon: string;
+  herdr?: string;
+  /** The account the daemon unit renders with, for the ownership checks. */
+  username: string;
+}
+
+/** One sampled run row: `state` keeps spend from in-flight rows, which are not
+ *  a telemetry statement yet. */
+export interface RunSpendRow {
+  state: RunState;
+  spendUsd: number;
+}
+
+/** Injectable seams. Every field defaults to the production wiring, which
+ *  reuses the package's existing transports — no second GitHub client, no new
+ *  Telegram HTTP call, no re-parsed status. */
+export interface DoctorDeps {
+  /** Defaults to {@link loadConfig}; a loader that throws is a finding, not a crash. */
+  loadConfig?: () => ConductorConfig;
+  /** `gh auth status` scope read — the wizard's own check, so setup consent
+   *  and `doctor` cannot disagree about what the token may do. */
+  scopes?: () => Promise<ScopeCheck>;
+  /** `gh api rate_limit` succeeds — the tracker's own liveness probe. */
+  rateLimitOk?: () => Promise<boolean>;
+  /** Whether `gh` can read one configured repo (scopes cover it). */
+  repoReadable?: (repo: string) => Promise<GhRepoRead>;
+  /** The exact labels present in one repo, for the exact-case check. */
+  repoLabels?: (repo: string) => Promise<GhLabelsRead>;
+  /** True when this is a systemd host (`/etc/systemd/system` exists). */
+  hasSystemd?: () => boolean;
+  /** The installed unit text at an absolute path, or undefined when absent. */
+  readUnit?: (path: string) => string | undefined;
+  /** stat one path; undefined when it does not exist. */
+  stat?: (path: string) => Stats | undefined;
+  /** uid of a username (`id -u`), or undefined when it cannot be resolved. */
+  uidOf?: (name: string) => number | undefined;
+  /** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */
+  dbIntegrity?: (path: string) => { ok: boolean; detail?: string };
+  /** The most recent completed runs for one project, newest first, limited. */
+  recentRuns?: (project: string, limit: number) => RunSpendRow[];
+  /** The bot health probe `status` uses (getMe etc.). */
+  telegramHealth?: (projectName: string) => Promise<TelegramHealth>;
+  /** The report-transport send, per project; only ever invoked with
+   *  `--probe-telegram`. Defaults to the report outbox's own transport. */
+  telegramSend?: (project: ProjectConfig) => ReportSend;
+  /** The canonical units to compare the installed units against. */
+  canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits;
+  /** Clock, so a run is deterministic in tests. */
+  now?: () => number;
+  /** The one opt-in side effect: send one self-identified Telegram probe. */
+  probeTelegram?: boolean;
+}
+
+/** The resolved probe set a run uses (defaults merged with the overrides). */
+export type Probes = Required<Omit<DoctorDeps, "telegramSend" | "probeTelegram">> &
+  Pick<DoctorDeps, "telegramSend" | "probeTelegram">;
+
+function messageOf(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+function firstLineOf(stderr: string): string {
+  const line = stderr.split("\n").find((l) => l.trim().length > 0)?.trim() ?? "";
+  return line.length > 120 ? `${line.slice(0, 120)}…` : line;
+}
+
+/** A gh failure → a bounded, human line (never a credential or a stack). */
+function ghFailureDetail(err: unknown): string {
+  if (err instanceof GhError) return firstLineOf(err.stderr);
+  return messageOf(err);
+}
+
+// --------------------------------------------------------------- production wiring
+
+async function defaultRepoReadable(repo: string): Promise<GhRepoRead> {
+  try {
+    await gh(["api", `repos/${repo}`]);
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, detail: ghFailureDetail(err) };
+  }
+}
+
+async function defaultRepoLabels(repo: string): Promise<GhLabelsRead> {
+  try {
+    const raw = await gh(["label", "list", "--repo", repo, "--limit", "1000", "--json", "name"]);
+    const parsed: unknown = JSON.parse(raw);
+    if (!Array.isArray(parsed)) return { ok: false, detail: "gh label list returned an unexpected shape" };
+    const labels: string[] = [];
+    for (const row of parsed) {
+      const name = (row as { name?: unknown } | null)?.name;
+      if (typeof name === "string" && name.length > 0) labels.push(name);
+    }
+    return { ok: true, labels };
+  } catch (err) {
+    return { ok: false, detail: ghFailureDetail(err) };
+  }
+}
+
+function defaultReadUnit(path: string): string | undefined {
+  try {
+    return readFileSync(path, "utf8");
+  } catch {
+    return undefined;
+  }
+}
+
+function defaultStat(path: string): Stats | undefined {
+  try {
+    return statSync(path);
+  } catch {
+    return undefined;
+  }
+}
+
+function defaultUidOf(name: string): number | undefined {
+  const ran = spawnSync("id", ["-u", name], { encoding: "utf8" });
+  if (ran.status !== 0) return undefined;
+  const uid = Number.parseInt((ran.stdout ?? "").trim(), 10);
+  return Number.isInteger(uid) && uid >= 0 ? uid : undefined;
+}
+
+/**
+ * `PRAGMA integrity_check`, opened **read-only** so the check itself can never
+ * mutate the store it is inspecting. A missing file is a fresh host, not a
+ * fault; a file that is not sqlite, or a check that reports anything but
+ * `ok`, is.
+ */
+function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } {
+  if (!existsSync(path)) return { ok: true, detail: "no runs store yet — nothing to check" };
+  let database: Database;
+  try {
+    database = new Database(path, { readonly: true });
+  } catch (err) {
+    return { ok: false, detail: `store cannot be opened read-only: ${messageOf(err)}` };
+  }
+  try {
+    const rows = database.query<{ integrity_check: string }, []>("PRAGMA integrity_check").all();
+    for (const row of rows) {
+      if (row["integrity_check"] !== "ok") {
+        return { ok: false, detail: `integrity_check: ${row["integrity_check"]}` };
+      }
+    }
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, detail: `integrity_check failed: ${messageOf(err)}` };
+  } finally {
+    database.close();
+  }
+}
+
+/**
+ * The most recent completed runs for one project, newest first — read on the
+ * same read-only connection the integrity check opens, because the store's own
+ * methods mutate on open and `doctor` is read-only. The state filter uses the
+ * store's own `LIVE_STATES` vocabulary, so `doctor` never re-decides what
+ * "completed" means.
+ */
+function defaultRecentRuns(project: string, limit: number): RunSpendRow[] {
+  const path = dbPath();
+  if (!existsSync(path)) return [];
+  let database: Database;
+  try {
+    database = new Database(path, { readonly: true });
+  } catch {
+    return [];
+  }
+  try {
+    const placeholders = [...LIVE_STATES].map(() => "?").join(", ");
+    const rows = database
+      .query<{ state: string; spendUsd: number }, [string, ...string[], number]>(
+        `SELECT state, spendUsd FROM runs
+          WHERE project = ? AND state NOT IN (${placeholders})
+          ORDER BY startedAt DESC, rowid DESC
+          LIMIT ?`,
+      )
+      .all(project, ...LIVE_STATES, limit);
+    const out: RunSpendRow[] = [];
+    for (const row of rows) {
+      const state = row.state as RunState;
+      // A state vocabulary the store never had is not "completed".
+      if (!LIVE_STATES.includes(state)) out.push({ state, spendUsd: row.spendUsd });
+    }
+    return out;
+  } catch {
+    return [];
+  } finally {
+    database.close();
+  }
+}
+
+/** The canonical units — exactly what `setup host`/`upgrade` stage. */
+function defaultCanonicalUnits(project: ProjectConfig, cfg: ConductorConfig): CanonicalUnits {
+  const caps = resolveCaps(project, cfg.defaults);
+  const plan = planHostRuntime(project, caps, telegramStateDir(), undefined, totalConfiguredWorkers(cfg));
+  const username = /^User=(.*)$/m.exec(plan.service.value)?.[1]?.trim() ?? "";
+  return {
+    daemon: plan.service.value,
+    ...(plan.herdrUnit === undefined ? {} : { herdr: plan.herdrUnit.value }),
+    username,
+  };
+}
+
+// --------------------------------------------------------------- finding helpers
+
+function passFinding(id: string, summary: string): Finding {
+  return { id, status: "pass", summary };
+}
+
+function warnFinding(id: string, summary: string, fix: string): Finding {
+  return { id, status: "warn", summary, fix };
+}
+
+function failFinding(id: string, summary: string, fix: string): Finding {
+  return { id, status: "fail", summary, fix };
+}
+
+// ------------------------------------------------------------------ probes
+
+function configProbe(problem: string | undefined): Finding {
+  if (problem === undefined) return passFinding("config", "config.json loads and validates");
+  return failFinding(
+    "config",
+    `config.json does not load: ${problem}`,
+    "repair it (run `omp-conductor setup`) before any other check can mean anything",
+  );
+}
+
+/**
+ * The config backup (#193): a pre-write copy at least as fresh as the file it
+ * protects. A config that was never written through the atomic writer — or
+ * that was hand-edited after the last save — has no backup for the bad next
+ * write to be rolled back from.
+ */
+function backupProbe(probes: Probes): Finding {
+  const backups = (() => {
+    try {
+      return readdirSync(configBackupDir()).filter((name) => name.startsWith("config.json.bak-"));
+    } catch {
+      return [];
+    }
+  })();
+  const config = probes.stat(configPath());
+  if (config === undefined) {
+    return backups.length === 0
+      ? passFinding("config-backup", "no config.json yet — nothing to back up")
+      : passFinding("config-backup", `${backups.length} config backup(s) present`);
+  }
+  if (backups.length === 0) {
+    return warnFinding(
+      "config-backup",
+      "config.json has no backup — the next bad config write destroys the only copy",
+      "re-save the config (run `omp-conductor setup`) so writeConfigRaw files a pre-write backup",
+    );
+  }
+  const newestMtime = backups
+    .map((name) => probes.stat(join(configBackupDir(), name))?.mtimeMs ?? 0)
+    .reduce((max, m) => Math.max(max, m), 0);
+  if (config.mtimeMs > newestMtime + 60_000) {
+    return warnFinding(
+      "config-backup",
+      `config.json is newer than every backup (${backups.length} file(s)) — the newest edit was never backed up`,
+      "save config.json through `omp-conductor setup` so a fresh pre-write backup lands",
+    );
+  }
+  return passFinding("config-backup", `config backup is fresh (${backups.length} file(s))`);
+}
+
+/** The store's `PRAGMA integrity_check` — corruption was previously silent. */
+function dbProbe(probes: Probes): Finding {
+  const result = probes.dbIntegrity(dbPath());
+  if (result.ok) {
+    return passFinding(
+      "db",
+      result.detail === undefined ? "conductor.db integrity is ok" : `conductor.db: ${result.detail}`,
+    );
+  }
+  return failFinding(
+    "db",
+    `conductor.db failed its integrity check: ${result.detail ?? "unknown"} — a corrupt store has been silent until nothing dispatches`,
+    "stop the daemon, restore the store from a backup, or rebuild it",
+  );
+}
+
+/** Every repo the config names: each project's tracker plus routing targets,
+ *  as GitHub slugs, deduplicated, in config order. */
+function configuredRepos(cfg: ConductorConfig | undefined): string[] {
+  if (cfg === undefined) return [];
+  const repos: string[] = [];
+  for (const project of cfg.projects) {
+    for (const repo of [
+      project.tracker.repo,
+      ...Object.values(project.routing.repos).map((r: RepoTarget) => repoSlugFor(r)),
+    ]) {
+      if (!repos.includes(repo)) repos.push(repo);
+    }
+  }
+  return repos;
+}
+
+/**
+ * gh auth: token alive, rate-limit readable, effective scopes cover every
+ * configured repo. The daemon's tracker reads go through the same transport,
+ * so a failure here is the expired-token class with the daemon already live.
+ */
+async function ghAuthProbe(probes: Probes, repos: string[]): Promise<Finding> {
+  const [scopes, rateOk] = await Promise.all([probes.scopes(), probes.rateLimitOk()]);
+  const failures: string[] = [];
+  const notes: string[] = [];
+  if (scopes !== undefined) {
+    if (scopes.ok) {
+      notes.push(scopes.login === undefined ? "authenticated" : `authenticated as ${scopes.login}`);
+      notes.push(`scopes: ${scopes.scopes.join(", ") || "none listed"}`);
+    } else {
+      failures.push(
+        scopes.login === undefined
+          ? "`gh auth status` says not authenticated"
+          : `token for ${scopes.login} is missing scopes ${scopes.missing.join(", ") || "(none listed)"}`,
+      );
+    }
+  } else {
+    failures.push("`gh auth status` did not answer");
+  }
+  if (rateOk) notes.push("rate-limit API readable");
+  else failures.push("`gh api rate_limit` fails — the daemon's tracker calls would fail the same way");
+
+  const reads = await Promise.all(repos.map(async (repo) => ({ repo, read: await probes.repoReadable(repo) })));
+  for (const r of reads) {
+    if (r.read.ok) notes.push(`${r.repo} readable`);
+    else failures.push(`configured repo ${r.repo} is not readable with this token${r.read.detail === undefined ? "" : ` (${r.read.detail})`}`);
+  }
+  if (failures.length === 0) {
+    return passFinding("gh-auth", `gh is valid ${notes.length === 0 ? "" : `— ${notes.join("; ")}`}`.trim());
+  }
+  return failFinding(
+    "gh-auth",
+    `${failures.join("; ")} — ${INCIDENTS.ghauth}`,
+    "run `gh auth login` and grant the scopes the wizard requires (setup's checkTokenScopes names them)",
+  );
+}
+
+/**
+ * Every label the dispatch loop depends on, with exact case — a case-mismatched
+ * label is documented to fail silently, so doctor names the near-miss instead
+ * of just the absence.
+ */
+async function labelProbe(probes: Probes, project: ProjectConfig): Promise<Finding> {
+  const wanted: { kind: string; name: string }[] = [
+    { kind: "queue", name: project.queueLabel },
+    { kind: "state", name: project.stateLabels.inProgress },
+    { kind: "state", name: project.stateLabels.blocked },
+    { kind: "state", name: project.stateLabels.failed },
+    ...Object.keys(project.routing.repos).map((key) => ({
+      kind: "routing",
+      name: `${project.routing.labelPrefix}${key}`,
+    })),
+  ];
+  const read = await probes.repoLabels(project.tracker.repo);
+  if (!read.ok) {
+    return failFinding(
+      "labels",
+      `cannot read the labels of ${project.tracker.repo}${read.detail === undefined ? "" : ` (${read.detail})`}`,
+      "check `gh` can list that repo's labels — the daemon reads them the same way",
+    );
+  }
+  const present = new Set(read.labels ?? []);
+  const norm = (s: string): string => s.toLowerCase().replace(/[-_\s:]+/g, "");
+  const nearByName = new Map<string, string>();
+  for (const label of read.labels ?? []) {
+    if (!nearByName.has(norm(label))) nearByName.set(norm(label), label);
+  }
+  const missing = wanted.filter((w) => !present.has(w.name));
+  if (missing.length === 0) {
+    return passFinding("labels", `${wanted.length} configured label(s) on ${project.tracker.repo} match exactly`);
+  }
+  const detail = missing.map((m) => {
+    const near = nearByName.get(norm(m.name));
+    return near === undefined || near === m.name ? `"${m.name}" (${m.kind})` : `"${m.name}" (${m.kind}) — near-miss present: "${near}"`;
+  });
+  const allNearMisses = missing.every((m) => {
+    const near = nearByName.get(norm(m.name));
+    return near !== undefined && near !== m.name;
+  });
+  return failFinding(
+    "labels",
+    `${detail.join("; ")} — ${INCIDENTS.labels}`,
+    allNearMisses
+      ? "fix the label case/format in the tracker repo (or change the config to the actual spelling)"
+      : "create the missing label(s) in the tracker repo (`gh label create`, or re-run `omp-conductor setup`)",
+  );
+}
+
+/** Line-level drift between an installed unit and the canonical render. */
+export function unitDrift(installed: string, canonical: string): { differences: string[] } {
+  const want = canonical.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
+  const have = installed.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
+  const wantSet = new Set(want);
+  const haveSet = new Set(have);
+  const differences = [
+    ...want.filter((l) => !haveSet.has(l)).map((l) => `missing/differs: ${l}`),
+    ...have.filter((l) => !wantSet.has(l)).map((l) => `extra: ${l}`),
+  ];
+  return { differences };
+}
+
+function driftLines(name: string, installed: string, canonical: string): string[] {
+  const { differences } = unitDrift(installed, canonical);
+  if (differences.length === 0) return [];
+  const shown = differences.slice(0, 4).join("; ");
+  const more = differences.length > 4 ? ` (+${differences.length - 4} more)` : "";
+  return [`${name} differs from the staged unit: ${shown}${more}`];
+}
+
+/**
+ * Installed unit vs the canonical staged rendering: any drift in User=,
+ * Environment, WorkingDirectory, ExecStart, Restart/SuccessExitStatus, or
+ * memory lines is the unit-drift class that once killed workers at turn 0.
+ */
+function unitProbe(probes: Probes, project: ProjectConfig | undefined, cfg: ConductorConfig | undefined): Finding {
+  if (!probes.hasSystemd()) return passFinding("systemd-unit", "no systemd directory on this host — nothing to check");
+  if (project === undefined || cfg === undefined) {
+    return passFinding("systemd-unit", "config unreadable — the canonical staged unit is unknown, nothing to compare");
+  }
+  const canonical = probes.canonicalUnits(project, cfg);
+  const installedDaemon = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME));
+  if (installedDaemon === undefined) {
+    return warnFinding("systemd-unit", `${STAGED_SERVICE_NAME} is not installed`, "install it: run `omp-conductor setup host` from the fleet account");
+  }
+  const problems = driftLines(STAGED_SERVICE_NAME, installedDaemon, canonical.daemon);
+  if (canonical.herdr !== undefined) {
+    const installedHerdr = probes.readUnit(join(SYSTEMD_UNIT_DIR, DEFAULT_HERDR_UNIT));
+    if (installedHerdr === undefined) {
+      problems.push(`${DEFAULT_HERDR_UNIT} is not installed (the staged plan provisions it)`);
+    } else {
+      problems.push(...driftLines(DEFAULT_HERDR_UNIT, installedHerdr, canonical.herdr));
+    }
+  }
+  if (problems.length === 0) {
+    return passFinding("systemd-unit", "installed units match the staged render");
+  }
+  return failFinding(
+    "systemd-unit",
+    [...problems, INCIDENTS.systemd].join("; "),
+    "re-stage and reinstall with `omp-conductor setup host` from the fleet account",
+  );
+}
+
+function installedUnitUser(probes: Probes): string | undefined {
+  const unit = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME));
+  if (unit === undefined) return undefined;
+  const match = /^User=(.*)$/m.exec(unit);
+  if (match === null) return undefined;
+  const value = match[1]?.trim().replace(/^"(.*)"$/, "$1") ?? "";
+  return value === "" ? undefined : value;
+}
+
+/**
+ * Ownership and modes of the unit file and the runtime dirs: a daemon that
+ * cannot write its own state is the empty-agent.db class, and a unit someone
+ * else can edit is a takeover.
+ */
+function ownershipProbe(probes: Probes): Finding {
+  if (!probes.hasSystemd()) return passFinding("systemd-ownership", "no systemd directory on this host — nothing to check");
+  const failures: string[] = [];
+  const warns: string[] = [];
+  const daemonPath = join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME);
+  const daemonStat = probes.stat(daemonPath);
+  if (daemonStat !== undefined) {
+    if (daemonStat.uid !== 0) failures.push(`${STAGED_SERVICE_NAME} is not owned by root`);
+    if ((daemonStat.mode & 0o022) !== 0) warns.push(`${STAGED_SERVICE_NAME} is group/other-writable`);
+  }
+  const unitUser = installedUnitUser(probes);
+  const fleetUid = unitUser === undefined ? process.getuid?.() : probes.uidOf(unitUser);
+  for (const [label, dir] of [["state dir", stateDir()], ["telegram state dir", telegramStateDir()]] as const) {
+    const st = probes.stat(dir);
+    if (st === undefined) continue;
+    if (fleetUid !== undefined && st.uid !== fleetUid) {
+      failures.push(
+        `${label} (${dir}) is owned by uid ${st.uid}, but the ${unitUser === undefined ? "current user" : `unit's account "${unitUser}"`} has uid ${fleetUid}`,
+      );
+    }
+    if ((st.mode & 0o022) !== 0) warns.push(`${label} (${dir}) is group/other-writable`);
+  }
+  if (failures.length === 0 && warns.length === 0) {
+    return passFinding("systemd-ownership", "unit file and runtime dirs are owned and permitted as deployed");
+  }
+  const summary = [...failures, ...warns].join("; ");
+  const fix = "reinstall the unit as root and chown the state dirs to the account the unit runs as";
+  if (failures.length === 0) {
+    return warnFinding("systemd-ownership", summary, fix);
+  }
+  return failFinding("systemd-ownership", summary, fix);
+}
+
+/** IANA timezone in reporting config — an invalid zone silently mis-schedules
+ *  the availability window and the daily digest (#273). */
+function timezoneProbe(project: ProjectConfig | undefined): Finding {
+  if (project === undefined) return passFinding("reporting-timezone", "no project resolved — nothing to check");
+  const tzs: [string, string | undefined][] = [
+    ["reporting.availability.timezone", project.reporting?.availability?.timezone],
+    ["reporting.digest.timezone", project.reporting?.digest?.timezone],
+  ];
+  const configured = tzs.filter(([, tz]) => tz !== undefined);
+  const bad = configured.filter(([, tz]) => tz !== undefined && !isKnownTimezone(tz));
+  if (bad.length === 0) {
+    return passFinding(
+      "reporting-timezone",
+      configured.length === 0
+        ? "no reporting timezone configured"
+        : `reporting timezone(s) valid: ${configured.map(([, tz]) => `"${tz}"`).join(", ")}`,
+    );
+  }
+  return failFinding(
+    "reporting-timezone",
+    `${bad.map(([label, tz]) => `${label}: "${tz}" is not a known IANA timezone`).join("; ")} — an invalid zone silently skips the availability window and the daily digest`,
+    "set a valid IANA timezone (e.g. Europe/London) in reporting.availability or reporting.digest",
+  );
+}
+
+/** The same acceptance the config validator uses (`Intl.DateTimeFormat`), so
+ *  `doctor` and `config.ts` cannot disagree about a zone. */
+export function isKnownTimezone(tz: string): boolean {
+  try {
+    new Intl.DateTimeFormat("en-GB", { timeZone: tz });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** The self-identified probe message `--probe-telegram` delivers through the
+ *  same report transport a real report would take. */
+function probeMessage(project: string, checkedAt: string): string {
+  return [
+    "omp-conductor doctor · Telegram delivery probe",
+    `project: ${project} · started ${checkedAt}`,
+    "Sent by `omp-conductor doctor --probe-telegram` to verify delivery end to end.",
+    "No action needed — you can ignore this message.",
+  ].join("\n");
+}
+
+/**
+ * Telegram delivery health — and, when `--probe-telegram` is set, one
+ * self-identified message through the report transport. A dead bot token is
+ * silent: reports leave the outbox and never land.
+ */
+async function telegramProbe(probes: Probes, project: ProjectConfig | undefined, checkedAt: number): Promise<Finding> {
+  if (project === undefined) return passFinding("telegram", "no project resolved — nothing to check");
+  const chatId = (project.escalation.telegramChatId ?? "").trim();
+  if (chatId === "") {
+    return passFinding("telegram", "no escalation.telegramChatId configured — nothing to probe");
+  }
+  const health = await probes.telegramHealth(project.name);
+  const failures: string[] = [];
+  const notes: string[] = [];
+  if (health.kind === "down") failures.push(`bot health: down (${health.detail ?? "getMe failed"})`);
+  else if (health.kind === "unconfigured") notes.push(`bot health: unconfigured (${health.detail ?? "no token"})`);
+  else if (health.kind === "degraded") notes.push(`bot health: degraded (${health.detail ?? ""})`);
+  else notes.push(`bot health: ok (${health.detail ?? ""})`);
+  if (probes.probeTelegram) {
+    const send = probes.telegramSend?.(project);
+    if (send === undefined) {
+      failures.push("probe requested but no send transport is wired");
+    } else {
+      try {
+        await send(probeMessage(project.name, new Date(checkedAt).toISOString()));
+        notes.push("probe message delivered");
+      } catch (err) {
+        failures.push(`probe send failed: ${messageOf(err)}`);
+      }
+    }
+  }
+  if (failures.length === 0) {
+    return passFinding("telegram", notes.join("; "));
+  }
+  return failFinding(
+    "telegram",
+    failures.join("; "),
+    "fix the bot token / omp-telegram install; `doctor --probe-telegram` proves delivery end to end",
+  );
+}
+
+/** Spend telemetry: warn when the last K completed runs all recorded $0.00 —
+ *  the USD cap cannot fire on zeros ($0.00 is not proof of no spend). */
+function spendProbe(rows: RunSpendRow[], limit: number): Finding {
+  if (rows.length < limit) {
+    return passFinding("spend-telemetry", `${rows.length} completed run(s) observed — fewer than ${limit}, nothing to judge yet`);
+  }
+  const window = rows.slice(0, limit);
+  if (window.every((r) => r.spendUsd === 0)) {
+    return warnFinding(
+      "spend-telemetry",
+      `the last ${limit} completed runs all recorded $0.00 spend — ${INCIDENTS.spend}`,
+      "verify harness spend reporting (the per-run spendUsd column / `omp usage --json`); a USD cap on top of zeros never fires",
+    );
+  }
+  const total = window.reduce((sum, r) => sum + r.spendUsd, 0);
+  return passFinding("spend-telemetry", `spend observed on the last ${window.length} completed runs ($${total.toFixed(2)} total)`);
+}
+
+// ------------------------------------------------------------- composition
+
+/**
+ * Run every probe and assemble the stable report.
+ *
+ * @param projectName the `--project NAME` value (undefined on single-project hosts)
+ * @param opts injected seams; every probe defaults to the production wiring
+ */
+export async function runDoctor(projectName: string | undefined, opts: DoctorDeps = {}): Promise<DoctorReport> {
+  const probes: Probes = { ...defaultProbes(), ...opts };
+  const checkedAt = probes.now();
+  const checkedAtIso = new Date(checkedAt).toISOString();
+
+  let cfg: ConductorConfig | undefined;
+  let configProblem: string | undefined;
+  try {
+    cfg = probes.loadConfig();
+  } catch (err) {
+    configProblem = messageOf(err);
+  }
+
+  let project: ProjectConfig | undefined;
+  let projectProblem: string | undefined;
+  if (cfg !== undefined) {
+    try {
+      project = findProject(cfg, projectName);
+    } catch (err) {
+      projectProblem = messageOf(err);
+    }
+  }
+
+  const findings: Finding[] = [];
+  findings.push(configProbe(configProblem));
+  findings.push(
+    cfg === undefined
+      ? passFinding("project", "config unreadable — no project to resolve")
+      : project === undefined
+        ? failFinding(
+            "project",
+            `no project resolved: ${projectProblem ?? "unknown error"}`,
+            "pass --project NAME (doctor checks one project per run)",
+          )
+        : passFinding("project", project.name),
+  );
+  findings.push(backupProbe(probes));
+  findings.push(dbProbe(probes));
+  findings.push(await ghAuthProbe(probes, configuredRepos(cfg)));
+  findings.push(
+    project === undefined
+      ? passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`)
+      : await labelProbe(probes, project),
+  );
+  findings.push(unitProbe(probes, project, cfg));
+  findings.push(ownershipProbe(probes));
+  findings.push(timezoneProbe(project));
+  findings.push(await telegramProbe(probes, project, checkedAt));
+  findings.push(spendProbe(project === undefined ? [] : probes.recentRuns(project.name, SPEND_SAMPLE_RUNS), SPEND_SAMPLE_RUNS));
+
+  const status: ReportStatus = findings.some((f) => f.status === "fail")
+    ? "fail"
+    : findings.some((f) => f.status === "warn")
+      ? "warn"
+      : "pass";
+  return {
+    project: project === undefined ? projectName ?? "(none resolved)" : project.name,
+    checkedAt: checkedAtIso,
+    status,
+    findings,
+  };
+}
+
+/** The default wiring — every production transport the rest of the package uses. */
+export function defaultProbes(): Probes {
+  return {
+    loadConfig,
+    scopes: checkTokenScopes,
+    rateLimitOk: async () => (await fetchRateLimit()) !== undefined,
+    repoReadable: defaultRepoReadable,
+    repoLabels: defaultRepoLabels,
+    hasSystemd: () => existsSync(SYSTEMD_UNIT_DIR),
+    readUnit: defaultReadUnit,
+    stat: defaultStat,
+    uidOf: defaultUidOf,
+    dbIntegrity: defaultDbIntegrity,
+    recentRuns: defaultRecentRuns,
+    telegramHealth: (projectName) => probeTelegramHealth(projectName),
+    telegramSend: telegramReportSend,
+    canonicalUnits: defaultCanonicalUnits,
+    now: Date.now,
+    probeTelegram: false,
+  };
+}