omp-conductor 0.13.0 → 0.15.0

package/src/privileged.ts

@@ -0,0 +1,264 @@
+/**
+ * The one place conductor runs a command as root.
+ *
+ * Every host-provisioning step used to be *printed* as a `sudo …` line for the
+ * operator to retype, which is not caution — it is an audit trail nobody keeps
+ * and a transcription error nobody catches. This module keeps the audit and
+ * removes the retyping: the caller composes structured steps, this renders all
+ * of them with their exact argv, takes **one** confirm, and then runs them.
+ *
+ * Three invariants, each of which a convenient shortcut would break:
+ *
+ *   - **The CLI never escalates itself.** Only the individual steps below are
+ *     privileged. `sudo omp-conductor setup host` would resolve the config,
+ *     `$OMP_CONDUCTOR_HOME`, `homedir()` and the unit's own `User=` as root and
+ *     bake the wrong account into a unit for a fleet that runs as somebody
+ *     else — a timer that goes green while writing state no worker can read.
+ *     {@link assertNoSelfExec} refuses to build such a step at all.
+ *   - **One confirm, everything named.** A confirm that authorises commands it
+ *     did not print is not consent, so the render happens first and lists the
+ *     exact argv — including which steps deliberately run *without* `sudo`.
+ *   - **A failure hands the work back.** The first non-`bestEffort` failure
+ *     stops the batch and prints the failed title, its stderr, and every
+ *     not-yet-run command verbatim, so an operator can finish by hand from the
+ *     terminal they are already looking at.
+ */
+
+import type { WizardUi } from "./wizard-ui.ts";
+
+export interface PrivilegedStep {
+  /** One line naming what this step accomplishes, shown in the plan. */
+  title: string;
+  /** The exact argv. No shell: no quoting rules, no word splitting, no `&&`. */
+  argv: readonly string[];
+  /**
+   * A failure here is reported and the batch continues. For steps whose failure
+   * is not a broken install — reloading a systemd that was never running, say.
+   */
+  bestEffort?: boolean;
+  /**
+   * Runs as the invoking account with **no** `sudo` prefix, while still
+   * appearing in the same plan under the same confirm.
+   *
+   * `setup graph`'s index-only clones are the case this exists for: a
+   * root-owned clone under the fleet user's cache is precisely the failure the
+   * escalation guard exists to prevent, but the operator must still see the
+   * clone and the timer install as one plan they approve once.
+   */
+  unprivileged?: boolean;
+}
+
+/** One spawn the runner needs. Injectable so tests never touch a real `sudo`. */
+export interface PrivilegedSpawn {
+  argv: readonly string[];
+  /**
+   * `sudo -v` needs the terminal for its password prompt, so its stdio is
+   * inherited whole and nothing is captured.
+   */
+  interactive: boolean;
+}
+
+export interface PrivilegedResult {
+  exitCode: number;
+  /** Captured stderr, or `""` for an interactive spawn that owned the terminal. */
+  stderr: string;
+}
+
+export interface PrivilegedDeps {
+  spawn(request: PrivilegedSpawn): Promise<PrivilegedResult>;
+  /** `undefined` on a platform without uids, which is never privileged here. */
+  getuid(): number | undefined;
+}
+
+export type PrivilegedOutcome =
+  /** The confirm was answered no, or dismissed. Nothing ran. */
+  | { kind: "declined" }
+  /** Every step ran; `bestEffort` failures are listed but did not stop the batch. */
+  | { kind: "completed"; softFailures: { step: PrivilegedStep; stderr: string }[] }
+  /** A required step failed. `remaining` are the steps that never ran. */
+  | {
+      kind: "failed";
+      step: PrivilegedStep;
+      exitCode: number;
+      stderr: string;
+      remaining: readonly PrivilegedStep[];
+    };
+
+export interface RunPrivilegedOptions {
+  deps?: PrivilegedDeps;
+  /** Confirm title. Defaults to a generic one; callers name their verb. */
+  title?: string;
+  /** Extra lines shown above the step list — what this batch is for. */
+  preamble?: readonly string[];
+}
+
+/**
+ * Shell-quoted rendering of one step, exactly as an operator would type it.
+ *
+ * The steps are argv precisely so nothing here is load-bearing at execution
+ * time; this is for the plan, the failure remainder and the print-only modes.
+ */
+export function formatStep(step: PrivilegedStep, asRoot = false): string {
+  const prefix = step.unprivileged || asRoot ? [] : ["sudo"];
+  return [...prefix, ...step.argv].map(shellQuote).join(" ");
+}
+
+function shellQuote(value: string): string {
+  return /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
+}
+
+/**
+ * Refuses to build a step that re-runs conductor itself.
+ *
+ * Called on every step before anything is rendered, because the failure it
+ * prevents is invisible: `sudo omp-conductor …` exits 0 and leaves a unit
+ * naming the wrong account. Cheap, and it turns a future refactor's mistake
+ * into a test failure rather than a broken fleet.
+ */
+function assertNoSelfExec(steps: readonly PrivilegedStep[]): void {
+  for (const step of steps) {
+    const offender = step.argv.find((a) => {
+      const base = a.split("/").pop() ?? a;
+      return base === "omp-conductor" || base === "cli.ts";
+    });
+    if (offender !== undefined) {
+      throw new Error(
+        `privileged step "${step.title}" would re-exec conductor (${offender}) — ` +
+          "only individual structured steps are ever privileged",
+      );
+    }
+  }
+}
+
+async function spawnReal(request: PrivilegedSpawn): Promise<PrivilegedResult> {
+  const proc = Bun.spawn(request.argv as string[], {
+    // stdin is inherited even for captured steps: a `sudo` whose timestamp
+    // expired mid-batch must be able to prompt rather than fail on a closed fd.
+    stdin: "inherit",
+    // stdout is inherited so a long step (`systemctl start` on a oneshot that
+    // indexes for minutes) is visible while it runs instead of after it.
+    stdout: "inherit",
+    stderr: request.interactive ? "inherit" : "pipe",
+  });
+  const [exitCode, stderr] = await Promise.all([
+    proc.exited,
+    request.interactive ? Promise.resolve("") : new Response(proc.stderr).text(),
+  ]);
+  return { exitCode, stderr };
+}
+
+export const DEFAULT_PRIVILEGED_DEPS: PrivilegedDeps = {
+  spawn: spawnReal,
+  getuid: () => process.getuid?.(),
+};
+
+/**
+ * Renders `steps`, takes one confirm, then runs them in order.
+ *
+ * As root, every step runs directly: there is nothing to escalate, and a `sudo`
+ * prefix on a root-run fleet is an extra dependency for no gain. Otherwise one
+ * `sudo -v` primes the credential with the terminal it needs, and each
+ * privileged step then runs as `sudo -- <argv>` — `--` so a step whose first
+ * argument begins with a dash cannot be read as a `sudo` flag.
+ */
+export async function runPrivileged(
+  steps: readonly PrivilegedStep[],
+  ui: WizardUi,
+  options: RunPrivilegedOptions = {},
+): Promise<PrivilegedOutcome> {
+  assertNoSelfExec(steps);
+  const deps = options.deps ?? DEFAULT_PRIVILEGED_DEPS;
+  const asRoot = deps.getuid() === 0;
+  const needsSudo = !asRoot && steps.some((s) => s.unprivileged !== true);
+
+  ui.notify(
+    [
+      ...(options.preamble ?? []),
+      ...(options.preamble === undefined || options.preamble.length === 0 ? [] : [""]),
+      ...steps.flatMap((step, index) => [
+        `${index + 1}. ${step.title}${step.unprivileged === true && !asRoot ? "  (as you, not root)" : ""}`,
+        `   ${formatStep(step, asRoot)}`,
+      ]),
+      "",
+      asRoot
+        ? "Running as root — these run directly, with no sudo."
+        : needsSudo
+          ? "sudo will ask for your password once, before the first step."
+          : "None of these need root.",
+      "Nothing has been run yet.",
+    ].join("\n"),
+    "info",
+  );
+
+  const go = await ui.confirm(
+    options.title ?? "Run these steps now?",
+    `${steps.length} step(s), in the order shown. Anything that fails stops the rest and prints what is left.`,
+  );
+  // `undefined` is a dismissal rather than a "no", but for a batch that has not
+  // started they mean the same thing: run nothing.
+  if (go !== true) return { kind: "declined" };
+
+  if (needsSudo) {
+    const primed = await deps.spawn({ argv: ["sudo", "-v"], interactive: true });
+    if (primed.exitCode !== 0) {
+      return {
+        kind: "failed",
+        step: { title: "authenticate with sudo", argv: ["sudo", "-v"] },
+        exitCode: primed.exitCode,
+        stderr: primed.stderr,
+        remaining: steps,
+      };
+    }
+  }
+
+  const softFailures: { step: PrivilegedStep; stderr: string }[] = [];
+  for (const [index, step] of steps.entries()) {
+    ui.notify(`[${index + 1}/${steps.length}] ${step.title}`, "info");
+    const argv =
+      step.unprivileged === true || asRoot ? step.argv : ["sudo", "--", ...step.argv];
+    const result = await deps.spawn({ argv, interactive: false });
+    if (result.exitCode === 0) continue;
+    if (step.bestEffort === true) {
+      softFailures.push({ step, stderr: result.stderr });
+      ui.notify(`  continued past a best-effort failure: ${firstLine(result.stderr)}`, "warning");
+      continue;
+    }
+    const remaining = steps.slice(index + 1);
+    ui.notify(formatFailure(step, result, remaining, asRoot), "error");
+    return { kind: "failed", step, exitCode: result.exitCode, stderr: result.stderr, remaining };
+  }
+  return { kind: "completed", softFailures };
+}
+
+function firstLine(stderr: string): string {
+  const line = stderr.trim().split("\n")[0];
+  return line === undefined || line === "" ? "no stderr" : line;
+}
+
+/**
+ * What the operator is left holding when a step fails: the step, why, and the
+ * rest of the batch verbatim so finishing by hand is a paste rather than a
+ * reconstruction.
+ */
+export function formatFailure(
+  step: PrivilegedStep,
+  result: PrivilegedResult,
+  remaining: readonly PrivilegedStep[],
+  asRoot = false,
+): string {
+  return [
+    `Stopped: ${step.title} exited ${result.exitCode}.`,
+    `  ${formatStep(step, asRoot)}`,
+    ...(result.stderr.trim() === ""
+      ? ["  (no stderr — its output is above)"]
+      : result.stderr.trimEnd().split("\n").map((l) => `  ${l}`)),
+    ...(remaining.length === 0
+      ? ["", "Nothing was left to run."]
+      : [
+          "",
+          `${remaining.length} step(s) did NOT run. To finish by hand:`,
+          "",
+          ...remaining.map((s) => `  ${formatStep(s, asRoot)}`),
+        ]),
+  ].join("\n");
+}

package/src/reports.ts

@@ -36,9 +36,21 @@
  * conclusion leaves something for the next pass to pick up.
  */
 
+import { availabilityDisposition, interruptDisposition } from "./availability.ts";
 import { readTelegramToken, sendTelegram, TelegramSendError } from "./escalate.ts";
 import { localDayKey } from "./digest-schedule.ts";
-import type { Escalation, ProjectConfig, ReportRecord, Store } from "./types.ts";
+import {
+  DIGEST_BACKLOG_LIMIT,
+  INTERRUPT_CATEGORIES,
+  type DigestBacklog,
+  type Escalation,
+  type HeldNotice,
+  type InterruptCategory,
+  type ProjectConfig,
+  type ReportEnqueue,
+  type ReportRecord,
+  type Store,
+} from "./types.ts";
 
 /**
  * Attempts before a report is written off. Six attempts across the backoff
@@ -85,6 +97,8 @@ export type ReportSend = (text: string) => Promise<number | undefined>;
 /** What one pass did, by report id. Returned for the daemon's log and the tests. */
 export interface ReportDeliveryPass {
   delivered: string[];
+  /** Material reports preserved without sending after policy or availability changed. */
+  deferred: string[];
   /** Known-failed, back in `pending` behind a backoff. */
   requeued: string[];
   /** The attempt ended without an answer. Left `sending` and flagged as a
@@ -108,7 +122,8 @@ export interface ReportOutbox {
 }
 
 export interface ReportOutboxDeps {
-  project: ProjectConfig;
+  /** A provider lets a resident daemon apply config edits at the next tick. */
+  project: ProjectConfig | (() => ProjectConfig);
   store: Store;
   /** A report nobody can deliver escalates through this. Optional so a unit
    *  test can exercise delivery without wiring an escalator. */
@@ -116,6 +131,8 @@ export interface ReportOutboxDeps {
   send?: ReportSend;
   now?: () => number;
   log?: (msg: string) => void;
+  /** False while the resident daemon cannot validate live delivery policy. */
+  deliveryAllowed?: () => boolean;
 }
 
 /**
@@ -200,6 +217,20 @@ export function formatOpenReports(
   return lines;
 }
 
+/** Durable rows still owed to a future digest. Always rendered in `status`: a
+ * quiet line proves the accumulator is empty, while a non-zero line makes loss
+ * or backlog visible without asking the session what it remembers. */
+export function formatDigestBacklog(backlog: DigestBacklog, now: number = Date.now()): string[] {
+  const age = (at: number | undefined): string =>
+    at === undefined ? "" : ` (oldest ${humanAge(Math.max(0, now - at))})`;
+  return [
+    "",
+    "digest backlog",
+    `  material events    ${backlog.materialCount}${age(backlog.materialOldestAt)}`,
+    `  held escalations   ${backlog.heldNoticeCount}${age(backlog.heldNoticeOldestAt)}`,
+  ];
+}
+
 function openReportDetail(r: ReportRecord, now: number): string {
   const flat = r.lastError?.replace(/\s+/g, " ").trim() ?? "";
   const error =
@@ -253,14 +284,119 @@ export function telegramReportSend(p: ProjectConfig): ReportSend {
         "definitive",
       );
     }
-    return await sendTelegram(token, chatId, text);
+    return await sendTelegram(token, chatId, text, { topicId: p.escalation.telegramTopicId });
+  };
+}
+
+/** Keep the mechanical catch-up comfortably inside Telegram's report wrapper. */
+const AVAILABILITY_REPORT_BODY_LIMIT = 3_200;
+const AVAILABILITY_REPORT_KEY_PREFIX = "availability/";
+
+interface AvailabilityReportMarker {
+  categories: InterruptCategory[];
+  urgent: boolean;
+}
+
+function availabilityReportMarker(report: ReportRecord): AvailabilityReportMarker | undefined {
+  const key = report.dedupeKey;
+  if (key === undefined || !key.startsWith(AVAILABILITY_REPORT_KEY_PREFIX)) return undefined;
+  const parts = key.slice(AVAILABILITY_REPORT_KEY_PREFIX.length).split("/");
+  const urgent = parts[0] === "urgent";
+  const encoded = (urgent ? parts[1] : parts[0]) ?? "";
+  return {
+    urgent,
+    categories: encoded
+      .split(",")
+      .filter((category): category is InterruptCategory =>
+        INTERRUPT_CATEGORIES.includes(category as InterruptCategory),
+      ),
+  };
+}
+
+function availabilityNoticeLine(notice: HeldNotice): string {
+  const flat = (text: string, limit: number): string => {
+    const value = text.replace(/\s+/g, " ").trim();
+    return value.length <= limit ? value : `${value.slice(0, limit - 1)}…`;
   };
+  return (
+    `- [${notice.category}] ${flat(notice.summary, 180)} ` +
+    `(${new Date(notice.createdAt).toISOString()}) — ${flat(notice.detail, 240)}`
+  );
+}
+
+/**
+ * Hand availability-held notices to the existing durable outbox when the live
+ * policy permits them again. Association and report creation are one SQLite
+ * transaction, so daemon downtime or a send failure cannot lose a notice.
+ */
+export function enqueueAvailableHeldNotices(
+  project: ProjectConfig,
+  store: Store,
+  now: number,
+): ReportEnqueue | undefined {
+  // The Store arbitrates the due-digest lease and this catch-up in the same
+  // SQLite transaction. A preflight here would reopen the snapshot race.
+  const categories = INTERRUPT_CATEGORIES.filter(
+    (category) => interruptDisposition(project.reporting, category, now) === "interrupt",
+  );
+  const urgentCategories = INTERRUPT_CATEGORIES.filter(
+    (category) =>
+      availabilityDisposition(project.reporting?.availability, category, now) === "interrupt",
+  );
+  const urgent = store.undigestedNotices(
+    project.name,
+    DIGEST_BACKLOG_LIMIT,
+    true,
+    urgentCategories,
+    true,
+  );
+  const eligible =
+    urgent.length > 0
+      ? urgent
+      : store.undigestedNotices(
+          project.name,
+          DIGEST_BACKLOG_LIMIT,
+          true,
+          categories,
+          false,
+        );
+  if (eligible.length === 0) return undefined;
+
+  const lines = [
+    "Working-hours catch-up",
+    "These interruptions were held durably while the operator was outside the configured availability window:",
+  ];
+  const selected: HeldNotice[] = [];
+  for (const notice of eligible) {
+    const line = availabilityNoticeLine(notice);
+    const next = [...lines, line].join("\n");
+    if (selected.length > 0 && next.length > AVAILABILITY_REPORT_BODY_LIMIT) break;
+    lines.push(line);
+    selected.push(notice);
+  }
+
+  const selectedCategories = INTERRUPT_CATEGORIES.filter((category) =>
+    selected.some((notice) => notice.category === category),
+  );
+  const urgentMarker = selected[0]?.urgent === true ? "urgent/" : "";
+  return store.enqueueAvailabilityReport(
+    {
+      project: project.name,
+      kind: "digest",
+      body: lines.join("\n"),
+      dedupeKey: `${AVAILABILITY_REPORT_KEY_PREFIX}${urgentMarker}${selectedCategories.join(",")}/${selected[0]!.id}`,
+      at: now,
+    },
+    selected.map((notice) => notice.id),
+  );
 }
 
 export function createReportOutbox(deps: ReportOutboxDeps): ReportOutbox {
-  const { project, store } = deps;
+  const { store } = deps;
+  const currentProject =
+    typeof deps.project === "function" ? deps.project : (): ProjectConfig => deps.project as ProjectConfig;
   const now = deps.now ?? Date.now;
-  const send = deps.send ?? telegramReportSend(project);
+  const injectedSend = deps.send;
   const log = deps.log ?? ((): void => {});
 
   /**
@@ -279,6 +415,7 @@ export function createReportOutbox(deps: ReportOutboxDeps): ReportOutbox {
    * undeliverable report pages exactly once and never every five minutes.
    */
   const pageUndeliverable = async (r: ReportRecord, error: string): Promise<void> => {
+    const project = currentProject();
     if (deps.escalate === undefined) return;
     try {
       await deps.escalate({
@@ -318,6 +455,64 @@ export function createReportOutbox(deps: ReportOutboxDeps): ReportOutbox {
   };
 
   const attempt = async (r: ReportRecord, pass: ReportDeliveryPass): Promise<void> => {
+    const project = currentProject();
+    if (deps.deliveryAllowed?.() === false) {
+      pass.deferred.push(r.id);
+      return;
+    }
+    const availabilityMarker = availabilityReportMarker(r);
+    if (availabilityMarker !== undefined) {
+      if (availabilityMarker.categories.length === 0) {
+        pass.deferred.push(r.id);
+        log(`availability catch-up ${r.id} has no valid category marker and was held fail-closed`);
+        return;
+      }
+      const dispositions = availabilityMarker.categories.map((category) =>
+        availabilityMarker.urgent
+          ? availabilityDisposition(project.reporting?.availability, category, now())
+          : interruptDisposition(project.reporting, category, now()),
+      );
+      if (dispositions.some((disposition) => disposition !== "interrupt")) {
+        if (store.releasePendingAvailabilityReport(r.id, project.name, r.ambiguous)) {
+          const reason = dispositions.includes("digest")
+            ? "reporting policy changed"
+            : "the availability window closed";
+          log(`availability catch-up ${r.id} returned to the digest because ${reason}`);
+        }
+        pass.deferred.push(r.id);
+        return;
+      }
+    }
+    if (r.kind === "material") {
+      const at = now();
+      const disposition = interruptDisposition(project.reporting, "material", at);
+      if (disposition !== "interrupt") {
+        const firstLine = r.body.split("\n", 1)[0]!;
+        const deferred = store.deferPendingReportToNotice(r.id, {
+          id: r.id,
+          project: project.name,
+          category: "material",
+          summary: (
+            r.ambiguous ? `POSSIBLE REPEAT of report ${r.id}: ${firstLine}` : firstLine
+          ).slice(0, 240),
+          detail: r.ambiguous
+            ? `This report may already have reached Telegram before its outcome was lost.\n\n${r.body}`
+            : r.body,
+          createdAt: at,
+          ...(disposition === "availability" ? { releaseOnAvailable: true } : {}),
+        });
+        if (deferred) {
+          pass.deferred.push(r.id);
+          log(
+            `report ${r.id} preserved for ${
+              disposition === "availability" ? "the next availability window" : "a digest"
+            } after reporting policy changed`,
+          );
+          return;
+        }
+      }
+    }
+    const send = injectedSend ?? telegramReportSend(project);
     const attemptId = crypto.randomUUID();
     // Losing this race is ordinary: another pass, or another daemon, already
     // owns the attempt. Returning is what keeps one report from being in flight
@@ -400,12 +595,14 @@ export function createReportOutbox(deps: ReportOutboxDeps): ReportOutbox {
 
   return {
     recover(staleAt: number): ReportRecord[] {
-      return store.recoverSendingReports(project.name, staleAt, now());
+      return store.recoverSendingReports(currentProject().name, staleAt, now());
     },
 
     async deliverDue(): Promise<ReportDeliveryPass> {
+      const project = currentProject();
       const pass: ReportDeliveryPass = {
         delivered: [],
+        deferred: [],
         requeued: [],
         uncertain: [],
         failed: [],

package/src/session-host.ts

@@ -49,6 +49,8 @@ export interface SessionHostSpec {
    * Absent, the verbs are registered and every one of them fails closed.
    */
   verbSocketPath?: string;
+  /** Deny every tool but reading and searching (#307) — the setup probes. */
+  readOnly?: boolean;
 }
 
 /** Parent → child. */
@@ -184,6 +186,7 @@ export async function runSessionHost(
       ...(spec.resume === undefined ? {} : { resume: spec.resume }),
       ...(spec.releaseGrants === undefined ? {} : { releaseGrants: spec.releaseGrants }),
       ...(spec.verbSocketPath === undefined ? {} : { verbSocketPath: spec.verbSocketPath }),
+      ...(spec.readOnly === undefined ? {} : { readOnly: spec.readOnly }),
       // The release audit lives in the daemon's state directory, which this
       // process may not be able to write and must not be trusted to. It
       // becomes a message; the parent performs the durable write.