omp-conductor 0.16.0 → 0.16.2

package/src/setup-wizard.ts

@@ -2005,7 +2005,7 @@ export async function setup(
   areaArg?: AmendAreaId,
   probes: SetupProbes = DEFAULT_PROBES,
   apply: SetupApplyDeps = DEFAULT_APPLY,
-): Promise<void> {
+): Promise<boolean> {
   const path = configPath();
   // A config that exists but does not parse is a fault to report, never
   // something to quietly replace: overwriting it would delete every project it
@@ -2021,7 +2021,7 @@ export async function setup(
   } catch (err) {
     if (!(err instanceof Cancelled)) throw err;
     ui.notify("Setup cancelled — nothing was changed.", "info");
-    return;
+    return false;
   }
   // `answers` is mutable below: the review loop re-asks one area at a time and
   // replaces only the fields that area owns, so everything else carries across.
@@ -2035,7 +2035,7 @@ export async function setup(
         `Run \`gh auth refresh -s repo,project\`, then run setup again.`,
       "error",
     );
-    return;
+    return false;
   }
 
   // Drafted here — after the interview, before the plan — because the plan's single
@@ -2183,7 +2183,7 @@ export async function setup(
       .join(" ");
 
   let plan = await derive();
-  if (plan === undefined) return;
+  if (plan === undefined) return false;
   ui.notify(planBlock(plan), "info");
   // -------- the consent gate (#417): a review loop, not a yes/no --------
   //
@@ -2206,12 +2206,12 @@ export async function setup(
     );
     if (choice === undefined) {
       ui.notify("Setup cancelled — nothing was changed, and the answers were discarded.", "info");
-      return;
+      return false;
     }
     if (choice === REVIEW_APPLY) break;
     if (choice === REVIEW_EXIT) {
       ui.notify("Setup exited without applying — the answers were discarded and nothing was changed.", "info");
-      return;
+      return false;
     }
     if (choice === REVIEW_REVIEW) {
       ui.notify(planBlock(plan), "info");
@@ -2259,7 +2259,7 @@ export async function setup(
       proseDrafted = true;
     }
     const redrafted = await derive();
-    if (redrafted === undefined) return;
+    if (redrafted === undefined) return false;
     plan = redrafted;
     ui.notify(`Edited "${area.label}" — the plan below is recomputed from the new answers.`, "info");
     ui.notify(planBlock(plan), "info");
@@ -2302,7 +2302,7 @@ export async function setup(
       // enables the timer, which does not touch the running daemon.
       await offerCodeGraph(plan.project, ui, apply.graphInstall);
       ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
-      return;
+      return false;
     }
     // No live workers: still prefer an explicit reload when this run *added* a
     // neighbour — auto-restart would bounce every other project's heartbeat for
@@ -2342,7 +2342,7 @@ export async function setup(
         "warning",
       );
       ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
-      return;
+      return false;
     }
   }
   apply.resume(plan.project.name);
@@ -2409,4 +2409,5 @@ export async function setup(
   // to create/verify the herdr pane. Added projects especially — the CLI wrote
   // tick + config but cannot start a herdr agent (#319).
   ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
+  return true;
 }

package/src/transcript.ts

@@ -1,5 +1,5 @@
 /** Read one property off an unvalidated transcript entry. */
-function prop(source: unknown, key: string): unknown {
+export function prop(source: unknown, key: string): unknown {
   if (source === null || typeof source !== "object") return undefined;
   return Reflect.get(source, key);
 }

package/src/upgrade-verify.ts

@@ -382,6 +382,19 @@ function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequ
   }
 }
 
+/**
+ * Doctor findings that only a privileged `setup host` can clear — the systemd
+ * units (and the recovery pair they name), and the herdr pane-shell key. A
+ * release that re-renders any of those templates fails them on every host
+ * whose `setup host` has not been re-run, so they are not rollback-worthy
+ * regression evidence: the upgrade is deliberately unable to perform the
+ * privileged install, and rolling back over drift it cannot clear would
+ * strand the fleet on the previous release with the same outstanding `setup
+ * host` (#598). The verifier escalates them instead — the doctor check's
+ * detail and the verify report name the drift and its fix.
+ */
+const PRIVILEGED_INSTALL_DRIFT = new Set(["systemd-unit", "systemd-recovery", "herdr-resume"]);
+
 /**
  * The independent checks the returning process runs. Each compares a live
  * fact with the journal's record of the world the install began in:
@@ -392,7 +405,9 @@ function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequ
  *    same predicate the in-process upgrade used, so a fleet that was armed
  *    stays armed and a live pane stays live;
  *  - `health` — the daemon that came back answers `/healthz`;
- *  - `doctor` — no failing finding.
+ *  - `doctor` — no failing finding beyond the privileged-install drift
+ *    ({@link PRIVILEGED_INSTALL_DRIFT}) only `setup host` can clear, which is
+ *    escalated rather than rolled back.
  */
 export async function runUpgradeChecks(
   deps: UpgradeVerifyDeps,
@@ -423,10 +438,17 @@ export async function runUpgradeChecks(
 
   const doctor = await deps.doctor(deps.projectName);
   const failing = doctor.findings.filter((finding) => finding.status === "fail");
+  const drift = failing.filter((finding) => PRIVILEGED_INSTALL_DRIFT.has(finding.id));
+  const regression = failing.filter((finding) => !PRIVILEGED_INSTALL_DRIFT.has(finding.id));
   checks.push({
     name: "doctor",
-    ok: failing.length === 0,
-    detail: failing.length === 0 ? `status ${doctor.status}` : `${failing[0]!.id}: ${failing[0]!.summary}`,
+    ok: regression.length === 0,
+    detail:
+      regression.length > 0
+        ? `${regression[0]!.id}: ${regression[0]!.summary}`
+        : drift.length > 0
+          ? `status ${doctor.status}; ${drift.map((f) => f.id).join(", ")} — privileged-install drift, run \`omp-conductor setup host\` from the fleet account`
+          : `status ${doctor.status}`,
   });
 }
 

package/src/upgrade.ts

@@ -14,9 +14,9 @@ import { livingDaemon, restartDaemon } from "./lifecycle.ts";
 import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
-  STAGED_SERVICE_NAME,
   planHostRuntime,
-  writeHostRuntime,
+  totalConfiguredWorkers,
+  type HostRuntimePlan,
 } from "./setup-host.ts";
 import {
   appendJournal,
@@ -95,6 +95,17 @@ export interface UpgradeDeps {
   sleep(ms: number): Promise<void>;
   env: NodeJS.ProcessEnv;
   log(message: string): void;
+  /**
+   * The host runtime this version's package renders, compared against what is
+   * actually installed: the daemon and herdr units, the recovery unit and its
+   * playbook at {@link RECOVER_SCRIPT_INSTALL_PATH}, the herdr pane-shell
+   * config and the herdr-conductor `config.env`. `upgrade` prints
+   * {@link HostRuntimePlan.drift} after a release that re-rendered those
+   * files, so the operator is told that `setup host` is owed before a later
+   * `doctor` has to discover it. Read-only: the plan renders and compares,
+   * it never writes a host file.
+   */
+  hostRuntime(): HostRuntimePlan;
   /**
    * The durable journal sink for the detached fleet installer (#486). Wired
    * to the state-dir journal by `upgrade-install`/`upgrade-rollback`; absent
@@ -135,6 +146,13 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   restartDaemon: async () => {
     await restartDaemon({});
   },
+  // The bare host-global plan — the same render the advisory's `setup host`
+  // command would install: no per-project tail, the recovery unit encoding no
+  // one project's name, and FLEET_CWDS derived from every configured project.
+  hostRuntime: () => {
+    const cfg = loadConfig();
+    return planHostRuntime(undefined, cfg.defaults, telegramStateDir(), undefined, totalConfiguredWorkers(cfg));
+  },
   sleep: Bun.sleep,
   env: process.env,
   log: (message) => process.stdout.write(`${message}\n`),
@@ -821,6 +839,37 @@ export async function rollbackUpgrade(
   if (failures.length > 0) throw new Error(failures.join("; "));
 }
 
+/**
+ * Name the host-runtime destinations this version's render no longer matches,
+ * after a successful upgrade — the installed units, the recovery playbook at
+ * {@link RECOVER_SCRIPT_INSTALL_PATH}, the herdr pane-shell config and the
+ * herdr-conductor `config.env`. A release that re-renders any of those files
+ * silently invalidates the installed copies; telling the operator here — with
+ * the exact command — is what stops the drift from surviving until a later
+ * `doctor` finds it (#598).
+ *
+ * The plan *is* the comparison: {@link UpgradeDeps.hostRuntime} renders every
+ * destination against disk, so a current host prints nothing and a drifted one
+ * names exactly the files only a privileged `setup host` can refresh. This
+ * never mutates a host file and never fails the upgrade — an unreadable host
+ * state is a warning line, not a reason to roll a successful install back.
+ */
+function logHostRuntimeDrift(deps: UpgradeDeps): void {
+  let plan: HostRuntimePlan;
+  try {
+    plan = deps.hostRuntime();
+  } catch (err) {
+    deps.log(
+      `host runtime: could not compare against this version's render — ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return;
+  }
+  if (plan.drift.length === 0) return;
+  deps.log("host runtime:");
+  for (const path of plan.drift) deps.log(`  ${path} differs from this version's render`);
+  deps.log("fix: run `omp-conductor setup host` from the fleet account to re-install the host units");
+}
+
 export async function upgradeConductor(
   options: UpgradeOptions = {},
   overrides: Partial<UpgradeDeps> = {},
@@ -871,6 +920,11 @@ export async function upgradeConductor(
         detail: "all three surfaces and every brief were already pinned to the target release",
       });
     }
+    // Packages are current, but the host units they render may be behind: the
+    // same drift this advisory names after an install applies when a release
+    // re-rendered a template since the operator's last `setup host`. Silent on
+    // a host whose runtime matches (#598).
+    logHostRuntimeDrift(deps);
     return {
       previousVersion: surfaces.cliVersion,
       ...release,
@@ -1118,6 +1172,11 @@ export async function upgradeConductor(
     throw new Error(`upgrade failed: ${failure}; previous installation restored; dispatch remains paused`);
   }
 
+  // The install landed; name any host-runtime destination this version no
+  // longer renders identically, so the operator knows `setup host` is owed
+  // before a later `doctor` reports it as drift (#598).
+  logHostRuntimeDrift(deps);
+
   return {
     previousVersion: surfaces.cliVersion,
     ...release,

package/src/worker.ts

@@ -10,6 +10,8 @@
  * sliding into a merge queue.
  */
 
+import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
 import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
 import type { GateShape, ReleaseBlockContext } from "./release-policy.ts";
 import type { Caps, ResolvedGrants, RunState } from "./types.ts";
@@ -321,6 +323,17 @@ export async function runWorker(
   const schedule = deps.schedule ?? scheduleWallClock;
 
   let session: AgentSessionLike;
+  // The per-issue reviewer brief (#542): when the staged omp settings turn the
+  // advisor on, drop a WATCHDOG.md rendered from this brief's own acceptance
+  // criteria into the worktree before the session exists — the advisor's
+  // watchdog discovery runs at session start, rooted at cwd, and a file that
+  // appears afterwards is a reviewer flying blind. The managed exclude block
+  // ignores that exact path (never `.omp/`), so it cannot reach the PR diff or
+  // a salvage commit. Best-effort: a reviewer brief is advisory, and a staging
+  // failure must not cost the run.
+  if (o.resume === undefined && o.ompSettingsFile !== undefined) {
+    stageIssueWatchdog(o.cwd, o.brief, o.ompSettingsFile);
+  }
   try {
     session = await deps.createSession({
       cwd: o.cwd,
@@ -384,12 +397,33 @@ export async function runWorker(
     | "modelRecoveries"
     | "autoRetryCount"
     | "autoCompactionCount";
+  // Folded once, on whichever exit path actually runs: every return below
+  // passes through `withSessionFacts`, so the advisor's separately-recorded
+  // spend (#542) is added to exactly one result and reported to the daemon
+  // exactly once.
+  let advisorSpendFolded = false;
   const withSessionFacts = (
     result: Omit<WorkerResult, ReliabilityKeys>,
   ): Omit<WorkerResult, ReliabilityKeys> => {
     const { sessionFile, modelFallbackMessage } = session;
+    if (!advisorSpendFolded) {
+      advisorSpendFolded = true;
+      // Advisor turns live in their own `__advisor*.jsonl` and never surface
+      // as primary `message_end`s, so without this read the run's spend — and
+      // therefore `caps.dailySpendUsd` — silently under-counts real provider
+      // consumption (the #46 shape). Read at the very end: the advisor's
+      // review of the final turn may still be landing.
+      const extra = advisorSpendUsd(sessionFile);
+      if (extra > 0) {
+        spendUsd += extra;
+        o.onSpend?.(spendUsd);
+      }
+    }
     return {
       ...result,
+      // Override whatever the exit path spelled: its literal captured the
+      // pre-fold `spendUsd`, and the advisor fold happened after.
+      spendUsd,
       ...(sessionFile === undefined ? {} : { sessionFile }),
       ...(modelFallbackMessage === undefined ? {} : { modelFallbackMessage }),
     };
@@ -807,6 +841,168 @@ function field(source: unknown, key: string): unknown {
   return Reflect.get(source, key);
 }
 
+/** Advisor transcripts are recorded under this reserved stem beside the session's. */
+const ADVISOR_TRANSCRIPT_PREFIX = "__advisor";
+
+/**
+ * Whether a staged omp-settings overlay (the file `materializeOmpSettings`
+ * writes under the run's session directory) turns the omp advisor on.
+ *
+ * The overlay is conductor's own YAML — written by `yaml.stringify` over a map
+ * this package controls — so this is deliberately not a general YAML parser:
+ * it scans for an `advisor:` mapping and asks whether one of its direct
+ * `enabled:` keys reads the literal `true`. Handles both the block form the
+ * overlay is written in (`advisor:\n  enabled: true`) and an inline flow map
+ * (`advisor: { enabled: true }`); everything else — `enabled: false`, no
+ * `advisor` key, an absent overlay — answers false. The staged settings are
+ * what the harness itself resolves, so this is the same truth the session
+ * acts on, not a parallel decode. Exported so the staging decision in
+ * {@link runWorker} is pinned by a unit test.
+ */
+export function overlayEnablesAdvisor(overlayText: string): boolean {
+  const lines = overlayText.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const match = /^(\s*)advisor\s*:\s*(.*)$/.exec(lines[i] ?? "");
+    if (match === null) continue;
+    if (/\benabled\s*:\s*true\b/.test(match[2] ?? "")) return true;
+    const indent = match[1] ?? "";
+    for (let j = i + 1; j < lines.length; j++) {
+      const line = lines[j];
+      if (line === undefined || line.trim() === "") continue;
+      // A line at or shallower than `advisor`'s own indent ends its block.
+      if (!line.startsWith(`${indent} `) && !line.startsWith(`${indent}\t`)) break;
+      if (/^\s*enabled\s*:\s*true\s*$/.test(line)) return true;
+    }
+  }
+  return false;
+}
+
+/** A heading that begins a brief's acceptance section, whatever the brief's casing. */
+const ACCEPTANCE_HEADING = /^#{1,4}\s+acceptance\s+criteri(?:a|on)\s*$/im;
+
+/**
+ * Render the per-issue reviewer brief (#542): the acceptance section of one
+ * worker brief, wrapped so the omp advisor checks the primary against *this
+ * run's own* criteria rather than generic taste. Returns `undefined` when the
+ * brief carries no acceptance section — nothing useful to review against.
+ *
+ * The section runs from the first heading mentioning acceptance criteria to
+ * the next heading of any level (or the end of the brief). Exported so a unit
+ * test can pin the rendering without standing up a session.
+ */
+export function renderIssueWatchdog(brief: string): string | undefined {
+  const lines = brief.split(/\r?\n/);
+  let start = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (ACCEPTANCE_HEADING.test(lines[i] ?? "")) {
+      start = i + 1;
+      break;
+    }
+  }
+  if (start === -1) return undefined;
+  const body: string[] = [];
+  for (let i = start; i < lines.length; i++) {
+    const line = lines[i];
+    if (line === undefined || /^#{1,4}\s/.test(line)) break;
+    body.push(line);
+  }
+  const trimmed = body.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+  if (trimmed === "") return undefined;
+  return (
+    "# Issue watchdog (rendered from this run's brief)\n\n" +
+    "This run's own acceptance criteria. Review the worker against these — not generic taste: " +
+    "a criterion is met only when the transcript or workspace shows it actually verified, and a " +
+    "worker claiming completion without evidence is a concern at least.\n\n" +
+    "## Acceptance criteria\n\n" +
+    `${trimmed}\n`
+  );
+}
+
+/**
+ * Stage the per-issue `WATCHDOG.md` into the worktree when the staged omp
+ * settings turn the advisor on (#542).
+ *
+ * The omp advisor's watchdog discovery is cwd-rooted, so per-issue guidance
+ * cannot ride the out-of-tree settings overlay: it must live inside the
+ * worktree, at `<cwd>/.omp/WATCHDOG.md`, which the managed exclude block
+ * ignores by that exact path (never the `.omp/` directory) so it can never
+ * reach the diff a worker ships or a salvage commit. Best-effort on purpose:
+ * a reviewer brief is advisory, and a worktree too broken to take it will fail
+ * the run on its own terms. Exported so the runWorker staging decision is
+ * testable without a session.
+ */
+export function stageIssueWatchdog(cwd: string, brief: string, overlayFile: string): boolean {
+  let overlayText: string;
+  try {
+    overlayText = readFileSync(overlayFile, "utf8");
+  } catch {
+    return false;
+  }
+  if (!overlayEnablesAdvisor(overlayText)) return false;
+  const watchdog = renderIssueWatchdog(brief);
+  if (watchdog === undefined) return false;
+  try {
+    const dir = join(cwd, ".omp");
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(join(dir, "WATCHDOG.md"), watchdog);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Total USD cost of one run's advisor turns (#542).
+ *
+ * Advisor turns are recorded to a separate `__advisor*.jsonl` beside the
+ * session transcript — the primary session never sees them as `message_end`s —
+ * so without this a run with an advisor under-counts `spendUsd` and the daily
+ * cap is theater, the #46 failure mode. Reads every advisor transcript in the
+ * session's advisor directory and sums the same `usage.cost` blocks
+ * {@link costUsdFromMessage} reads off primary messages: one spelling, so the
+ * two cannot drift. Unreadable or absent transcripts bill nothing — the run
+ * with no advisor stays a zero, and a corrupt advisor log must not crash the
+ * settlement.
+ */
+export function advisorSpendUsd(sessionFile: string | undefined): number {
+  if (sessionFile === undefined || !sessionFile.endsWith(".jsonl")) return 0;
+  // The harness records advisor transcripts in the directory named after the
+  // primary transcript stem (`<dir>/<stem>/__advisor*.jsonl`); slicing the
+  // suffix is exactly what the harness's own cost loader does.
+  const dir = sessionFile.slice(0, -".jsonl".length);
+  let names: string[];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return 0;
+  }
+  let total = 0;
+  for (const name of names) {
+    if (!name.startsWith(ADVISOR_TRANSCRIPT_PREFIX) || !name.endsWith(".jsonl")) continue;
+    let text: string;
+    try {
+      text = readFileSync(join(dir, name), "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of text.split("\n")) {
+      if (line.trim() === "") continue;
+      let entry: unknown;
+      try {
+        entry = JSON.parse(line);
+      } catch {
+        continue;
+      }
+      if (field(entry, "type") !== "message") continue;
+      const message = field(entry, "message");
+      if (field(message, "role") !== "assistant") continue;
+      const cost = costUsdFromMessage(message);
+      if (cost !== undefined) total += cost;
+    }
+  }
+  return total;
+}
+
 /** Flatten an assistant message's content blocks to their plain text. */
 /**
  * The newest assistant text, flattened out of whatever block shape the harness

package/src/worktree.ts

@@ -156,8 +156,20 @@ const EXCLUDE_END = "# <<< omp-conductor";
  * the 2026-08-07 incident's exact shape (`.scratch82/env.sh`). Broader
  * conventions belong in a repo's own `.gitignore`, where its operator chooses
  * them, rather than being imposed by whatever dispatcher happens to be driving.
+ *
+ * The advisor watchdog (`#542`) is the deliberate exception, and the exact-path
+ * spelling is the whole point: with the advisor on, the dispatcher writes a
+ * per-issue `<worktree>/.omp/WATCHDOG.md` from the brief's acceptance criteria
+ * so the mid-run reviewer checks the worker against its own issue — and that
+ * file must never reach the diff a worker ships or the salvage commit. The
+ * standing rule says an ignored *new* file is invisible to salvage, so a name
+ * that could plausibly be a deliverable must never appear; `.omp/WATCHDOG.md`
+ * is the narrowest name that can. Only this exact file is excluded, never the
+ * `.omp/` directory — a directory-wide ignore would hide any future
+ * deliverable an operator legitimately places under `.omp/` (the very trap the
+ * `.scratch*` history describes), and the blind spot stays one path wide.
  */
-const LOCAL_EXCLUDE = [".scratch*/"];
+const LOCAL_EXCLUDE = [".scratch*/", ".omp/WATCHDOG.md"];
 
 /**
  * Adds the managed block to an `info/exclude`, preserving everything else.

package/systemd/omp-conductor-recover.sh

@@ -273,7 +273,7 @@ fresh_rollback_snapshot() {
   newest=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 1 || true)
   [[ -n $newest ]] || return 1
   now=$(epoch_now)
-  ts=$(stat -c %Y "$newest" 2>/dev/null || true)
+  ts=$(stat -c %Y "$newest" 2>/dev/null || stat -f %m "$newest" 2>/dev/null || true)
   [[ -n $ts && $((now - ts)) -le $ROLLBACK_AGE_S ]] || return 1
   printf '%s\n' "$newest"
 }

package/systemd/recover-unit-test.sh

@@ -126,14 +126,14 @@ unit_state() { # <case-dir> <unit> <state|absent>
 }
 
 # A pre-upgrade snapshot `upgrade` durably keeps (configBackupDir()):
-# config.json.pre-upgrade-<ts>. The test pass the age in seconds.
+# config.json.pre-upgrade-<ts>. The test passes the age in seconds.
 write_snapshot() { # <case-dir> <age-seconds>
   local d="$1" age="${2:-0}" now
   now=$(date +%s)
   local snap="$d/state/backups/config/config.json.pre-upgrade-$now"
   printf '%s\n' '{"preUpgrade":true}' >"$snap"
   if (( age > 0 )); then
-    touch -d "@$(( now - age ))" "$snap"
+    perl -e 'utime $ARGV[1], $ARGV[1], $ARGV[0] or die "utime: $!"' "$snap" "$(( now - age ))"
   fi
   printf '%s\n' "$snap"
 }