omp-conductor 0.3.18 → 0.3.19

package/src/upgrade.ts

@@ -0,0 +1,327 @@
+import { readFileSync } from "node:fs";
+import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
+import { setPaused, statusSnapshot } from "./daemon.ts";
+import { fleetLayers, type DispatchLayer, type FleetLayers } from "./fleet.ts";
+import { livingDaemon, restartDaemon } from "./lifecycle.ts";
+import { findProject, loadConfig } from "./config.ts";
+import { renderBriefForProject } from "./setup.ts";
+
+const PACKAGE = "omp-conductor";
+const HERDR_PLUGIN = "herdr-conductor";
+const HERDR_SOURCE = "TerrifiedBug/conductor/herdr";
+const HERDR_UNIT = "herdr-fleet.service";
+const DRAIN_POLL_MS = 5_000;
+const RECOVERY_POLL_MS = 2_000;
+const RECOVERY_ATTEMPTS = 30;
+
+export interface UpgradeCommandResult {
+  code: number;
+  stdout: string;
+  stderr: string;
+}
+
+export interface UpgradeOptions {
+  version?: string;
+  project?: string;
+}
+
+export interface UpgradeResult {
+  previousVersion: string;
+  version: string;
+  gitHead: string;
+  alreadyCurrent: boolean;
+  dispatch: DispatchLayer;
+}
+
+export interface UpgradeDeps {
+  run(command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
+  snapshot(project?: string): { liveWorkers: number };
+  layers(project?: string): FleetLayers;
+  brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
+  daemonIdentity(): { running: boolean; project?: string };
+  setPaused(value: boolean): void;
+  restartDaemon(): Promise<void>;
+  sleep(ms: number): Promise<void>;
+  env: NodeJS.ProcessEnv;
+  log(message: string): void;
+}
+
+async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
+  try {
+    const child = Bun.spawn([command, ...args], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+    const stdout = new Response(child.stdout).text();
+    const stderr = new Response(child.stderr).text();
+    const code = await child.exited;
+    return { code, stdout: await stdout, stderr: await stderr };
+  } catch (err) {
+    return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+const DEFAULT_DEPS: UpgradeDeps = {
+  run: runCommand,
+  snapshot: statusSnapshot,
+  layers: fleetLayers,
+  brief: (projectName) => {
+    const project = findProject(loadConfig(), projectName);
+    const layout = inspectBriefLayout(project.workspaceRoot, renderBriefForProject(project));
+    if (layout.kind !== "overlay") return { kind: layout.kind, current: false };
+    const policy = readFileSync(layout.policyPath, "utf8");
+    const live = readFileSync(layout.orchestratorPath, "utf8");
+    return { kind: layout.kind, current: live === renderBriefForProject(project, policy) };
+  },
+  daemonIdentity: () => {
+    const daemon = livingDaemon();
+    return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
+  },
+  setPaused,
+  restartDaemon: async () => {
+    await restartDaemon({});
+  },
+  sleep: Bun.sleep,
+  env: process.env,
+  log: (message) => process.stdout.write(`${message}\n`),
+};
+
+function commandLine(command: string, args: readonly string[]): string {
+  return [command, ...args].join(" ");
+}
+
+async function mustRun(
+  deps: UpgradeDeps,
+  command: string,
+  args: readonly string[],
+): Promise<UpgradeCommandResult> {
+  const ran = await deps.run(command, args);
+  if (ran.code === 0) return ran;
+  const detail = ran.stderr.trim() || ran.stdout.trim() || `exit ${ran.code}`;
+  throw new Error(`${commandLine(command, args)} failed: ${detail}`);
+}
+
+function parseRegistry(raw: string): { version: string; gitHead: string } {
+  let value: unknown;
+  try {
+    value = JSON.parse(raw);
+  } catch {
+    throw new Error("npm returned invalid release metadata");
+  }
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("npm returned invalid release metadata");
+  }
+  const version = Reflect.get(value, "version");
+  const gitHead = Reflect.get(value, "gitHead");
+  if (typeof version !== "string" || version.length === 0) {
+    throw new Error("npm release metadata has no version");
+  }
+  if (typeof gitHead !== "string" || !/^[0-9a-f]{40}$/i.test(gitHead)) {
+    throw new Error(`npm release ${version} has no full gitHead`);
+  }
+  return { version, gitHead };
+}
+
+function ompPluginVersion(raw: string): string | undefined {
+  try {
+    const parsed = JSON.parse(raw) as { npm?: Array<{ name?: unknown; version?: unknown }> };
+    const plugin = parsed.npm?.find((entry) => entry.name === PACKAGE);
+    return typeof plugin?.version === "string" ? plugin.version : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function herdrPluginSource(raw: string): string | undefined {
+  const line = raw.split("\n").find((candidate) => /^- herdr-conductor\b/.test(candidate.trim()));
+  return line?.match(/\[([^\]]+)\]\s*$/)?.[1];
+}
+
+async function inspectSurfaces(deps: UpgradeDeps): Promise<{
+  cliVersion: string;
+  ompVersion?: string;
+  herdrSource?: string;
+}> {
+  const session = deps.env["HERDR_SESSION"] || "fleet";
+  const [cli, omp, herdr] = await Promise.all([
+    mustRun(deps, "omp-conductor", ["--version"]),
+    mustRun(deps, "omp", ["plugin", "list", "--json"]),
+    mustRun(deps, "herdr", ["--session", session, "plugin", "list"]),
+  ]);
+  return {
+    cliVersion: cli.stdout.trim(),
+    ompVersion: ompPluginVersion(omp.stdout),
+    herdrSource: herdrPluginSource(herdr.stdout),
+  };
+}
+
+function expectedHerdrSource(gitHead: string): string {
+  return `github:${HERDR_SOURCE}@${gitHead}`;
+}
+
+function surfacesCurrent(
+  surfaces: Awaited<ReturnType<typeof inspectSurfaces>>,
+  version: string,
+  gitHead: string,
+): boolean {
+  return (
+    surfaces.cliVersion === version &&
+    surfaces.ompVersion === version &&
+    surfaces.herdrSource === expectedHerdrSource(gitHead)
+  );
+}
+
+async function waitForDrain(deps: UpgradeDeps, project?: string): Promise<void> {
+  let last = -1;
+  while (true) {
+    const workers = deps.snapshot(project).liveWorkers;
+    if (workers === 0) return;
+    if (workers !== last) deps.log(`waiting for ${workers} live worker(s) to finish`);
+    last = workers;
+    await deps.sleep(DRAIN_POLL_MS);
+  }
+}
+
+function recoveryProblem(
+  layers: FleetLayers,
+  initial: FleetLayers,
+  liveWorkers: number,
+): string | undefined {
+  if (layers.ticks !== initial.ticks) return `ticks changed from ${initial.ticks} to ${layers.ticks}`;
+  if (layers.recovery !== initial.recovery) {
+    return `recovery changed from ${initial.recovery} to ${layers.recovery}`;
+  }
+  if (initial.herdr === "active" && layers.herdr !== "active") return `Herdr is ${layers.herdr}`;
+  if (initial.herdr === "active" && initial.recovery === "clear" && layers.pane !== "live") {
+    return `orchestrator pane is ${layers.pane}`;
+  }
+  if (initial.dispatch !== "stopped" && !layers.daemon.running) return "dispatch daemon is not running";
+  if (initial.dispatch === "stopped" && layers.daemon.running) return "stopped dispatch daemon was started";
+  if (initial.dispatch !== "stopped" && !layers.paused) return "dispatch resumed before verification";
+  if (liveWorkers !== 0) return `${liveWorkers} worker(s) appeared during verification`;
+  return undefined;
+}
+
+async function waitForRecovery(
+  deps: UpgradeDeps,
+  initial: FleetLayers,
+  project?: string,
+): Promise<void> {
+  let problem = "recovery did not settle";
+  for (let attempt = 0; attempt < RECOVERY_ATTEMPTS; attempt += 1) {
+    problem = recoveryProblem(deps.layers(project), initial, deps.snapshot(project).liveWorkers) ?? "";
+    if (problem === "") return;
+    await deps.sleep(RECOVERY_POLL_MS);
+  }
+  throw new Error(`upgrade verification failed: ${problem}`);
+}
+
+async function upgradeBrief(
+  deps: UpgradeDeps,
+  kind: BriefLayout["kind"],
+  project?: string,
+): Promise<void> {
+  const selected = project === undefined ? [] : ["--project", project];
+  if (kind === "legacy-handwritten") {
+    await mustRun(deps, "omp-conductor", ["brief-upgrade", "--retrofit", "--apply", ...selected]);
+  }
+  await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
+}
+
+export async function upgradeConductor(
+  options: UpgradeOptions = {},
+  overrides: Partial<UpgradeDeps> = {},
+): Promise<UpgradeResult> {
+  const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  if (deps.env["HERDR_ENV"] !== undefined) {
+    throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
+  }
+
+  const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
+  const daemon = deps.daemonIdentity();
+  if (daemon.running && options.project !== undefined && daemon.project !== options.project) {
+    const active = daemon.project === undefined ? "an unrecorded project" : daemon.project;
+    throw new Error(
+      `active daemon serves ${active}; refusing to upgrade --project ${options.project} while it is running`,
+    );
+  }
+  const project = options.project ?? daemon.project;
+
+  const release = parseRegistry(
+    (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
+  );
+  const initial = deps.layers(project);
+  const surfaces = await inspectSurfaces(deps);
+  const brief = deps.brief(project);
+  const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
+  if (!installNeeded && brief.current) {
+    return {
+      previousVersion: surfaces.cliVersion,
+      ...release,
+      alreadyCurrent: true,
+      dispatch: initial.dispatch,
+    };
+  }
+
+  if (brief.kind === "missing") throw new Error("no ORCHESTRATOR.md exists for the configured project");
+  if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
+
+  if (initial.dispatch === "running") deps.setPaused(true);
+  await waitForDrain(deps, project);
+
+  deps.log(`upgrading conductor ${surfaces.cliVersion} → ${release.version}`);
+  if (installNeeded) {
+    await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
+    await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
+    if (surfaces.herdrSource?.startsWith("local:")) {
+      await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
+    }
+    await mustRun(deps, "herdr", [
+      "plugin",
+      "install",
+      HERDR_SOURCE,
+      "--ref",
+      release.gitHead,
+      "--yes",
+    ]);
+  }
+  await upgradeBrief(deps, brief.kind, project);
+
+  if (initial.herdr === "active") {
+    await mustRun(deps, "systemctl", ["restart", HERDR_UNIT]);
+  }
+  if (initial.dispatch !== "stopped") {
+    await deps.restartDaemon();
+  }
+
+  const installed = await inspectSurfaces(deps);
+  if (!surfacesCurrent(installed, release.version, release.gitHead)) {
+    throw new Error(
+      `upgrade verification failed: cli=${installed.cliVersion}, omp=${installed.ompVersion ?? "missing"}, ` +
+        `herdr=${installed.herdrSource ?? "missing"}`,
+    );
+  }
+  if (initial.herdr === "active") {
+    const active = await mustRun(deps, "systemctl", ["is-active", HERDR_UNIT]);
+    if (active.stdout.trim() !== "active") throw new Error(`${HERDR_UNIT} is not active after restart`);
+  }
+  await waitForRecovery(deps, initial, project);
+  await deps.sleep(1_000);
+  await waitForRecovery(deps, initial, project);
+
+  if (initial.dispatch === "running") {
+    deps.setPaused(false);
+    if (deps.layers(project).dispatch !== "running") {
+      throw new Error("upgrade installed successfully, but dispatch did not resume");
+    }
+  }
+
+  return {
+    previousVersion: surfaces.cliVersion,
+    ...release,
+    alreadyCurrent: false,
+    dispatch: initial.dispatch,
+  };
+}

package/src/worker.ts

@@ -11,10 +11,14 @@
  */
 
 import { createSession, disposeSession } from "./omp.ts";
-import type { Caps, RunState } from "./types.ts";
+import type { ReleaseShape } from "./release-policy.ts";
+import type { Caps, ReleasePolicy, RunState } from "./types.ts";
 
-/** A PR link the worker pushed, recognised anywhere in its report. */
-const PR_URL_PATTERN = /https:\/\/github\.com\/\S+\/pull\/\d+/;
+/** Structured evidence fields from the worker's final report. */
+const PR_URL_PATTERN = /^pr:\s*(https:\/\/github\.com\/\S+\/pull\/\d+)\s*$/im;
+const HEAD_SHA_PATTERN = /^head:\s*([0-9a-f]{40})\s*$/im;
+const PUSHED_GREEN_PATTERN = /^state:\s*pushed-green\s*$/im;
+const BLOCKED_PATTERN = /^state:\s*blocked\s*$/im;
 
 /** `{{KEY}}` placeholders in a brief template. */
 const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
@@ -42,7 +46,20 @@ export interface WorkerOpts {
    * the harness to pick, which is what an unconfigured project wants.
    */
   model?: string;
+  /** Effective release/deploy gate for this session. */
+  releasePolicy?: ReleasePolicy;
+  /** Durable audit sink for rejected release/deploy calls. */
+  onReleaseBlocked?: (shape: ReleaseShape) => void;
+  /**
+   * Reads the effective ceiling at each turn boundary. Omitted, the configured
+   * startup cap remains fixed for the run.
+   */
+  maxTurns?: () => number;
+  /** Synchronous cap latch; called before the session abort begins. */
+  onKilled?: (by: KilledBy) => void;
   onTurn?: (n: number) => void;
+  /** Cumulative USD spend, reported as each cost-bearing message finishes. */
+  onSpend?: (usd: number) => void;
   /**
    * The transcript path, handed over the moment the session opens it rather
    * than at the end with {@link WorkerResult.sessionFile}. Both report the same
@@ -66,6 +83,7 @@ export interface RunWorkerDeps {
 export interface WorkerResult {
   state: RunState;
   prUrl?: string;
+  headSha?: string;
   turns: number;
   spendUsd: number;
   report: string;
@@ -101,23 +119,30 @@ export function renderBrief(template: string, vars: Record<string, string>): str
 }
 
 /**
- * Read the run's outcome out of the worker's final report.
+ * Read the run's structured outcome evidence from its final report.
  *
- * Success has to be claimed explicitly (`pushed-green`); everything else,
- * including an empty or unparseable report, is a failure. Defaulting the other
- * way would let a session that died mid-thought be reported as merge-ready.
+ * A textual `pushed-green` claim is not success by itself. The exact state line
+ * must carry both a PR URL and the head SHA observed after CI; the daemon then
+ * asks the tracker to verify those facts independently. Missing or malformed
+ * evidence fails closed.
  */
-export function deriveResult(report: string): { state: RunState; prUrl?: string } {
-  const haystack = report.toLowerCase();
-  const state: RunState = haystack.includes("pushed-green")
-    ? "pushed-green"
-    : haystack.includes("ci-red")
-      ? "failed"
-      : haystack.includes("blocked")
-        ? "blocked"
-        : "failed";
-  const prUrl = PR_URL_PATTERN.exec(report)?.[0];
-  return prUrl === undefined ? { state } : { state, prUrl };
+export function deriveResult(report: string): {
+  state: RunState;
+  prUrl?: string;
+  headSha?: string;
+} {
+  const prUrl = PR_URL_PATTERN.exec(report)?.[1];
+  const headSha = HEAD_SHA_PATTERN.exec(report)?.[1]?.toLowerCase();
+  if (PUSHED_GREEN_PATTERN.test(report) && prUrl !== undefined && headSha !== undefined) {
+    return { state: "pushed-green", prUrl, headSha };
+  }
+
+  const state: RunState = BLOCKED_PATTERN.test(report) ? "blocked" : "failed";
+  return {
+    state,
+    ...(prUrl === undefined ? {} : { prUrl }),
+    ...(headSha === undefined ? {} : { headSha }),
+  };
 }
 
 /**
@@ -145,8 +170,8 @@ export async function runWorker(
   o: WorkerOpts,
   deps: RunWorkerDeps = { createSession },
 ): Promise<WorkerResult> {
-  // Read the caps once, by value: `o.caps` belongs to the caller's config.
-  const { workerMaxTurns, workerWallClockMs } = o.caps;
+  const { workerWallClockMs } = o.caps;
+  const maxTurns = o.maxTurns ?? (() => o.caps.workerMaxTurns);
 
   const session = await deps.createSession({
     cwd: o.cwd,
@@ -154,6 +179,8 @@ export async function runWorker(
     ...(o.model === undefined ? {} : { model: o.model }),
     // Prevention half of #24: structured file tools cannot leave this worktree.
     confineToCwd: true,
+    ...(o.releasePolicy === undefined ? {} : { releasePolicy: o.releasePolicy }),
+    ...(o.onReleaseBlocked === undefined ? {} : { onReleaseBlocked: o.onReleaseBlocked }),
   });
 
   // Before the first turn, not after the last: a caller that only learns the
@@ -195,6 +222,7 @@ export async function runWorker(
   const kill = (by: KilledBy) => {
     if (killedBy !== undefined) return;
     killedBy = by;
+    o.onKilled?.(by);
     clearWallClock();
     session.abort();
     // An aborted session may never reach a terminal `agent_end`. The cap is the
@@ -208,7 +236,7 @@ export async function runWorker(
     // `message_end`s and would burn the cap on a run that is behaving.
     turns += 1;
     o.onTurn?.(turns);
-    if (turns > workerMaxTurns) kill("turns");
+    if (turns > maxTurns()) kill("turns");
   });
 
   session.on("message_end", (event) => {
@@ -223,16 +251,25 @@ export async function runWorker(
     // transcripts, 2026-08-07). The earlier agent_end.telemetry path never
     // fired, so every run recorded $0 and the daily cap was theater (#46).
     const cost = costUsdFromMessage(message);
-    if (cost !== undefined) spendUsd += cost;
+    if (cost !== undefined) {
+      spendUsd += cost;
+      o.onSpend?.(spendUsd);
+    }
   });
 
   session.on("agent_end", (event) => {
     // Fallback for harnesses that only attach cost on the terminal event.
     const estimated = field(field(field(event, "telemetry"), "cost"), "estimatedUsd");
-    if (typeof estimated === "number" && Number.isFinite(estimated) && estimated > 0) {
+    if (
+      spendUsd === 0 &&
+      typeof estimated === "number" &&
+      Number.isFinite(estimated) &&
+      estimated > 0
+    ) {
       // Prefer message totals when both exist — do not double-count a run that
       // already accumulated per-message costs.
-      if (spendUsd === 0) spendUsd += estimated;
+      spendUsd = estimated;
+      o.onSpend?.(spendUsd);
     }
 
     // Anything that is not literally `false` — including garbage or nothing at
@@ -283,12 +320,12 @@ export async function runWorker(
     return withSessionFacts({ state: "killed", turns, spendUsd, report, killedBy });
   }
 
-  const { state, prUrl } = deriveResult(report);
-  return withSessionFacts(
-    prUrl === undefined
-      ? { state, turns, spendUsd, report }
-      : { state, prUrl, turns, spendUsd, report },
-  );
+  return withSessionFacts({
+    ...deriveResult(report),
+    turns,
+    spendUsd,
+    report,
+  });
 }
 
 /**

package/src/worktree.ts

@@ -608,3 +608,69 @@ export async function removeWorktree(
 
   await git(["worktree", "prune"], mirrorPath);
 }
+
+export type RetainedWorktreeCleanup =
+  | { kind: "removed" }
+  | { kind: "retained"; reason: "dirty" | "unpushed" | "unknown"; detail: string };
+
+/**
+ * Reap a terminal run's tree and local mirror branch without deleting the only
+ * copy of work. Tracker state is proved by the caller; this function proves the
+ * local half after refreshing remote refs. Any ambiguity retains everything.
+ */
+export async function cleanupRetainedWorktree(
+  mirrorPath: string,
+  worktreePath: string,
+  branch: string,
+): Promise<RetainedWorktreeCleanup> {
+  if (!existsSync(mirrorPath)) {
+    return existsSync(worktreePath)
+      ? { kind: "retained", reason: "unknown", detail: "mirror is missing" }
+      : { kind: "removed" };
+  }
+
+  try {
+    if (existsSync(worktreePath)) {
+      const dirty = await git(["status", "--porcelain"], worktreePath);
+      if (dirty !== "") {
+        return { kind: "retained", reason: "dirty", detail: "worktree has uncommitted changes" };
+      }
+      const actual = await git(["rev-parse", "--abbrev-ref", "HEAD"], worktreePath);
+      if (actual !== branch) {
+        return {
+          kind: "retained",
+          reason: "unknown",
+          detail: `worktree is on ${actual}, expected ${branch}`,
+        };
+      }
+    }
+
+    // A deleted remote branch can make a pushed commit look local-only until
+    // the default branch is fetched. Failure is ambiguity, never permission.
+    await git(["fetch", "--prune", "origin"], mirrorPath);
+
+    const ref = `refs/heads/${branch}`;
+    if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) {
+      const unique = await git(["rev-list", ref, "--not", "--remotes"], mirrorPath);
+      if (unique !== "") {
+        return {
+          kind: "retained",
+          reason: "unpushed",
+          detail: `${branch} has commits absent from every remote ref`,
+        };
+      }
+    }
+
+    await removeWorktree(mirrorPath, worktreePath);
+    if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) {
+      await git(["branch", "-D", branch], mirrorPath);
+    }
+    return { kind: "removed" };
+  } catch (err) {
+    return {
+      kind: "retained",
+      reason: "unknown",
+      detail: err instanceof Error ? err.message : String(err),
+    };
+  }
+}

package/systemd/omp-conductor.service.example

@@ -16,8 +16,10 @@
 #      fleet on a ≤8 GB box.
 #
 # Install:
-#   sudo cp omp-conductor.service.example /etc/systemd/system/omp-conductor.service
-#   sudo systemctl daemon-reload && sudo systemctl enable --now omp-conductor
+#   sudo install -m 0644 omp-conductor.service.example /etc/systemd/system/omp-conductor.service
+#   sudo systemctl daemon-reload
+#   sudo systemctl enable omp-conductor.service
+#   sudo systemctl restart omp-conductor.service
 
 [Unit]
 Description=omp-conductor dispatch daemon
@@ -38,7 +40,7 @@ WorkingDirectory=/home/fleet
 
 # Foreground daemon so systemd tracks MainPID. `omp-conductor start` backgrounds;
 # under a unit, call `daemon` directly.
-ExecStart=/home/fleet/.local/bin/omp-conductor daemon --port 7432
+ExecStart=/home/fleet/.local/bin/omp-conductor daemon --port 8787
 
 Restart=on-failure
 # Handled SIGTERM exits 143; without this, Restart=on-failure brings the unit