omp-conductor 0.15.12 → 0.16.0

package/src/lifecycle.ts

@@ -25,6 +25,10 @@ import { spawn, spawnSync } from "node:child_process";
 import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+// Type-only: erased at runtime, so the "free of every other module" property
+// below survives — this module still opens no store, loads no config and runs
+// no `gh`, and `stop`/`status` keep working when the config is broken.
+import type { DaemonStopDraft } from "./types.ts";
 
 /**
  * The systemd unit name operators are expected to install for a supervised
@@ -480,6 +484,58 @@ export async function startDaemon(
   );
 }
 
+/**
+ * The delivery facts a mediated stop/restart records at the exact chokepoint:
+ * the request's own provenance, how the stop is about to be delivered, and the
+ * daemon pid it targets (#378). The recorder is injected by the CLI callers
+ * (which own the store); this module stays free of it.
+ */
+export interface StopDelivery {
+  /** The request facts built by the CLI (commands/stop.ts, commands/restart.ts). */
+  provenance: DaemonStopDraft;
+  /** Whether the stop is about to go through `systemctl` or a raw signal. */
+  via: "systemctl" | "signal";
+  /** The daemon pid being stopped, when one was known. */
+  pid?: number;
+}
+
+/** The injected recorder invoked immediately before a stop/restart is signalled. */
+export type StopDeliveryFn = (stop: StopDelivery) => void;
+
+/**
+ * The honest provenance for a daemon that received a stop with no mediated
+ * request anywhere: unattributed by construction — Linux exposes no sender
+ * identity for a signal, so the record must say "unattributed" rather than
+ * guess one — plus what the receiving process actually knows: its own pid,
+ * the moment (stamped by the store), the runtime directory, and the projects
+ * it serves with their live-run counts.
+ *
+ * Storage belongs at the caller: the daemon's SIGINT/SIGTERM path writes the
+ * returned draft through the store before/while draining. This builds the
+ * facts, store-free, so a raw `kill`/out-of-band `systemctl stop` — no
+ * conductor CLI in the delivery path at all — still leaves a durable row the
+ * next `omp-conductor status` can show.
+ */
+export function externalStopProvenance(o: {
+  /** The daemon pid that received the signal, when known. */
+  daemonPid?: number;
+  /** The daemon's runtime directory (host paths only — never secrets). */
+  runtimeDir: string;
+  /** Every served project with its live-run count at signal time. */
+  affected: { project: string; live: number }[];
+  reason?: string;
+}): DaemonStopDraft {
+  return {
+    controlPath: "external signal",
+    scope: "global",
+    ...(o.daemonPid === undefined ? {} : { daemonPid: o.daemonPid }),
+    runtimeDir: o.runtimeDir,
+    affected: o.affected,
+    reason: o.reason ?? "external signal — no mediated stop request was recorded",
+    unattributed: true,
+  };
+}
+
 /**
  * How the last stop actually landed. Callers print this so an operator can
  * tell a supervised stop from a bare SIGTERM without reading the journal.
@@ -510,7 +566,35 @@ export interface RestartResult {
  * a stopped unit). The grace period is a deadline, not a clean drain: a tick
  * with a worker in flight can run for that worker's whole wall clock.
  */
-export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopResult> {
+export async function stopDaemon(
+  o: {
+    timeoutMs?: number;
+    /**
+     * The request facts for this stop, when a mediated caller carries them.
+     * With `record`, the durable row is written immediately before the signal
+     * is sent — the exact ordering "provenance precedes signalling" refers to
+     * — and only when a stop actually lands: a request that finds no daemon
+     * stops nothing and records nothing. Drain-style callers whose restart may
+     * never execute additionally record the request at entry themselves.
+     */
+    provenance?: DaemonStopDraft;
+    /** Invoked just before the daemon is signalled, with the delivery method. */
+    record?: StopDeliveryFn;
+  } = {},
+): Promise<StopResult> {
+  const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
+    if (o.provenance === undefined || o.record === undefined) return;
+    o.record({
+      provenance: {
+        ...o.provenance,
+        controlPath: `${o.provenance.controlPath} via ${via}`,
+        ...(pid === undefined ? {} : { daemonPid: pid }),
+      },
+      via,
+      pid,
+    });
+  };
+
   const rec = livingDaemon();
   if (rec === undefined) {
     // `livingDaemon` already cleared a stale file; this covers the unparseable
@@ -521,6 +605,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
       throw new Error(ownershipUnknown("stop", decision.reason));
     }
     if (decision.kind === "stop") {
+      recordStop("systemctl", decision.mainPid);
       await runSystemdStop(decision.mainPid, o.timeoutMs);
       clearRecord();
       return { kind: "stopped", pid: decision.mainPid, via: "systemctl" };
@@ -536,6 +621,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
     throw new Error(ownershipUnknown("stop", decision.reason));
   }
   if (decision.kind === "stop") {
+    recordStop("systemctl", rec.pid);
     await runSystemdStop(rec.pid, o.timeoutMs);
     clearRecord();
     return { kind: "stopped", pid: rec.pid, via: "systemctl" };
@@ -544,6 +630,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
   // decision.kind === "not-ours": confirmed no unit, inactive unit, no systemd
   // binary, or a unit whose MainPID is somebody else. Only a *confirmed*
   // negative is safe to signal.
+  recordStop("signal", rec.pid);
   const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
   if (!gone) {
     // The record stays: something is still holding that pid, and forgetting
@@ -577,8 +664,29 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
  * Returns the record of the process that is now answering `/healthz`.
  */
 export async function restartDaemon(
-  o: { port?: number; project?: string; timeoutMs?: number } = {},
+  o: {
+    port?: number;
+    project?: string;
+    timeoutMs?: number;
+    /** Request facts and recorder, same semantics as {@link stopDaemon}: the
+     *  durable row is written immediately before the restarting signal. */
+    provenance?: DaemonStopDraft;
+    record?: StopDeliveryFn;
+  } = {},
 ): Promise<RestartResult> {
+  const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
+    if (o.provenance === undefined || o.record === undefined) return;
+    o.record({
+      provenance: {
+        ...o.provenance,
+        controlPath: `${o.provenance.controlPath} via ${via}`,
+        ...(pid === undefined ? {} : { daemonPid: pid }),
+      },
+      via,
+      pid,
+    });
+  };
+
   const previous = livingDaemon();
   const ownership = probeUnit();
   if (ownership.kind === "unknown") {
@@ -591,6 +699,7 @@ export async function restartDaemon(
     // startDaemon() — that leaves the unit failed while handing the operator
     // a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
     // Any manager refusal is terminal; so is unproven ownership afterwards.
+    recordStop("systemctl", previous?.pid);
     const record = await restoreFailedUnit(o.timeoutMs, o.project);
     return { previous, record, via: "systemctl" };
   }
@@ -602,6 +711,7 @@ export async function restartDaemon(
     // Ownership is proven. A refused/timed-out restart must not fall through
     // to stopDaemon's signal path — that is the exact bounce this module exists
     // to prevent (SIGTERM → exit 143 → Restart=on-failure → new MainPID).
+    recordStop("systemctl", ownership.pid);
     const ran = systemctl(["restart", SYSTEMD_UNIT]);
     if (!ran.ok) {
       throw new Error(systemctlFailure("restart", ran));
@@ -615,6 +725,7 @@ export async function restartDaemon(
 
   // Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
   // is somebody else. The detached CLI daemon is the only path left.
+  recordStop("signal", previous?.pid);
   await stopDaemon({ timeoutMs: o.timeoutMs });
   const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
   return { previous, record, via: "cli" };

package/src/log.ts

@@ -0,0 +1,40 @@
+/**
+ * Shared daemon logging and best-effort escalation delivery.
+ *
+ * Split out of `daemon.ts` so the extracted admission module (and any future
+ * extracted concern) can log and escalate without importing the composition
+ * root back — which would be the circular-import failure mode the daemon split
+ * (#571) exists to rule out. `daemon.ts` composes; the leaf modules talk only
+ * to other leaves.
+ */
+
+import { escalationIssueRef } from "./escalate.ts";
+import type { Escalation } from "./types.ts";
+
+/** One timestamped line to stderr — the daemon's canonical log sink. */
+export function log(msg: string): void {
+  process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
+}
+
+/** A stable one-line rendering of an unknown thrown value. */
+export function errText(e: unknown): string {
+  return e instanceof Error ? (e.stack ?? e.message) : String(e);
+}
+
+/**
+ * Best-effort escalation delivery: never let a failed page take down the
+ * pass, and say in the log that it failed. Structural `escalate` only — the
+ * caller's full dep object satisfies it without importing anything back.
+ */
+export async function safeEscalate(
+  d: { escalate(e: Escalation): Promise<void> },
+  e: Escalation,
+): Promise<boolean> {
+  try {
+    await d.escalate(e);
+    return true;
+  } catch (err) {
+    log(`escalation for ${escalationIssueRef(e.issue)} could not be delivered: ${errText(err)}`);
+    return false;
+  }
+}

package/src/model-fallback.ts

@@ -26,6 +26,7 @@ import type { FailureClass, RunRecord } from "./types.ts";
 export const FAILOVER_CLASSES: readonly FailureClass[] = [
   "provider-transient",
   "provider-credit",
+  "provider-capacity",
 ];
 
 /** The default for a project's `modelFallbackThreshold` when it is absent or
@@ -44,7 +45,7 @@ export interface ProviderFailureFacts {
   /** How many terminal runs at the head of the chain failed as a provider. */
   streak: number;
   /** Those runs' classes, newest first — the set is either one class or a mix
-   *  of `provider-transient` and `provider-credit`. */
+   *  of `provider-transient`, `provider-credit` and `provider-capacity`. */
   classes: FailureClass[];
   /**
    * The model the most recent failure dispatched on, when its row recorded
@@ -97,7 +98,7 @@ export interface ModelChoice {
  * returned unchanged and `fallback` stays false — today's dispatch byte for
  * byte. Once the streak reaches the threshold the chain advances one slot per
  * extra failure, clamped to the last model: the chain is exhausted there, and
- * the existing escalation path (the provider-transient strike cap) settles it,
+ * the existing escalation path (the provider-class strike cap) settles it,
  * naming every model tried.
  */
 export function resolveDispatchModel(args: {

package/src/omp-settings.ts

@@ -0,0 +1,114 @@
+/**
+ * The fleet-owned omp settings overlay (#537).
+ *
+ * Workers inherit omp configuration from the daemon account's global
+ * `~/.omp/agent/config.yml` and nothing else. This module is the channel for
+ * saying "this project's workers run with *these* omp settings" without
+ * editing that fleet-wide file: a project's `ompSettings` map (an opaque
+ * settings tree the harness owns) is materialised to a YAML overlay *outside
+ * the worktree* — under the run's own session directory, alongside the
+ * transcript and the control socket — and threaded to the session through
+ * omp's own `Settings.init({ configFiles: [<path>] })` seam.
+ *
+ * Two properties make this channel distinct from the obvious alternative of
+ * writing `<worktree>/.omp/config.yml`:
+ *
+ *   - The overlay is fleet-owned, never a file in the git checkout, so the
+ *     diff a worker eventually ships is untouched and nothing needs an ignore
+ *     entry. The worktree ignore block's standing rule — an ignored new file
+ *     is invisible to salvage — is exactly why an in-tree staging was refused.
+ *   - It is rewritten from config on every dispatch, so a config edit takes
+ *     effect on the next attempt (and a resumed attempt reuses the kept
+ *     session directory with the *current* config, not the one as of first
+ *     provision).
+ *
+ * Conductor validates YAML shape only — omp owns the schema. An unknown key or
+ * a wrong-typed value is omp's to reject when it resolves the overlay, never
+ * conductor's to interpret.
+ */
+
+import { unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { stringify } from "yaml";
+import { stateDir } from "./config.ts";
+import type { ProjectConfig } from "./types.ts";
+
+/** The overlay filename inside a run's session directory. */
+export const OMP_SETTINGS_FILE = "omp-settings.yml";
+
+/**
+ * The effective overlay map for a project: the opaque `ompSettings` map plus
+ * the retry keys (`retry.modelFallback`, `retry.fallbackChains.default`)
+ * derived from `modelFallbacks` — #539's staging half, moved onto this general
+ * channel so there is one way to stage settings into a worker session, not one
+ * seam per feature.
+ *
+ * `ompSettings.retry` is the operator's explicit word: when it is already a
+ * mapping, the derived retry block is skipped entirely (the overlay then says
+ * what it says, and the dispatch-failover chain #286 reads is unchanged — only
+ * omp's *retry-time* model swap respects the explicit stanza). Otherwise the
+ * derived block is merged in, preserving #539's behaviour exactly.
+ *
+ * Returns `undefined` when nothing is staged — a project with neither an
+ * `ompSettings` map nor a model-fallback chain gets no overlay at all, so its
+ * dispatch is byte-for-byte today's.
+ */
+export function ompSettingsOverlay(
+  p: Pick<ProjectConfig, "ompSettings" | "modelFallbacks">,
+): Record<string, unknown> | undefined {
+  const map: Record<string, unknown> = { ...(p.ompSettings ?? {}) };
+  const chain = p.modelFallbacks;
+  const explicitRetry =
+    typeof map.retry === "object" && map.retry !== null && !Array.isArray(map.retry);
+  if (chain !== undefined && chain.length > 0 && !explicitRetry) {
+    map.retry = {
+      modelFallback: true,
+      fallbackChains: { default: [...chain] },
+    };
+  }
+  return Object.keys(map).length === 0 ? undefined : map;
+}
+
+/**
+ * The directory every run's session dir lives under — where the overlay is
+ * materialised at dispatch. `doctor` probes its writability to report whether
+ * the next dispatch's overlay can actually land.
+ */
+export function sessionRootDir(): string {
+  return join(stateDir(), "sessions");
+}
+
+/**
+ * Materialises a project's effective overlay to `<sessionDir>/omp-settings.yml`
+ * and returns its absolute path, or `undefined` when nothing is staged.
+ *
+ * When nothing is staged, any overlay a previous attempt wrote is removed, so
+ * a project that dropped its map leaves no stale file in a kept session dir —
+ * the resume path reuses that directory, and an orphaned overlay with nobody
+ * reading it would be the exact silent fake this channel exists to avoid.
+ *
+ * Re-runs every dispatch, so a kept worktree picks up a config change on the
+ * next attempt rather than at the next worktree creation. A non-staged
+ * project's dispatch degrades to the harness discovering settings exactly as
+ * it always has.
+ */
+export function materializeOmpSettings(
+  project: Pick<ProjectConfig, "ompSettings" | "modelFallbacks">,
+  sessionDir: string,
+): string | undefined {
+  const path = join(sessionDir, OMP_SETTINGS_FILE);
+  const overlay = ompSettingsOverlay(project);
+  if (overlay === undefined) {
+    try {
+      unlinkSync(path);
+    } catch {
+      // Nothing written — a fresh session dir, or the absent case on first
+      // dispatch. Either way there is no stale overlay to remove.
+    }
+    return undefined;
+  }
+  // 0600: the overlay can carry provider names and model selectors, and the
+  // session dir already holds the run's private channels.
+  writeFileSync(path, stringify(overlay), { mode: 0o600 });
+  return path;
+}

package/src/omp.ts

@@ -44,6 +44,14 @@ export interface AgentSessionLike {
    * `"*"` for every event. The payload is the harness's own event union, which
    * this package cannot name without the peer dependency, so it arrives as
    * `unknown` and each caller narrows the two or three fields it reads.
+   *
+   * One event is not the harness's at all: `"session_exit"` fires when the
+   * underlying session *process* has terminated, carrying
+   * `{ type: "session_exit", code?: number | null }`. It is the single real
+   * terminal signal — a crashed or killed session is never confused with one
+   * that merely stopped streaming (`"agent_end"`). A supervisor may receive it
+   * after its own `dispose()`, and must decide what an exit means from its own
+   * state rather than assuming every exit is a crash.
    */
   on(event: string, cb: (e: unknown) => void): void;
   abort(): void;
@@ -160,6 +168,17 @@ export async function createLocalSession(opts: {
   releaseGrants?: ResolvedGrants;
   /** Durable audit callback invoked only when that gate rejects a call. */
   onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
+  /**
+   * The fleet-owned omp settings overlay staged into this session's omp
+   * settings (#537): an absolute path to a YAML overlay the daemon materialised
+   * from the project's `ompSettings` map (plus the retry keys derived from
+   * `modelFallbacks`), passed to `Settings.init({ configFiles: [<path>] })` so
+   * it layers on top of the ordinary global/project discovery — never an
+   * isolated instance, which would drop providers and approvals out from under
+   * the worker. Absent, no settings are passed and the harness discovers
+   * exactly as it does today.
+   */
+  ompSettingsFile?: string;
   /**
    * The conductor verb socket this session's mutation tools call (#126).
    *
@@ -235,11 +254,31 @@ export async function createLocalSession(opts: {
     conductorVerbs(opts.verbSocketPath),
   ];
 
+  // The fleet-owned omp settings overlay (#537): when the daemon staged one,
+  // layer it into a Settings instance over the ordinary global/project
+  // discovery. `Settings.init({ cwd, configFiles })` keeps the daemon account's
+  // global config (providers, approvals, modelRoles) and any project
+  // `.omp/config.yml` intact — the overlay is deep-merged *after* them — and
+  // never `Settings.isolated()`, which would build from the overlay alone and
+  // drop everything out from under the worker. Absent a staged overlay, no
+  // settings are passed and the harness discovers exactly as it does today.
+  let overlaySettings: unknown = undefined;
+  if (opts.ompSettingsFile !== undefined) {
+    const Settings = Reflect.get(namespace, "Settings");
+    if (typeof Settings === "function" && typeof Settings.init === "function") {
+      overlaySettings = await Settings.init({
+        cwd: opts.cwd,
+        configFiles: [opts.ompSettingsFile],
+      });
+    }
+  }
+
   const created = await mod.createAgentSession({
     cwd: opts.cwd,
     // A raw pattern rather than a resolved Model: the harness resolves it
     // after extensions load, so we never have to import its model registry.
     ...(opts.model === undefined ? {} : { modelPattern: opts.model }),
+    ...(overlaySettings === undefined ? {} : { settings: overlaySettings }),
     sessionManager,
     // A private registry per session, never the process-global default: that
     // one admits only one "Main" identity per generation, so a second session
@@ -416,6 +455,13 @@ export interface CreateSessionOptions {
   role: SessionRole;
   releaseGrants?: ResolvedGrants;
   onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
+  /**
+   * Absolute path to the fleet-owned omp settings overlay (#537), forwarded to
+   * the child through the host spec so the far side's `createLocalSession`
+   * loads it via `Settings.init({ configFiles: [<path>] })`. Absent, nothing
+   * is staged and the session discovers settings as it does today.
+   */
+  ompSettingsFile?: string;
 
   /**
    * Where the control socket is bound. The daemon puts it beside the run's own
@@ -554,6 +600,7 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     ...(opts.releaseGrants === undefined ? {} : { releaseGrants: opts.releaseGrants }),
     ...(opts.verbSocketPath === undefined ? {} : { verbSocketPath: opts.verbSocketPath }),
     ...(opts.readOnly === undefined ? {} : { readOnly: opts.readOnly }),
+    ...(opts.ompSettingsFile === undefined ? {} : { ompSettingsFile: opts.ompSettingsFile }),
   };
 
   const log = opts.onChildLog ?? ((line: string) => process.stderr.write(`${line}\n`));
@@ -714,8 +761,24 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     pending.clear();
   };
 
+  // The one terminal event this proxy owns outright: the session process is
+  // gone, whether by crash, kill, or a clean dispose of its own. Delivered to
+  // subscribers *before* any failure is surfaced, so a supervisor learning of
+  // the death can also read its exit code off the same event. `disposing` is
+  // the proxy's own intent (its disposer asked the child to stop), not the
+  // subscriber's, so the event is emitted in both branches and each observer
+  // decides what an exit means from its own state.
+  const emitSessionExit = (code: number | null): void => {
+    const type = "session_exit";
+    const event = { type, code };
+    for (const cb of handlers.get(type) ?? []) cb(event);
+    for (const cb of handlers.get("*") ?? []) cb(event);
+  };
+
   void child.exited.then(async (code) => {
     onExit();
+    const exitCode = typeof code === "number" ? code : null;
+    emitSessionExit(exitCode);
     // A child that cannot start writes `start-error` over the socket, not the
     // pipes — and its exit can be dispatched before the accept of a connection
     // that already completed in the kernel. Let the socket settle before the