omp-conductor 0.15.10 → 0.15.12

package/src/model-fallback.ts

@@ -0,0 +1,177 @@
+/**
+ * The model-failover chain (#286): which model the next attempt of one run
+ * chain dispatches on, and how the choice is worded so a merged branch built
+ * on a different model stays attributable months later.
+ *
+ * Pure and synchronous: the daemon reads the run rows from the store, hands
+ * them here, and does what the returned choice says. Nothing in this module
+ * reaches a store or a provider — the same split as `failure-class.ts`, kept
+ * here (and NOT there) because `failure-class.ts` owns the *verdict* while
+ * this owns the *response to a run of provider verdicts*.
+ *
+ * The trigger is deliberately counted from the store, never from the
+ * harness's `modelFallbackMessage`: that message means the harness already
+ * downgraded internally and is evidence about a run, not a decision; keying
+ * on it would fire the feature when nothing needed switching and stay silent
+ * when a provider returns 402 cleanly.
+ */
+
+import type { FailureClass, RunRecord } from "./types.ts";
+
+/**
+ * Failure classes that move a chain onto its next model — and ONLY these: a
+ * cap kill means the slice was too big, and switching models in response
+ * would hide a decomposition problem behind a provider one (#286, #490).
+ */
+export const FAILOVER_CLASSES: readonly FailureClass[] = [
+  "provider-transient",
+  "provider-credit",
+];
+
+/** The default for a project's `modelFallbackThreshold` when it is absent or
+ *  unusable. Two consecutive provider-class failures, then the next attempt
+ *  moves to the next model. */
+export const DEFAULT_MODEL_FALLBACK_THRESHOLD = 2;
+
+/** True for the classes {@link FAILOVER_CLASSES} names, so a non-provider
+ *  failure can never push a chain sideways. */
+export function isProviderFailureClass(cls: string | undefined): cls is FailureClass {
+  return cls !== undefined && (FAILOVER_CLASSES as readonly string[]).includes(cls);
+}
+
+/** Consecutive provider-class failures at the head of one run chain. */
+export interface ProviderFailureFacts {
+  /** How many terminal runs at the head of the chain failed as a provider. */
+  streak: number;
+  /** Those runs' classes, newest first — the set is either one class or a mix
+   *  of `provider-transient` and `provider-credit`. */
+  classes: FailureClass[];
+  /**
+   * The model the most recent failure dispatched on, when its row recorded
+   * one. Undefined when the rows predate the column or the project has no
+   * chain — the log then falls back to the configured primary, or to "harness
+   * default" for a project with no `workerModel`.
+   */
+  previousModel: string | undefined;
+}
+
+/**
+ * Counts the consecutive provider-class failures at the head of a run chain.
+ * `runs` is in store order (oldest first, per `Store.runsForIssue`); the walk
+ * starts at the newest row and stops at the first row that is not a
+ * provider-class failure — a success, a non-provider failure, an unclassified
+ * row, a live row — so "sticky per chain, never global" falls out: a fresh
+ * issue has no rows and starts on the primary, and one issue's bad luck can
+ * never move another issue's chain.
+ */
+export function providerFailureFacts(runs: readonly RunRecord[]): ProviderFailureFacts {
+  const facts: ProviderFailureFacts = { streak: 0, classes: [], previousModel: undefined };
+  for (let i = runs.length - 1; i >= 0; i--) {
+    const run = runs[i]!;
+    if (!isProviderFailureClass(run.failureClass)) break;
+    facts.streak += 1;
+    facts.classes.push(run.failureClass);
+    if (facts.previousModel === undefined && run.model !== undefined) {
+      facts.previousModel = run.model;
+    }
+  }
+  return facts;
+}
+
+/** The model the next attempt should dispatch on, and whether that is a
+ *  failover at all. */
+export interface ModelChoice {
+  /** Model to dispatch on; `undefined` leaves the harness default, exactly as
+   *  an unconfigured project dispatches today. */
+  model: string | undefined;
+  /** True when the choice came from the fallback chain — the failover has
+   *  fired and the tick report must say so. */
+  fallback: boolean;
+}
+
+/**
+ * Where one run chain's next attempt dispatches, given the chain configuration
+ * and the store-derived failure streak.
+ *
+ * With no chain (empty or absent `modelFallbacks`) the primary model is
+ * returned unchanged and `fallback` stays false — today's dispatch byte for
+ * byte. Once the streak reaches the threshold the chain advances one slot per
+ * extra failure, clamped to the last model: the chain is exhausted there, and
+ * the existing escalation path (the provider-transient strike cap) settles it,
+ * naming every model tried.
+ */
+export function resolveDispatchModel(args: {
+  workerModel: string | undefined;
+  modelFallbacks: readonly string[] | undefined;
+  threshold: number;
+  streak: number;
+}): ModelChoice {
+  const { workerModel, modelFallbacks, threshold, streak } = args;
+  if (modelFallbacks === undefined || modelFallbacks.length === 0) {
+    return { model: workerModel, fallback: false };
+  }
+  if (streak < threshold) return { model: workerModel, fallback: false };
+  const index = Math.min(streak - threshold, modelFallbacks.length - 1);
+  return { model: modelFallbacks[index] ?? workerModel, fallback: true };
+}
+
+/**
+ * The clause the tick report annexes to a failover dispatch:
+ * `on <fallback> after N provider-transient failures on <previous>`, the shape
+ * the issue asked for (`#123 attempt 3 on <fallback> after 2
+ * provider-transient failures on <primary>`). `undefined` when no failover
+ * fired, so the ordinary dispatch log line is untouched.
+ */
+export function fallbackClause(
+  choice: ModelChoice,
+  facts: ProviderFailureFacts,
+  workerModel: string | undefined,
+): string | undefined {
+  if (!choice.fallback) return undefined;
+  const classWord = new Set(facts.classes).size === 1 ? facts.classes[0]! : "provider";
+  const countWord = facts.streak === 1 ? "failure" : "failures";
+  const previous = facts.previousModel ?? workerModel ?? "harness default";
+  return `on ${choice.model} after ${facts.streak} ${classWord} ${countWord} on ${previous}`;
+}
+
+/** One recorded model dispatch in a chain: which model, on which attempt. */
+export interface ModelTried {
+  model: string;
+  attempt: number;
+}
+
+/** Every recorded model of a chain, in chain order — what the exhaustion
+ *  escalation names when it says "every model tried". Rows without a recorded
+ *  model (no chain configured, or written before the column existed) are
+ *  absent: an unnamed primary is the harness default, which the escalation
+ *  wording covers without pretending to know its name. */
+export function modelsTried(runs: readonly RunRecord[]): ModelTried[] {
+  const tried: ModelTried[] = [];
+  for (const run of runs) {
+    if (run.model === undefined) continue;
+    tried.push({ model: run.model, attempt: run.attempt });
+  }
+  return tried;
+}
+
+/**
+ * `"acme/a (attempts 1-2), acme/b (attempt 3)"` — consecutive attempts
+ * grouped under their model, in chain order.
+ */
+export function formatModelsTried(tried: readonly ModelTried[]): string {
+  const groups: { model: string; attempts: number[] }[] = [];
+  for (const { model, attempt } of tried) {
+    const last = groups[groups.length - 1];
+    if (last !== undefined && last.model === model) last.attempts.push(attempt);
+    else groups.push({ model, attempts: [attempt] });
+  }
+  return groups
+    .map(({ model, attempts }) => {
+      const span =
+        attempts.length === 1
+          ? `attempt ${attempts[0]}`
+          : `attempts ${attempts[0]}-${attempts[attempts.length - 1]}`;
+      return `${model} (${span})`;
+    })
+    .join(", ");
+}

package/src/notices.ts

@@ -0,0 +1,9 @@
+import { createHash } from "node:crypto";
+
+/** The deterministic primary id of a held notice derived from its escalation
+ * key. Sha256 of the key — 64 hex characters, a shape the digest handoff
+ * accepts verbatim — keeps the re-derived notice from a later poll idempotent
+ * against `ON CONFLICT(id) DO NOTHING` while remaining a real ledger row id. */
+export function heldNoticeId(key: string): string {
+  return createHash("sha256").update(`held-notice\0${key}`).digest("hex");
+}

package/src/omp.ts

@@ -19,14 +19,15 @@ import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 
 import { readOnlySession, worktreeConfinement } from "./confinement.ts";
-import { releasePolicyTripwire, type ReleaseBlockContext } from "./release-policy.ts";
+import { releasePolicyTripwire, type GateShape, type ReleaseBlockContext } from "./release-policy.ts";
 import type {
   HostToParent,
   ParentToHost,
   SessionHostSpec,
 } from "./session-host.ts";
 import { decodeFrames, encodeFrame } from "./session-host.ts";
-import type { ReleaseShape, ResolvedGrants, SessionRole } from "./types.ts";
+import type { ResolvedGrants, SessionRole } from "./types.ts";
+import { SESSION_ROLE_ENV } from "./types.ts";
 import { conductorVerbs } from "./verbs/client.ts";
 
 const OMP_PACKAGE = "@oh-my-pi/pi-coding-agent";
@@ -158,7 +159,7 @@ export async function createLocalSession(opts: {
   /** Install the release/deploy tool-call gate with these per-shape grants. */
   releaseGrants?: ResolvedGrants;
   /** Durable audit callback invoked only when that gate rejects a call. */
-  onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
+  onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
   /**
    * The conductor verb socket this session's mutation tools call (#126).
    *
@@ -414,7 +415,7 @@ export interface CreateSessionOptions {
   resume?: boolean;
   role: SessionRole;
   releaseGrants?: ResolvedGrants;
-  onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
+  onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
 
   /**
    * Where the control socket is bound. The daemon puts it beside the run's own
@@ -559,6 +560,11 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
   const argv = [process.execPath, opts.hostModule ?? SESSION_HOST, JSON.stringify(spec)];
   const child = Bun.spawn(argv, {
     cwd: opts.cwd,
+    // Stamp the session's own role on the child, so a process the agent runs —
+    // `omp-conductor report` from its sandbox — can tell a worker session from
+    // the operator's shell. Direct CLI runs outside a spawned session inherit
+    // nothing and stay the orchestrator surface.
+    env: opts.role === undefined ? undefined : { ...process.env, [SESSION_ROLE_ENV]: opts.role },
     stdin: "ignore",
     // The harness writes progress to stdout; both streams are the daemon's log,
     // never the protocol. The protocol has its own socket precisely so a chatty
@@ -617,6 +623,67 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     return stderrTail.trim();
   };
 
+  /**
+   * The child's own start failure, if it got one out over the socket before it
+   * died — the socket counterpart of the pipe drain.
+   *
+   * A child that cannot start can connect and write `start-error` in the same
+   * event-loop turn its exit is dispatched in: the connect already completed
+   * in the kernel, the accept is still queued, and closing the listener in
+   * that window drops the frame with the reason. So every exit route waits
+   * this out BEFORE {@link cleanup} closes the server, then prefers the frame
+   * over a bare exit code. Bounded like the pipes: a dead child that never
+   * connected has nothing on the wire, and 2s only ever delays an
+   * already-failed startup.
+   *
+   * Memoized: the pre-connect catch and the exit handler both arrive here, and
+   * both must read the SAME socket — two independent readers would race on the
+   * first `destroy()` and one of them would lose the frame.
+   */
+  let startErrorReading: Promise<string | undefined> | undefined;
+  const startErrorFromSocket = (): Promise<string | undefined> => {
+    startErrorReading ??= (async (): Promise<string | undefined> => {
+      const socket = await Promise.race([
+        attached.catch(() => undefined),
+        new Promise<undefined>((resolve) => {
+          setTimeout(() => resolve(undefined), DRAIN_GRACE_MS).unref?.();
+        }),
+      ]);
+      if (socket === undefined) return undefined;
+      try {
+        return await new Promise<string | undefined>((resolve) => {
+          let buffer = "";
+          let settled = false;
+          const finish = (message: string | undefined): void => {
+            if (settled) return;
+            settled = true;
+            clearTimeout(timer);
+            resolve(message);
+          };
+          const timer = setTimeout(() => finish(undefined), DRAIN_GRACE_MS);
+          timer.unref?.();
+          socket.on("data", (chunk: Buffer) => {
+            buffer += chunk.toString("utf8");
+            const { frames, rest } = decodeFrames(buffer);
+            buffer = rest;
+            for (const frame of frames) {
+              const message = frame as { t?: unknown; message?: unknown };
+              if (message.t === "start-error" && typeof message.message === "string") {
+                finish(message.message);
+                return;
+              }
+            }
+          });
+          socket.once("error", () => finish(undefined));
+          socket.once("close", () => finish(undefined));
+        });
+      } finally {
+        socket.destroy();
+      }
+    })();
+    return startErrorReading;
+  };
+
   const cleanup = (): void => {
     server.close();
     rmSync(socketPath, { force: true });
@@ -649,15 +716,21 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
 
   void child.exited.then(async (code) => {
     onExit();
+    // A child that cannot start writes `start-error` over the socket, not the
+    // pipes — and its exit can be dispatched before the accept of a connection
+    // that already completed in the kernel. Let the socket settle before the
+    // listener closes, so the child's own words survive the ordering.
+    if (disposing) {
+      cleanup();
+      return;
+    }
+    const startError = await startErrorFromSocket();
     cleanup();
-    // A child that exits during teardown exited because we asked it to. Only an
-    // exit while the session is supposed to be live is a failure — and it has
-    // to reach whoever is awaiting a prompt, or the dispatcher waits forever
-    // for a turn from a process that is gone.
-    if (disposing) return;
     // The tail first, so the message carries the child's own words.
     await settledTail();
-    fail(`omp-conductor session child exited ${String(code)} before the session ended`);
+    fail(
+      startError ?? `omp-conductor session child exited ${String(code)} before the session ended`,
+    );
   });
 
   let socket: Socket;
@@ -681,15 +754,20 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
     ]);
   } catch (err) {
     child.kill("SIGKILL");
+    // The child's exit can win the race above against the accept of a
+    // connection that completed in the kernel — then its `start-error` frame
+    // is still queued, and closing the listener now would throw it away.
+    // Read the socket first (bounded, like the pipes), and prefer its words
+    // over a bare exit code.
+    const startError = await startErrorFromSocket();
     cleanup();
     // Killed first, so the pipes are already closing, and only then read. This is
     // the route a child that dies before connecting takes — a missing peer
     // dependency, say — and reading `stderrTail` synchronously here raced the
     // drain loops and reported a bare exit code instead of the reason.
     const tail = await settledTail();
-    throw new Error(
-      `${err instanceof Error ? err.message : String(err)}${tail === "" ? "" : `\nchild output:\n${tail}`}`,
-    );
+    const reason = startError ?? (err instanceof Error ? err.message : String(err));
+    throw new Error(`${reason}${tail === "" ? "" : `\nchild output:\n${tail}`}`);
   }
 
   let buffer = "";
@@ -739,6 +817,8 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
             tool: message.tool,
             reason: message.reason,
             args: message.args,
+            ...(message.matched === undefined ? {} : { matched: message.matched }),
+            ...(message.invocation === undefined ? {} : { invocation: message.invocation }),
           });
           break;
       }