omp-conductor 0.10.0 → 0.12.0

package/src/types.ts

@@ -114,6 +114,14 @@ export interface RepoTarget {
    * the worker brief.
    */
   graphProject?: string;
+  /**
+   * Ordered-migration-chain guard (#227): repo-relative directory whose *.py
+   * files declare `revision`/`down_revision`. When set, `conductor_pr_merge`
+   * refuses a merge that would collide with the base branch's chain (id reuse,
+   * a deleted migration, or a combined graph with more than one head). Absent
+   * = no chain check.
+   */
+  migrations?: { dir: string };
 }
 
 /**
@@ -140,6 +148,59 @@ export type ReportScope = (typeof REPORT_SCOPES)[number];
  */
 export const DEFAULT_REPORT_SCOPE: ReportScope = "material";
 
+/**
+ * The four answers the wizard offers for "how loud should the orchestrator be".
+ * Three are legacy presets; `quiet` is the explicit form: interrupt only for
+ * escalations, fleet stops and confirmed failures, with one daily rollup
+ * (#229).
+ */
+export type ReportScopeChoice = ReportScope | "quiet";
+
+/**
+ * Which categories may interrupt the operator's phone. Each maps to a tier-2
+ * escalation channel: `tier2` (a plain decision), `decision-needed`, a
+ * fleet-stopping condition, a confirmed failure, or any material event. What an
+ * `interruptOn` list omits is not dropped — it accumulates for the digest.
+ * Data rather than booleans so the config validator, the escalator and the
+ * orchestrator prompt all enumerate the same vocabulary (#229).
+ */
+export const INTERRUPT_CATEGORIES = [
+  "tier2",
+  "decision-needed",
+  "fleet-stopped",
+  "confirmed-failure",
+  "material",
+] as const;
+
+export type InterruptCategory = (typeof INTERRUPT_CATEGORIES)[number];
+
+/** When the daily/report rollup is due. */
+export const DIGEST_CADENCES = ["none", "per-tick", "daily"] as const;
+
+export type DigestCadence = (typeof DIGEST_CADENCES)[number];
+
+/**
+ * Explicit reporting policy: which categories interrupt, and when the daily
+ * rollup happens. `scopePreset` is set when the config came from a legacy
+ * `scope:` preset, and drives the preset tick prompt text.
+ */
+export interface ReportingPolicy {
+  interruptOn: InterruptCategory[];
+  digest: { cadence: DigestCadence; at?: string; timezone?: string };
+  /** Set when the policy came from a legacy `scope` preset. */
+  scopePreset?: ReportScope;
+}
+
+/**
+ * The default policy: the legacy `material` preset, spelled out so an explicit
+ * config can name it and the wizard can show it.
+ */
+export const DEFAULT_REPORT_POLICY: ReportingPolicy = {
+  interruptOn: [...INTERRUPT_CATEGORIES],
+  digest: { cadence: "per-tick" },
+  scopePreset: DEFAULT_REPORT_SCOPE,
+};
+
 /**
  * The irreversible tool-call shapes this package recognises and gates. Data
  * here rather than in `release-policy.ts` because the config validator, the
@@ -431,6 +492,9 @@ export const LABEL_REASONS = [
   "promoted-to-queue",
   /** Re-briefed after a failed attempt, and put back on the queue. */
   "re-briefed",
+  /** Delivered: the PR merged and the issue closed, and the queue label comes
+   *  off as completed lifecycle cleanup — not superseded, not a re-brief (#230). */
+  "completed",
   /** Parked: it needs a decision only a human can make. */
   "needs-human",
   "duplicate",
@@ -534,11 +598,13 @@ export interface ProjectConfig {
    */
   policy?: ProjectPolicy;
   /**
-   * How loud the orchestrator is. Optional on disk — a config written before
-   * this key existed loads as {@link DEFAULT_REPORT_SCOPE} — so read it through
-   * `resolveReportScope` rather than reaching for `.scope` directly.
+   * How loud the orchestrator is, and when the daily rollup happens. Optional
+   * on disk — a config written before this key existed loads as
+   * {@link DEFAULT_REPORT_POLICY} — so read it through `resolveReportScope`
+   * rather than reaching for `.scope` directly. A legacy `scope:` value is
+   * accepted and materialised as a preset (`.scopePreset`).
    */
-  reporting?: { scope: ReportScope };
+  reporting?: ReportingPolicy;
   /** Parent directory for per-run worktrees. */
   workspaceRoot: string;
   /** Cache of bare clones, so N runs share one fetch instead of N. */
@@ -832,6 +898,7 @@ export const FAILURE_CLASSES = [
   "orphan-dirty",
   "settlement-stuck",
   "provider-credit",
+  "provider-transient",
   "unknown",
 ] as const;
 
@@ -969,6 +1036,9 @@ export interface DispatchSummary {
   ready: number;
   routed: number;
   admitted: number;
+  /** Ready issues carrying a state label — claimed by a live or settling run.
+   *  Optional: persisted old rows lack it, so readers use `?? 0` (#228). */
+  claimed?: number;
   /** True only for system/API failures, never ordinary policy holds. */
   degraded: boolean;
   holds: AdmissionHoldSummary[];
@@ -1223,6 +1293,15 @@ export interface Store {
   wasNotified(key: string): boolean;
   recordDispatch(project: string, summary: DispatchSummary): void;
   latestDispatch(project: string): DispatchSummary | undefined;
+  /** The newest `digest:` dedupe key a project has run toward without ending in
+   *  failure — what the off-schedule-digest refusal compares against (#229). */
+  lastDigestDedupeKey(project: string): string | undefined;
+  /** Hold a tier-2 escalation the interrupt policy deferred to the digest. */
+  addHeldNotice(notice: HeldNoticeDraft): void;
+  /** Held notices still owed (not yet re-surfaced by a digest pass). */
+  undigestedNotices(project: string): HeldNotice[];
+  /** Mark every undigested notice for a project as re-surfaced. */
+  markNoticesDigested(project: string, at: number): void;
   /** Add one bounded observation to the per-day friction rollup. */
   recordFriction(project: string, observation: FrictionObservation): void;
   /** Repeated signals not surfaced within the supplied cooldown window. */
@@ -1385,6 +1464,28 @@ export interface Escalation {
   summary: string;
   detail?: string;
   runId?: string;
+  /** Tier-2-only interrupt category; defaults to `"tier2"`. Drives whether the
+   *  escalation pages now or is held for the digest under the project's
+   *  `reporting.interruptOn` policy (#229). */
+  category?: InterruptCategory;
+}
+
+/** A tier-2 escalation the project's interrupt policy deferred to the digest. */
+export interface HeldNotice {
+  id: string;
+  category: InterruptCategory;
+  summary: string;
+  detail: string;
+  createdAt: number;
+}
+
+/** What it takes to persist one held notice. */
+export interface HeldNoticeDraft {
+  project: string;
+  category: InterruptCategory;
+  summary: string;
+  detail: string;
+  createdAt: number;
 }
 
 /**
@@ -1511,6 +1612,11 @@ export const VERB_REFUSALS = [
   "release-shape-not-executable",
   /** Config unreadable, project unknown, repo unrouted. Fail closed. */
   "config-unreadable",
+  /** Merging would corrupt the ordered migration chain in `migrations.dir` (#227). */
+  "chain-conflict",
+  /** The PR diff or the base-branch chain could not be read — fail closed rather
+   *  than merge blind into a chain that might be about to fork (#227). */
+  "chain-unreadable",
   /** The privileged half ran and the underlying command failed. */
   "action-failed",
 ] as const;

package/src/verbs/server.ts

@@ -42,6 +42,8 @@ import { randomUUID } from "node:crypto";
 import { createServer, type Server, type Socket } from "node:net";
 
 import { resolvePolicy, resolveReleaseGrants } from "../config.ts";
+import { chainEntriesFromDiff, chainViolations } from "../chain-check.ts";
+import type { readBaseChain as readBaseChainType } from "../gitops.ts";
 import { releaseRefusal } from "../release-policy.ts";
 import { LIVE_STATES } from "../store.ts";
 import type {
@@ -181,6 +183,12 @@ export interface VerbDeps {
   pausedAt: () => number | undefined;
   log: (message: string) => void;
   now: () => number;
+  /**
+   * Read-only access to the ordered-migration-chain guard. A read seam, kept
+   * off `VerbActions` (the mutation surface) on purpose: `prMergeVerb` consults
+   * whether the merge would fork the chain, it never changes it (#227).
+   */
+  chain: { readBaseChain: typeof readBaseChainType };
 }
 
 /**
@@ -728,6 +736,47 @@ async function prMergeVerb(
       );
     }
 
+    const repoTarget = Object.values(project.routing.repos).find((r) => r.name === target.repo);
+    const chainDir = repoTarget?.migrations?.dir;
+    if (chainDir !== undefined && repoTarget !== undefined) {
+      // Checked against the base tip at merge time, not against a stale advisory:
+      // the author hit two merged PRs leaving main with two Alembic heads (#227).
+      const diff = await deps.tracker.prDiff(prUrl).catch(() => undefined);
+      if (diff === undefined) {
+        return refuse(
+          "chain-unreadable",
+          `refused: the diff of ${prUrl} could not be read, so the migration chain in ${chainDir} cannot be verified. Failing closed: a wrong merge here is a silent production schema drift (#227).`,
+          target.issue,
+        );
+      }
+      const { added, changed, deleted } = chainEntriesFromDiff(diff.files, chainDir);
+      if (added.length + changed.length + deleted.length > 0) {
+        if (diff.truncated) {
+          return refuse(
+            "chain-unreadable",
+            `refused: the diff of ${prUrl} was truncated and it touches ${chainDir} — the chain cannot be verified from a partial diff.`,
+            target.issue,
+          );
+        }
+        const base = await deps.chain.readBaseChain(project, repoTarget, chainDir);
+        if (!base.ok) {
+          return refuse(
+            "chain-unreadable",
+            `refused: could not read ${chainDir} at the tip of ${repoTarget.defaultBranch} (${base.stderr}).`,
+            target.issue,
+          );
+        }
+        const violations = chainViolations({ base: base.entries, added, changed, deleted });
+        if (violations.length > 0) {
+          return refuse(
+            "chain-conflict",
+            `refused: merging ${prUrl} would corrupt the ordered migration chain in ${chainDir} — ${violations.join("; ")}. Rebase onto the current tip, regenerate the migration, and call again.`,
+            target.issue,
+          );
+        }
+      }
+    }
+
     const outcome = await deps.actions.mergePr(prUrl, headSha);
     if (!outcome.ok) {
       return refuse("action-failed", `refused: gh could not merge:\n${outcome.stderr}`, target.issue);

package/src/worker.ts

@@ -25,6 +25,11 @@ const STATE_LINE_PATTERN = /^state:\s*\S+\s*$/im;
 /** `{{KEY}}` placeholders in a brief template. */
 const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
 
+function scheduleWallClock(callback: () => void, delayMs: number): () => void {
+  const timer = setTimeout(callback, delayMs);
+  return () => clearTimeout(timer);
+}
+
 /**
  * Which ceiling stopped a run. Only ever set alongside `state: "killed"`: the
  * turn counter caught a loop, or the wall clock caught a session that was stuck
@@ -32,6 +37,24 @@ const PLACEHOLDER_PATTERN = /\{\{([A-Za-z0-9_]+)\}\}/g;
  */
 export type KilledBy = "turns" | "wallclock";
 
+export type WorkerPausePhase = "running" | "pausing" | "paused";
+
+export interface WorkerPauseControl {
+  phase(): WorkerPausePhase;
+  /** Cooperative park; resolves when the harness is idle. Rejects when the
+   * run is settling, cap-killed, or already pausing/paused. */
+  pause(): Promise<void>;
+  /** Returns a parked session to work with a continuation prompt. Throws
+   * when the phase is not `paused`. */
+  resume(): void;
+}
+
+/** What a resumed session is told. One literal so tests can pin it. */
+export const RESUME_PROMPT =
+  "The operator paused this session and has now resumed it. Continue exactly where you left off: " +
+  "re-check the outcome of your last action before repeating it, then keep working your original " +
+  "brief to the same report contract.";
+
 export interface WorkerOpts {
   brief: string;
   cwd: string;
@@ -87,15 +110,21 @@ export interface WorkerOpts {
    * would be a path that fails to open rather than an absence.
    */
   onSessionFile?: (path: string) => void;
+  /**
+   * Installs the run's operator pause controller the moment the session
+   * exists; absent = no pause surface (tests, one-shot callers).
+   */
+  onPauseControl?: (control: WorkerPauseControl) => void;
 }
 
 /**
- * The one collaborator worth injecting: starting a session is the only thing
- * `runWorker` does that needs a real harness. Defaulted, so production callers
- * never pass it and a test can hand over a fake without a live peer dependency.
+ * Session creation plus the clock seam needed to prove wall-clock behavior
+ * without sleeping. Production callers use the defaults.
  */
 export interface RunWorkerDeps {
   createSession: typeof createSession;
+  now?: () => number;
+  schedule?: (callback: () => void, delayMs: number) => () => void;
 }
 
 export interface WorkerResult {
@@ -203,6 +232,8 @@ export async function runWorker(
 ): Promise<WorkerResult> {
   const { workerWallClockMs } = o.caps;
   const maxTurns = o.maxTurns ?? (() => o.caps.workerMaxTurns);
+  const now = deps.now ?? Date.now;
+  const schedule = deps.schedule ?? scheduleWallClock;
 
   const session = await deps.createSession({
     cwd: o.cwd,
@@ -249,18 +280,32 @@ export async function runWorker(
   // merged (#217).
   let claim: { prUrl: string; headSha: string } | undefined;
   let killedBy: KilledBy | undefined;
-  // Bun's global timer handle; cleared on every exit path below.
-  let timer: Timer | undefined;
+  // Canceler for the armed wall clock; invoked on every exit path below.
+  let cancelWallClock: (() => void) | undefined;
+  let wallClockRemainingMs = workerWallClockMs;
+  let wallClockArmedAt = now();
+  let pausePhase: WorkerPausePhase = "running";
+  let resumeWaiter: PromiseWithResolvers<string> | undefined;
+  // A terminal agent_end or cap has settled the run.
+  let done = false;
   // Resolved by the first terminal `agent_end`, and by every cap kill. Only
   // ever awaited when the harness has already said it is not finished.
   const { promise: settled, resolve: settle } = Promise.withResolvers<void>();
+  // Resolves whenever a pause request must interrupt the existing
+  // non-terminal agent_end wait. Re-armed after its resume prompt is consumed.
+  let pauseRequested = Promise.withResolvers<void>();
   // Set by a non-terminal `agent_end`: the harness will resume this session.
   let resuming = false;
 
   const clearWallClock = () => {
-    if (timer === undefined) return;
-    clearTimeout(timer);
-    timer = undefined;
+    if (cancelWallClock === undefined) return;
+    cancelWallClock();
+    cancelWallClock = undefined;
+  };
+
+  const armWallClock = () => {
+    wallClockArmedAt = now();
+    cancelWallClock = schedule(() => kill("wallclock"), wallClockRemainingMs);
   };
 
   const kill = (by: KilledBy) => {
@@ -271,9 +316,80 @@ export async function runWorker(
     session.abort();
     // An aborted session may never reach a terminal `agent_end`. The cap is the
     // outcome now, so nothing may still be waiting for one.
+    done = true;
     settle();
   };
 
+  const takeResumePrompt = async (): Promise<string | undefined> => {
+    const waiter = resumeWaiter;
+    if (waiter === undefined) return undefined;
+    const prompt = await Promise.race([
+      waiter.promise,
+      settled.then(() => undefined),
+    ]);
+    if (resumeWaiter === waiter) {
+      resumeWaiter = undefined;
+      pauseRequested = Promise.withResolvers<void>();
+    }
+    return killedBy !== undefined || done ? undefined : prompt;
+  };
+
+  o.onPauseControl?.({
+    phase: () => pausePhase,
+    pause: async () => {
+      if (killedBy !== undefined || done) {
+        throw new Error("the run is settling; nothing left to pause");
+      }
+      if (pausePhase !== "running") throw new Error(`the worker is already ${pausePhase}`);
+      if (resumeWaiter !== undefined) {
+        throw new Error("the worker resume is still starting");
+      }
+      pausePhase = "pausing";
+      resumeWaiter = Promise.withResolvers<string>();
+      pauseRequested.resolve();
+      // Bank the remaining wall clock before the abort: a slow drain must not
+      // be cap-killed mid-park (#238 acceptance 3).
+      wallClockRemainingMs = Math.max(
+        1_000,
+        wallClockRemainingMs - (now() - wallClockArmedAt),
+      );
+      clearWallClock();
+      try {
+        await session.park();
+      } catch (err) {
+        pausePhase = "running";
+        if (killedBy !== undefined || done) {
+          resumeWaiter = undefined;
+          throw err;
+        }
+        // The abort may already have unwound prompt() even though park itself
+        // failed. Wake that path with the same defensive continuation rather
+        // than leaving the worker hung on an orphaned waiter.
+        armWallClock();
+        resumeWaiter?.resolve(RESUME_PROMPT);
+        throw err;
+      }
+      if (killedBy !== undefined || done) {
+        pausePhase = "running";
+        resumeWaiter = undefined;
+        throw new Error("the run settled while pausing");
+      }
+      pausePhase = "paused";
+    },
+    resume: () => {
+      if (pausePhase !== "paused") {
+        throw new Error(
+          pausePhase === "pausing"
+            ? "still pausing — wait until it reports paused"
+            : "the worker is not paused",
+        );
+      }
+      pausePhase = "running";
+      armWallClock();
+      resumeWaiter?.resolve(RESUME_PROMPT);
+    },
+  });
+
   session.on("turn_start", () => {
     // The documented watchdog signal, and the honest one: `turn_start` fires
     // exactly once per turn, whereas one turn can emit several assistant
@@ -326,6 +442,8 @@ export async function runWorker(
     // all — is a finished run.
     const isTerminal = field(event, "isTerminal");
     if (shouldComplete(typeof isTerminal === "boolean" ? { isTerminal } : {})) {
+      if (pausePhase !== "running") return;
+      done = true;
       settle();
       return;
     }
@@ -336,14 +454,33 @@ export async function runWorker(
   // callback does not drop the handle itself: every exit runs `clearWallClock()`
   // exactly once instead, and clearing an already-fired handle is a documented
   // no-op — cheaper than assuming a fired timer holds nothing.
-  timer = setTimeout(() => kill("wallclock"), workerWallClockMs);
+  armWallClock();
 
   try {
-    await session.prompt(o.brief);
-    // `prompt()` returning is not the end of the run once the harness has
-    // announced a resume: finishing here would hand back a truncated report as
-    // the final result. Wait for the terminal `agent_end`, or for a cap.
-    if (resuming && killedBy === undefined) await settled;
+    let next: string | undefined = o.brief;
+    while (next !== undefined) {
+      try {
+        await session.prompt(next);
+      } catch (cause) {
+        // Our own abort surfaces here on some paths: a cap kill (existing
+        // behavior) or an operator park (new). Anything else is a real failure.
+        if (killedBy === undefined && resumeWaiter === undefined) throw cause;
+      }
+      next = undefined;
+      if (killedBy !== undefined) break;
+      if (resumeWaiter === undefined && resuming && !done) {
+        // A non-terminal agent_end ordinarily waits for the harness to finish
+        // later. A pause request must wake that wait so resume can prompt the
+        // same session instead of hanging behind the old settlement promise.
+        await Promise.race([settled, pauseRequested.promise]);
+      }
+      if (killedBy !== undefined) break;
+      if (resumeWaiter !== undefined) {
+        next = await takeResumePrompt();
+        if (next === undefined) break;
+        continue;
+      }
+    }
   } catch (cause) {
     // Our own abort surfaces here on some paths; that is a kill, not a crash.
     if (killedBy === undefined) {
@@ -356,6 +493,7 @@ export async function runWorker(
       });
     }
   } finally {
+    done = true;
     // Runs on every exit, including the early return above: a live timer keeps
     // the dispatcher process alive long after the run it was guarding.
     clearWallClock();