omp-conductor 0.18.2 → 0.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.18.2",
+  "version": "0.19.0",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/schema/config.schema.json

@@ -35,6 +35,18 @@
           ],
           "description": "Rolling-day spend ceiling; null means no spend gate (default 25)"
         },
+        "maxRunSpendUsd": {
+          "anyOf": [
+            {
+              "type": "number",
+              "exclusiveMinimum": 0
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "description": "Ceiling on one run, reserved from the daily budget before it launches; null derives it from dailySpendUsd. Must not exceed dailySpendUsd"
+        },
         "planUsage": {
           "anyOf": [
             {
@@ -272,6 +284,18 @@
                 ],
                 "description": "Rolling-day spend ceiling; null means no spend gate (default 25)"
               },
+              "maxRunSpendUsd": {
+                "anyOf": [
+                  {
+                    "type": "number",
+                    "exclusiveMinimum": 0
+                  },
+                  {
+                    "type": "null"
+                  }
+                ],
+                "description": "Ceiling on one run, reserved from the daily budget before it launches; null derives it from dailySpendUsd. Must not exceed dailySpendUsd"
+              },
               "planUsage": {
                 "anyOf": [
                   {
@@ -334,6 +358,8 @@
           "workerModel": {},
           "modelFallbacks": {},
           "modelFallbackThreshold": {},
+          "workerEscalationModel": {},
+          "requireOauthProviders": {},
           "ompSettings": {},
           "workerAdvisor": {},
           "escalation": {

package/src/admission.ts

@@ -16,6 +16,7 @@
 
 import { join } from "node:path";
 import { log, errText, safeEscalate } from "./log.ts";
+import { oauthFenceVerdict, type CredentialClassProbeResult } from "./credential-class.ts";
 import type {
   Caps,
   EffectiveModel,
@@ -35,6 +36,7 @@ import type {
 import { readPlanUsage, type PlanUsageStatus, type UsageSource } from "./usage.ts";
 import type { CriticalBaseProbe, CriticalBaseVerdict, LaneFile, LaneSource, RunLaneProbe } from "./gitops.ts";
 import { repoSlugFor } from "./gitops.ts";
+import { LIVE_STATES } from "./store.ts";
 import { branchName, type Routed } from "./routing.ts";
 import { parseDependsOn, buildDependencyCycles } from "./depends-on.ts";
 import type { DependencyCycle, DependencyNodeReader } from "./depends-on.ts";
@@ -79,6 +81,18 @@ export interface AdmissionDeps {
    * {@link Tracker.issueBody}. The daemon always wires it.
    */
   probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
+  /**
+   * Reads one provider's current credential class from the harness (#852).
+   *
+   * Injected because the production reader spawns a subprocess against the live
+   * credential store, and admission's tests must be able to drive every verdict
+   * — including "the probe could not answer" — without one. Unset, the fence
+   * cannot verify anything, so a project that declares
+   * `requireOauthProviders` is held rather than dispatched: the whole point is
+   * that unverified and wrong cost the same. A project that declares nothing
+   * never calls it.
+   */
+  probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
 }
 /** `stops` are the operational ends that each require one resume. */
 export function hasContinuationBudget(stops: number, maxContinuations: number): boolean {
@@ -93,6 +107,52 @@ export function hasFailedAttemptBudget(failures: number, maxAttempts: number): b
   return failures < maxAttempts;
 }
 
+/**
+ * Local midnight, matching how a human reads "today".
+ *
+ * ponytail: a rolling 24h window would be fairer to a run that started at
+ * 23:50, but midnight is what someone checking a morning spend report expects.
+ * Upgrade path is a `capWindow: "day" | "rolling24h"` config key.
+ *
+ * Lives here, with the spend gates that read it, and is imported by the
+ * composition root rather than the other way round: one definition of "today"
+ * for the fleet-stop check, the reservation gate and the status line, because
+ * three midnights would eventually disagree.
+ */
+export function startOfToday(): number {
+  const d = new Date();
+  d.setHours(0, 0, 0, 0);
+  return d.getTime();
+}
+
+/**
+ * What one run is allowed to spend, and therefore what admission reserves for
+ * it before launching anything (#851).
+ *
+ * `null` means unbounded: no daily cap and no explicit per-run ceiling, so
+ * there is nothing to reserve and nothing to enforce.
+ *
+ * An explicit `maxRunSpendUsd` always wins. Otherwise the allowance is derived
+ * as the daily cap divided by the worker cap — the largest per-run reservation
+ * that still lets the fleet run at its configured concurrency. Reserving the
+ * WHOLE daily cap per run would bound overshoot just as well, but it would
+ * also serialise every fleet on defaults (a $25 cap and two workers would only
+ * ever admit one run), turning a spend guard into a concurrency change nobody
+ * asked for. Dividing keeps both promises: the day's total is still bounded by
+ * the cap, because N reservations of cap/N are the cap.
+ *
+ * A zero worker cap has no runs to allow, and is guarded rather than divided
+ * by.
+ */
+export function runSpendAllowanceUsd(
+  caps: Pick<Caps, "dailySpendUsd" | "maxRunSpendUsd" | "maxConcurrentWorkers">,
+): number | null {
+  if (caps.maxRunSpendUsd !== null) return caps.maxRunSpendUsd;
+  if (caps.dailySpendUsd === null) return null;
+  const workers = Math.max(1, caps.maxConcurrentWorkers);
+  return caps.dailySpendUsd / workers;
+}
+
 /** A candidate cleared for dispatch, with the attempt number it will run as.
  *  `lane` is the effective file-lane declaration admission resolved for it
  *  (#608): the exact snapshot the overlap gate enforced. Dispatch renders this
@@ -437,13 +497,58 @@ export async function admitCandidates(
     holds.push({ issue, reason, ...(detail === undefined ? {} : { detail }) });
   };
 
+  // Runs holding an ACTIVE MUTATION LEASE (#899): the only rows whose files a
+  // candidate must not touch, because they are the only rows something is
+  // currently writing through.
+  //
+  // Deliberately narrower than `activeRuns`, and this is the whole of the fix.
+  // A `pushed-green` / `pushed-pending` row is worker-free by construction —
+  // its PR is live but no process owns its branch — and seeding lane occupancy
+  // from it meant a settled artifact held its entire diff forever: measured on
+  // this fleet, worker-free #782 / PR #870 held `omp/src/daemon.ts` and idled
+  // slots against unrelated candidates that merely overlapped it.
+  //
+  // What does NOT change: `activeRuns` still answers same-issue ownership, the
+  // sibling/parent occupancy and the open-PR gate, so releasing the *lane*
+  // never lets the artifact's own issue be re-implemented (the named silent
+  // fake of #899). Only file occupancy narrows.
+  //
+  // A dispatched, unsettled review revision is a lease too. Its run row is
+  // `running` the moment `claimRunForReview` fires, so it is normally already
+  // in `LIVE_STATES`; it is unioned here anyway because the claim and the
+  // launch are not the same instant, and a lane the gate briefly forgets is a
+  // lane two writers can share.
+  const leasedRuns = store.leasedRuns(project.name);
+
+  // One run's actual lane: the files it has touched relative to base, read from
+  // its worktree and/or its branch in the mirror. Fails OPEN by construction —
+  // an unreadable probe yields no files — because this side of the interlock is
+  // automatic and continuous, and a gate that refused well-formed work whenever
+  // a git read hiccuped would idle the fleet on a transient. (The deliberate
+  // counterpart is `conductor_pr_recover`, which is a single human-initiated
+  // call with a retryable refusal, and therefore fails closed — #925.)
+  const probeRunFiles = async (run: RunRecord): Promise<LaneFile[]> => {
+    if (d.probeWorktreeLane === undefined) return [];
+    const base = project.routing.repos[run.repo]?.defaultBranch ?? "main";
+    try {
+      return await d.probeWorktreeLane({
+        worktree: run.worktree,
+        baseRef: `refs/remotes/origin/${base}`,
+        ...(run.branch === "" ? {} : { branchRef: `refs/heads/${run.branch}` }),
+        mirror: join(project.mirrorRoot, `${run.repo}.git`),
+      });
+    } catch {
+      return [];
+    }
+  };
+
   // The file-lane interlock (#555): repo -> file -> the issue holding that file.
-  // Seeded from what a live run actually *occupies*, never from its issue body:
-  // the probed worktrees/branches of active runs (#288) unioned — since #744 —
-  // with the lane declaration admission persisted on the row at dispatch, which
-  // is the exact enforced snapshot, not a re-parse. The declared half is what
-  // survives across passes: a run admitted with a lane it has not yet written
-  // still occupies until it reaches a terminal state and leaves the active set,
+  // Seeded from what a leaseholder actually *occupies*, never from its issue
+  // body: the probed worktrees/branches of runs holding a mutation lease (#288,
+  // #899) unioned — since #744 — with the lane declaration admission persisted
+  // on the row at dispatch, which is the exact enforced snapshot, not a
+  // re-parse. The declared half is what survives across passes: a run admitted
+  // with a lane it has not yet written still occupies while it holds its lease,
   // and the declared lanes are extended as this pass admits candidates, so two
   // overlapping candidates cannot both clear the gate in one tick. Keyed by
   // repo because a path only collides within its own checkout: a `daemon.ts` on
@@ -466,31 +571,16 @@ export async function admitCandidates(
       }
       if (!perRepo.has(file)) perRepo.set(file, { issue: run.issue, source });
     };
-    for (const run of activeRuns) {
+    for (const run of leasedRuns) {
       // The probed actual half first, so the concrete read outnames the
       // declaration for a file the run has already written.
-      if (d.probeWorktreeLane !== undefined) {
-        const base = project.routing.repos[run.repo]?.defaultBranch ?? "main";
-        let files: LaneFile[];
-        try {
-          files = await d.probeWorktreeLane({
-            worktree: run.worktree,
-            baseRef: `refs/remotes/origin/${base}`,
-            ...(run.branch === "" ? {} : { branchRef: `refs/heads/${run.branch}` }),
-            mirror: join(project.mirrorRoot, `${run.repo}.git`),
-          });
-        } catch {
-          // Fail open, like an unreadable lane: the gate never refuses a
-          // well-formed issue because one probe could not be answered.
-          files = [];
-        }
-        for (const { file, source } of files) occupy(run, file, source);
-      }
+      for (const { file, source } of await probeRunFiles(run)) occupy(run, file, source);
       // The declared durable half (#744): the lane admission persisted at
       // dispatch. Occupies even before the run has written anything, on every
       // pass — the declaration lives on the row, not in a per-pass memo.
-      // Released by the active-set filter exactly like probed occupancy: a
-      // terminal run leaves `activeRuns` and its files stop occupying here.
+      // Released by the lease filter exactly like probed occupancy: a run that
+      // settles or is terminal stops holding a lease and its files stop
+      // occupying here (#899).
       if (run.lane !== undefined && run.lane.files.length > 0) {
         for (const file of run.lane.files) occupy(run, file, "declared");
       }
@@ -566,6 +656,36 @@ export async function admitCandidates(
   //
   // Placed after the cheap local busy-set read and before the first tracker
   // call, so a held fleet spends no GitHub API budget discovering it is held.
+  // The credential-class fence (#852), asked once per pass for the same reason
+  // the plan allowance is: it is a property of the fleet's credentials, not of
+  // any candidate, and its answer is identical for all of them. Placed
+  // immediately before the plan gate so both fleet-wide questions sit together,
+  // and so a fleet billing to the wrong credential spends no tracker budget
+  // discovering it is held.
+  //
+  // It clears by itself: the operator re-authenticates and the next pass passes.
+  // No `resume` is required and no escalation is raised here — a held candidate
+  // renders its reason in `status`, and inventing a page for a condition an
+  // operator fixes in a minute is how a guard costs more attention than it saves
+  // (the #110 reasoning, applied to the same shape of gate).
+  const requiredOauth = project.requireOauthProviders ?? [];
+  if (requiredOauth.length > 0 && routed.length > 0) {
+    const fence = await oauthFenceVerdict(
+      requiredOauth,
+      d.probeCredentialClass ??
+        (async (provider) => ({
+          ok: false,
+          reason: `no credential probe is wired for ${provider}`,
+        })),
+    );
+    if (!fence.ok) {
+      for (const r of routed) hold(r.issue.number, "credential-class", fence.reason);
+      log(`credential-class gate holding ${String(routed.length)} candidate(s): ${fence.reason}`);
+      store.reconcileGrooming(project.name, holds);
+      return { admitted: [], holds };
+    }
+  }
+
   const plan = await readPlanUsage(caps.planUsage, d.usage);
   if (plan.blocking) {
     for (const r of routed) hold(r.issue.number, "plan-usage-cap");
@@ -682,6 +802,23 @@ export async function admitCandidates(
     });
   }
 
+  // The spend reservation (#851). Read once per pass: what today has already
+  // cost, plus what every live run has committed but not yet spent. The daily
+  // cap on its own is a post-spend stop — it compares spend-to-date at
+  // admission and pauses the fleet on the NEXT pass — so one expensive run
+  // could cross it by its entire cost before anything objected. On 2026-08-21
+  // one `@slow` run spent an estimated $25.16 against a $25.00 cap and the
+  // fleet paused only after it had settled.
+  const runAllowanceUsd = runSpendAllowanceUsd(caps);
+  const dailyCapUsd = caps.dailySpendUsd;
+  const budgeted = runAllowanceUsd !== null && dailyCapUsd !== null;
+  const spentTodayUsd = budgeted ? store.spendSince(project.name, startOfToday()) : 0;
+  const liveReservedUsd = budgeted ? store.reservedSpendUsd(project.name) : 0;
+  // Candidates admitted earlier in THIS pass have no run row yet (the claim
+  // happens in dispatch), so the pass counts its own reservations or it would
+  // admit a whole slot's worth of runs against one slot's budget.
+  let passReservedUsd = 0;
+
   for (const r of routed) {
     const issue = r.issue.number;
     const cyclePath = memberCycle.get(issue);
@@ -718,6 +855,24 @@ export async function admitCandidates(
       continue;
     }
 
+    // Reserve before launching, never after settling: the whole allowance is
+    // taken out of the day's budget up front, so the worst case is bounded by
+    // what was reserved rather than by what a run turns out to cost. A run
+    // that settles for less releases the remainder simply by leaving the live
+    // set, which is why the next pass can admit again without any release
+    // bookkeeping. Distinct from the fleet-stopping `daily-spend-cap`: this
+    // holds one candidate while admitted work is still running.
+    if (budgeted && spentTodayUsd + liveReservedUsd + passReservedUsd + runAllowanceUsd > dailyCapUsd) {
+      const remaining = dailyCapUsd - spentTodayUsd - liveReservedUsd - passReservedUsd;
+      const detail =
+        `reserving $${runAllowanceUsd.toFixed(2)} for this run would exceed today's $${dailyCapUsd.toFixed(2)} ` +
+        `budget: $${spentTodayUsd.toFixed(2)} spent, $${(liveReservedUsd + passReservedUsd).toFixed(2)} reserved by ` +
+        `runs in flight, $${Math.max(0, remaining).toFixed(2)} left`;
+      hold(issue, "spend-reservation", detail);
+      log(`#${issue} held (spend-reservation): ${detail}`);
+      continue;
+    }
+
     const priorRuns = store.attemptsFor(project.name, issue);
     const failures = store.failuresFor(project.name, issue);
     if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) {
@@ -952,6 +1107,36 @@ export async function admitCandidates(
         log(`#${issue} skipped: open PR ${closer.url} already closes it`);
         continue;
       }
+      // The continuation reacquire (#925). A continuation resumes writing the
+      // retained branch, so the files that matter are the ones that branch
+      // ACTUALLY carries — not the lane its issue declares. The declared gate
+      // below cannot stand in for this: a continuation's declaration may be
+      // narrower than its predecessor's diff (or absent entirely), and the
+      // whole point of #899 is that the retained branch's files were released
+      // the moment its worker finished, so an unrelated worker may legitimately
+      // hold them by the time the continuation is admitted.
+      //
+      // Its own predecessor can never block it: a retained row is terminal or
+      // `pushed-*`, so it holds no lease, and the holder check ignores this
+      // issue's own rows in any case.
+      if (retained !== undefined) {
+        const carried = await probeRunFiles(retained);
+        if (carried.length > 0) {
+          const occupied = await ensureLaneOccupancy();
+          const perRepo = occupied.get(r.repo.name);
+          const collision = carried
+            .map(({ file }) => ({ file, holder: perRepo?.get(file) }))
+            .find(({ holder }) => holder !== undefined && holder.issue !== issue);
+          if (collision !== undefined) {
+            const detail =
+              `${collision.file} held by run #${collision.holder!.issue} (${collision.holder!.source})` +
+              ` — carried by retained branch ${retained.branch}`;
+            hold(issue, "file-lane", detail);
+            log(`#${issue} held (file-lane): ${detail}`);
+            continue;
+          }
+        }
+      }
       log(`#${issue} continuing retained PR ${closer.url} ${resume}`);
     }
 
@@ -1161,6 +1346,7 @@ export async function admitCandidates(
     }
 
     admitted.push({ r, attempt: priorRuns + 1, lane, model: await modelFor(r) });
+    if (budgeted) passReservedUsd += runAllowanceUsd;
     liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
     // Same-pass sibling occupancy for the file-lane gate: once admitted, a
     // candidate's declared lane occupies for the rest of the pass, so a later