omp-conductor 0.15.13 → 0.16.1

package/src/types.ts

@@ -657,6 +657,26 @@ export interface ProjectConfig {
    * when absent or unusable.
    */
   modelFallbackThreshold?: number;
+  /**
+   * An opaque omp settings map layered into every worker session this project
+   * dispatches (#537). At dispatch it is materialised verbatim to a fleet-owned
+   * YAML overlay under the run's session directory — never inside the worktree,
+   * whose diff is a PR — and passed to the session as an omp settings overlay
+   * (`Settings.init({ configFiles: [<path>] })`), so it layers on top of both
+   * the daemon account's global config and any project `<cwd>/.omp/config.yml`
+   * without erasing either.
+   *
+   * Conductor validates YAML shape only: a non-mapping is dropped at load, and
+   * everything inside the map is omp's schema to own — an unknown key is omp's
+   * to reject, never conductor's to understand. A config edit takes effect on
+   * the next dispatch (the overlay is rewritten on every attempt), and a
+   * project without the field dispatches byte-for-byte as it always has.
+   *
+   * The retry keys derived from {@link modelFallbacks} are merged into the
+   * effective overlay (unless the map already names a `retry` mapping), which
+   * is where #539's staging lives.
+   */
+  ompSettings?: Record<string, unknown>;
   /**
    * How a stuck run reaches a human, what to do when it cannot, and who runs
    * the session that triages it. See {@link ORCHESTRATOR_MODES}.
@@ -751,6 +771,10 @@ export interface ConductorConfig {
   version: typeof CONFIG_VERSION;
   defaults: Caps;
   projects: ProjectConfig[];
+  /** Absolute directory for restorable `conductor.db` snapshots; defaults to
+   *  `<stateDir()>/backups/db` when omitted. A snapshot that cannot land there
+   *  is `doctor`'s `db-backup` failure. */
+  dbBackupDir?: string;
 }
 
 /**
@@ -1102,6 +1126,11 @@ export const FAILURE_CLASSES = [
   "settlement-stuck",
   "provider-credit",
   "provider-transient",
+  /** A run drowned in in-session provider rate limits (repeated HTTP 429s) and
+   *  never got clear of the throttle before the cap swallowed it. Distinct from
+   *  `provider-credit` (402) and `provider-transient` (a single stream fault)
+   *  because its remedy is different (#573). */
+  "provider-capacity",
   /** A reviewer closed pushed-green or pushed-pending work without merging it:
    *  a review decision, not a worker failure. */
   "returned-for-revision",
@@ -1151,6 +1180,30 @@ export interface BaseHealth {
   checkedAt: number;
 }
 
+/**
+ * A per-repo merge freeze, set when a watched merge (or live base observation)
+ * turns the base red and lifted mechanically on green or by the operator's
+ * `unfreeze` verb. While `clearedAt` is absent the repo is frozen: `prMergeVerb`
+ * refuses further merges to it with the `base-red-freeze` refusal naming the
+ * suspected culprit. It is repo-scoped, never global, so sibling repos keep
+ * merging while one base is broken.
+ */
+export interface BaseFreeze {
+  repo: string;
+  /** The merged SHA suspected of breaking the base. */
+  culpritSha: string;
+  /** Failing workflow/run evidence naming the break. */
+  detail?: string;
+  /** When the freeze was set (or last refreshed). */
+  setAt: number;
+  /** When the freeze was lifted — absent means the repo is actively frozen. */
+  clearedAt?: number;
+  /** Who lifted it: `daemon` for mechanical recovery, else the operator. */
+  clearedBy?: string;
+  /** Why it was lifted: `base-green` or the operator's override reason. */
+  clearedReason?: string;
+}
+
 /**
  * Execution state is separate from the tracker's own labels on purpose: labels
  * are coarse and human-editable, while the loop needs to distinguish "pushed
@@ -1194,6 +1247,39 @@ export interface RunRecord {
   /** Effective turn ceiling for this run; operators may only raise it. */
   maxTurns: number;
   spendUsd: number;
+  /** How many in-session HTTP 429 responses the harness retried before the run
+   *  ended, counted by the worker from the session's `message_end` events while
+   *  the run was live (#573). Absent means the column predates the count or no
+   *  worker reported one — never "no rate limit happened": a healthy run
+   *  records 0. The classifier only reads it as "sustained" above a threshold,
+   *  so a single retried 429 is noise. */
+  provider429Count?: number;
+  /**
+   * The model that actually wrote this run's messages, read from the newest
+   * assistant `message_end`'s `AssistantMessage.model` (#539, #535 slice 1).
+   * Distinct from {@link RunRecord.model} (what the daemon dispatched on):
+   * this is what the harness resolved and actually used, present even for a run
+   * that never failed over, so "which model wrote this" is answerable without
+   * reading a transcript. Absent means the run predates the column or recorded
+   * no message carrying a model.
+   */
+  resolvedModel?: string;
+  /** The provider that wrote them, from the same message. */
+  resolvedProvider?: string;
+  /**
+   * Every within-run model fallback the harness applied (`retry_fallback_applied`),
+   * newest first. The `to` target is what a settlement report names when a run
+   * swapped providers mid-run instead of dying on a throttled primary (#539).
+   */
+  retryFallbacks?: { from: string; to: string }[];
+  /** `retry_fallback_succeeded` events: within-run fallbacks the harness recovered on. */
+  retryFallbackSucceeded?: number;
+  /** Assistant messages whose `retryRecovery.recovery === "model"` (#539). */
+  modelRecoveries?: number;
+  /** `auto_retry_start` events: in-session provider retries the harness ran. */
+  autoRetryCount?: number;
+  /** `auto_compaction_start` events: in-session context compactions. */
+  autoCompactionCount?: number;
   /** omp session transcript, so a human can read what the worker actually did. */
   sessionFile?: string;
   prUrl?: string;
@@ -1288,6 +1374,8 @@ export type AdmissionHoldReason =
   | "plan-usage-cap"
   | "shutting-down"
   | "stale-base"
+  | "file-lane"
+  | "depends-on"
   | "unroutable:no-repo-label"
   | "unroutable:multiple-repo-labels"
   | "unroutable:unknown-repo";
@@ -1298,6 +1386,14 @@ export interface AdmissionHoldSummary {
   count: number;
   /** Queue-order sample, capped before persistence and rendering. */
   issues: number[];
+  /**
+   * Per-issue detail, aligned by index with {@link AdmissionHoldSummary.issues}
+   * and capped the same way. The file-lane interlock (#555) fills it so an
+   * operator reading status or the digest can see *which file* blocked the
+   * held issue and *which running issue* holds it — the same question the
+   * plain `reason` group answers only with a count.
+   */
+  details?: string[];
 }
 
 /** Persisted outcome of the latest completed dispatch tick. */
@@ -1437,9 +1533,29 @@ export interface ReportRecord {
   updatedAt: number;
   /** Backoff gate: a `pending` row is due only once this has passed. */
   nextAttemptAt: number;
-  /** Telegram's own id for the delivered message, read out of the response
-   *  body rather than inferred from the HTTP status. */
+/** Telegram's own id for the delivered message, read out of the response
+   *  body rather than inferred from the HTTP status. A report split into
+   *  several messages records the first part's id here; {@link messageIds}
+   *  carries every part. */
   messageId?: number;
+  /** Every Telegram message id a split delivery accepted, in order. Absent
+   *  for rows delivered before multi-part sends existed (#566), and whenever
+   *  Telegram answered without an id — `ok: true` is the verdict, the id is a
+   *  courtesy. */
+  messageIds?: number[];
+  /**
+   * Parts of the current delivery contract already accepted, persisted as the
+   * retry's resume point. The split is a pure function of the message text, so
+   * a count is the whole state: a retry whose message is byte-identical to the
+   * attempt that wrote this watermark (see {@link sentPartsHash}) starts at
+   * part `sentParts` instead of re-sending confirmed parts (#566).
+   */
+  sentParts?: number;
+  /** SHA-256 of the message text the `sentParts` watermark was written under.
+   * Progress is only valid while the text is unchanged — a report flagged
+   * ambiguous re-renders with a possible-repeat banner, which reshapes the
+   * split, so the watermark must not carry across it. */
+  sentPartsHash?: string;
   deliveredAt?: number;
   /** Bounded text of the last known failure, surfaced verbatim by `status`. */
   lastError?: string;
@@ -1555,10 +1671,22 @@ export type DecisionState = (typeof DECISION_STATES)[number];
  */
 export const DECISION_TTL_MS = 7 * 24 * 60 * 60_000;
 
+/** What a pending decision row is owed — answered by a human, or a condition
+ *  the orchestrator set for itself with no human in the loop. The split is a
+ *  stored column, never inferred from the wording or from whether a condition
+ *  is attached: a real question may legitimately carry a condition ("ask me
+ *  once this PR merges"), and a watch is defined by having no human, not by
+ *  having one. */
+export type DecisionKind = "question" | "watch";
+
 /** One question put to the operator, and its answer if it has one. */
 export interface DecisionRecord {
   id: string;
   project: string;
+  /** Whether this row is a question a human must answer or a watch the
+   *  orchestrator set for itself. A watch turns up under its own heading and
+   *  is never offered to the operator to resolve. */
+  kind: DecisionKind;
   /** The question verbatim, as it was sent. Re-asking must not reword it. */
   question: string;
   /** What is waiting on the answer — an issue, a release, a PR. Free text,
@@ -1583,6 +1711,11 @@ export interface DecisionRecord {
 export interface DecisionDraft {
   project: string;
   question: string;
+  /** `"watch"` marks a row the orchestrator set for itself. Absent means a
+   *  question a human must answer — the default, so every existing caller
+   *  (`message` with a non-material category, `decision open`) stays a
+   *  question without remembering to say so. */
+  kind?: DecisionKind;
   blocks?: string;
   condition?: string;
   at: number;
@@ -1642,6 +1775,35 @@ export interface Store {
     project: string,
     sinceEpochMs: number,
   ): { repo: string; baseRef?: string }[];
+  /** The per-repo base-red freeze row, active or cleared, for one routed repo. */
+  baseFreeze(project: string, repo: string): BaseFreeze | undefined;
+  /** Every base-red freeze row for a project, active first, for `status`. */
+  freezes(project: string): BaseFreeze[];
+  /**
+   * Set or refresh an *active* base-red freeze for one repo, naming the
+   * suspected culprit merge and the failing evidence. Refreshing an
+   * already-active freeze keeps the repo frozen while the same head is still
+   * red. Returns true only when it *newly* froze the repo — a transition worth
+   * surfacing as a material event — and false for a pure refresh so the queue
+   * digest is not flooded by per-tick re-observation of a still-red base.
+   */
+  setBaseFreeze(
+    project: string,
+    freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
+  ): boolean;
+  /**
+   * Lift an active freeze — the mechanical green recovery or the operator's
+   * `unfreeze`. Records who lifted it and the ledger reason. Returns true only
+   * when an active freeze was actually lifted (so a recovery/override event is
+   * emitted once, never on an already-clear row).
+   */
+  clearBaseFreeze(
+    project: string,
+    repo: string,
+    by: string,
+    reason: string,
+    at?: number,
+  ): boolean;
   /** Newest attempt per issue that preserved work or failed to, so `status`
    *  can name every WIP tip a re-claim would build on and every tree that is
    *  still the only copy. */
@@ -1808,8 +1970,21 @@ export interface Store {
    *  means somebody else claimed it first — the guard against the same report
    *  being concurrently in flight twice. */
   claimReport(id: string, attemptId: string, at: number): ReportRecord | undefined;
-  /** Record the message id Telegram returned, and the moment it did. */
-  markReportDelivered(id: string, attemptId: string, messageId: number | undefined, at: number): boolean;
+  /** Record the message ids Telegram returned for every accepted part, and
+   *  the moment the last one landed. `messageId` (the first part's id) is
+   *  derived here, so the two can never disagree. */
+  markReportDelivered(id: string, attemptId: string, messageIds: readonly number[], at: number): boolean;
+  /** Record progress mid-split: the first `sentParts` parts of the delivery
+   *  contract fingerprinted by `sentPartsHash` are confirmed accepted. Written
+   *  before the next part ships, so a crash in the window can only ever
+   *  re-send the part that was in flight, never confirmed ones. */
+  markReportPartsSent(
+    id: string,
+    attemptId: string,
+    sentParts: number,
+    sentPartsHash: string,
+    at: number,
+  ): boolean;
   /** The attempt ended without an answer: Telegram may hold the message. The
    *  row *stays* `sending`, because that state means exactly "outcome unknown",
    *  and is flagged {@link ReportRecord.ambiguous} so the retry that stale
@@ -2182,6 +2357,14 @@ export const VERB_REFUSALS = [
   "checks-not-green",
   /** Another merge is in flight for this project. */
   "merge-in-flight",
+  /**
+   * The routed repository's base branch is frozen because a watched merge (or
+   * live base observation) turned it red — a base-red-freeze. Further merges to
+   * that repo stay refused until the base is observed green again or the
+   * operator lifts the freeze with `omp-conductor unfreeze <repo>`; sibling
+   * repos are unaffected.
+   */
+  "base-red-freeze",
   /** The label is not in this project's own vocabulary. */
   "label-not-in-vocabulary",
   /** The label is a lifecycle label; those transitions stay the daemon's (#26). */

package/src/unblock.ts

@@ -35,7 +35,7 @@
  * failed-attempt budget.
  */
 
-import { hasContinuationBudget, hasFailedAttemptBudget } from "./daemon.ts";
+import { hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
 import { projectLabels } from "./label-projection.ts";
 import { LIVE_STATES } from "./store.ts";
 import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";

package/src/upgrade-verify.ts

@@ -26,7 +26,7 @@ import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
 import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
 import type { DoctorReport } from "./doctor.ts";
-import type { FleetLayers } from "./fleet.ts";
+import type { FleetLayers } from "./status-render.ts";
 import type { ReportDraft, ReportKind } from "./types.ts";
 
 /** One command run to completion with captured output. The transient unit, the

package/src/upgrade.ts

@@ -8,9 +8,8 @@ import {
   LEGACY_HERDR_SESSION_HINT,
   resolveHerdrSession,
   telegramStateDir,
-  type DispatchLayer,
-  type FleetLayers,
 } from "./fleet.ts";
+import type { DispatchLayer, FleetLayers } from "./status-render.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
 import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";

package/src/verbs/server.ts

@@ -884,6 +884,31 @@ async function prMergeVerb(
     if (paused !== undefined) return paused;
   }
 
+  // Base-red-freeze gate (#283): a repo whose base is frozen must not accept
+  // another merge, no matter how green this PR's own checks look. Deliberately
+  // placed here — after target-repo identity resolution, before the merge lock
+  // and every `gh` call — so the refusal is a verb-level assertion, not a
+  // daemon-side flag the merge path could miss. Only the frozen repo is gated:
+  // sibling repos keep merging. The freeze lifts mechanically on a green base
+  // re-observation or via the operator's sanctioned `omp-conductor unfreeze
+  // <repo>` — never by label surgery or a DB edit.
+  const freezeRepo = target === undefined ? routedRepo?.name : target.repo;
+  if (freezeRepo !== undefined) {
+    const freeze = deps.store.baseFreeze(project.name, freezeRepo);
+    if (freeze !== undefined && freeze.clearedAt === undefined) {
+      return refuse(
+        "base-red-freeze",
+        `refused: merges to ${freezeRepo} are frozen because its base is red at ` +
+          `${freeze.culpritSha.slice(0, 8)} (base-red-freeze). ${freeze.detail ?? ""} ` +
+          "This is not a merge-lock or check verdict: the base that merges land on is broken, so " +
+          "`conductor_pr_merge` for this repo is refused until the base is observed green again or the " +
+          `operator lifts the freeze with \`omp-conductor unfreeze ${freezeRepo}\`. The override is the sanctioned ` +
+          "path — never work around this refusal with label surgery or a DB edit.",
+        issue,
+      );
+    }
+  }
+
   // Taken before any network call, so two concurrent callers contend here
   // rather than both spending a `gh` round trip and racing at the merge.
   const holderId = randomUUID();