omp-conductor 0.15.11 → 0.15.13

package/src/types.ts

@@ -336,6 +336,16 @@ export const SESSION_ROLES = ["worker", "orchestrator"] as const;
 
 export type SessionRole = (typeof SESSION_ROLES)[number];
 
+/**
+ * Env var the daemon sets on a spawned session so the reporting CLI can tell a
+ * worker session from the operator's shell. The daemon stamps it on the
+ * session-host child (and thus on every tool process it spawns); a direct CLI
+ * run — the operator's shell or a systemd unit — has it absent and is treated
+ * as the orchestrator, preserving the historical surface. {@link SESSION_ROLES} is
+ * the closed vocabulary: a value outside it must be refused, never assumed.
+ */
+export const SESSION_ROLE_ENV = "OMP_CONDUCTOR_SESSION_ROLE";
+
 /**
  * Who holds an authority the daemon itself never exercises. Declared as data
  * for the same reason as {@link REPORT_SCOPES}: the validator, the wizard and
@@ -632,6 +642,21 @@ export interface ProjectConfig {
    * answered the question wants.
    */
   workerModel?: string;
+  /**
+   * Ordered fallback models for this project, tried after {@link workerModel}
+   * once a run chain has suffered enough consecutive provider-class failures
+   * (stream stalls and credit refusals). Each new issue still starts on the
+   * primary; recovery is sticky to the chain, never global (#286). Empty or
+   * absent preserves the pre-failover dispatch exactly: every attempt stays on
+   * the primary until the existing escalation path takes over.
+   */
+  modelFallbacks?: string[];
+  /**
+   * Consecutive provider-class failures on one run chain after which the next
+   * dispatch moves to the next model in {@link modelFallbacks}. Defaults to 2
+   * when absent or unusable.
+   */
+  modelFallbackThreshold?: number;
   /**
    * How a stuck run reaches a human, what to do when it cannot, and who runs
    * the session that triages it. See {@link ORCHESTRATOR_MODES}.
@@ -688,6 +713,20 @@ export interface ProjectConfig {
   workspaceRoot: string;
   /** Cache of bare clones, so N runs share one fetch instead of N. */
   mirrorRoot: string;
+  /**
+   * Git treeish (commit SHAs or refs) on the base branch that every preserved
+   * continuation branch must contain before the dispatcher may reattach it
+   * (#428). A base safety fix protects only branches forked after it landed; a
+   * continuation forked before it still carries the dangerous code and, on a
+   * shared host, re-running the lifecycle suite it retains is what SIGTERMed
+   * the production daemon. With a marker configured, the dispatcher refuses to
+   * reattach any preserved branch missing it, holds the issue as `stale-base`
+   * (distinct from capacity or dependency holds), and never silently merges
+   * base into the user's work. Absent or empty, ordinary stale continuations
+   * keep today's behaviour. Optional and hand-edited, like
+   * {@link recoveryMerges}.
+   */
+  criticalBase?: string[];
 }
 
 /**
@@ -728,6 +767,16 @@ export interface ReadyIssue {
   updatedAt: string;
 }
 
+/**
+ * One issue comment as the worker brief renders it: author login and verbatim
+ * body, in the tracker's own order (oldest first), so a later correction
+ * visibly supersedes an earlier note.
+ */
+export interface IssueComment {
+  author: string;
+  body: string;
+}
+
 /**
  * How a pull request ended, in tracker-agnostic terms. Lowercase because the
  * loop's vocabulary is lowercase; mapping GitHub's `MERGED`/`CLOSED`/`OPEN` onto
@@ -879,6 +928,16 @@ export interface Tracker {
    * Complete — follows pagination to the end.
    */
   listOpenIssues(): Promise<ReadyIssue[]>;
+  /**
+   * The issue's comments, oldest first. The dispatcher renders them into the
+   * worker brief, so grooming the orchestrator posted as a comment reaches the
+   * worker's opening prompt without any runtime read.
+   *
+   * Throws when the tracker could not be read: the caller must never mistake
+   * "could not read" for "no comments", which is exactly the failure mode this
+   * read exists to prevent (#517).
+   */
+  listComments(issue: number): Promise<IssueComment[]>;
   addLabel(issue: number, label: string): Promise<void>;
   removeLabel(issue: number, label: string): Promise<void>;
   comment(issue: number, body: string): Promise<void>;
@@ -1154,6 +1213,16 @@ export interface RunRecord {
    *  copy of real work, so the tree was kept and the issue is held out of
    *  dispatch until an operator acknowledges it. */
   salvageError?: string;
+  /**
+   * The model this attempt actually dispatched on, written at dispatch when the
+   * project configures a failover chain (`modelFallbacks`). Absent means the
+   * run predates the column or the project has no chain — never "the harness
+   * downgraded"; the harness's own downgrade is carried as
+   * `modelFallbackMessage` on the worker result instead, because that is
+   * evidence about this run while this column is attribution for the chain
+   * (#286).
+   */
+  model?: string;
   /** When an operator accepted the loss or recovered the tree by hand
    *  (`unblock --force`). Clears the hold without erasing what happened. */
   salvageAckAt?: number;
@@ -1218,6 +1287,7 @@ export type AdmissionHoldReason =
   | "daily-spend-cap"
   | "plan-usage-cap"
   | "shutting-down"
+  | "stale-base"
   | "unroutable:no-repo-label"
   | "unroutable:multiple-repo-labels"
   | "unroutable:unknown-repo";
@@ -1258,6 +1328,7 @@ export type FrictionAdmissionReason =
   | "parent-lookup-error"
   | "issue-state-lookup-error"
   | "open-pr-lookup-error"
+  | "stale-base"
   | "unroutable:no-repo-label"
   | "unroutable:multiple-repo-labels"
   | "unroutable:unknown-repo";
@@ -1665,6 +1736,25 @@ export interface Store {
   /** Start the cooldown only after a tick carrying these signals was sent. */
   markFrictionSurfaced(project: string, kinds: readonly FrictionKind[], at: number): void;
   markNotified(key: string): void;
+  /** The live embedded-orchestrator-down incident for a project, if any. */
+  orchestratorIncident(project: string): OrchestratorIncident | undefined;
+  /** Record a new incident. `false` when one is already open for the project —
+   *  dedupe so a flapping orchestrator opens (and pages) once per incident. */
+  openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean;
+  /** Count tier-1 escalations diverted to an issue comment. No-op when no
+   *  incident is open, so a healthy or external orchestrator never accumulates. */
+  bumpOrchestratorDiverted(project: string, by?: number): void;
+  /** Close the open incident and hand back what it accumulated, for the
+   *  recovery page's downtime and diverted count. `undefined` when none. */
+  closeOrchestratorIncident(project: string, at: number): OrchestratorIncident | undefined;
+  /** Append one daemon stop/restart provenance line (#378). Host-wide — never
+   *  partitioned by project, so any project's status reads the same records. */
+  recordDaemonStop(draft: DaemonStopDraft): DaemonStop;
+  /** The newest stop/restart provenance line, or `undefined` when none was
+   *  ever recorded. Read by `status` while the daemon is down and after the
+   *  next start — the debrief line an operator gets instead of a bare
+   *  "daemon not running". */
+  latestDaemonStop(): DaemonStop | undefined;
   /** Record one observed GitHub rate-limit refusal (the tracker's hook). Rows
    *  older than 24h are pruned in the same write (#198). */
   recordGhRefusal?(at: number): void;
@@ -1881,6 +1971,98 @@ export interface HeldNoticeDraft {
   urgent?: true;
 }
 
+/**
+ * Why the embedded orchestrator is down. `start-failed` covers a session that
+ * never came up (the daemon's `startOrchestrator` threw); `crashed` covers a
+ * session that died after a healthy start (the terminal event reached the
+ * handle and it reported not alive).
+ */
+export type OrchestratorDownMode = "start-failed" | "crashed";
+
+/**
+ * The durable orchestrator-down incident, one per project, re-derived across
+ * daemon restarts so a restart while still down cannot forget it.
+ *
+ * Rows persist while the incident is open and are removed on recovery. The
+ * escalate path writes it from the store and the reconcile closes it against
+ * the same store, so a daemon killed mid-incident leaves the row for the next
+ * one — the records ledger advances only on an finished outcome, never on a
+ * process that is gone.
+ */
+export interface OrchestratorIncident {
+  project: string;
+  mode: OrchestratorDownMode;
+  /** Bounded, human-readable cause. Normally the start error or a "session
+   *  child exited N" note. */
+  cause?: string;
+  /** Epoch ms when this incident began. Stable for the life of the row, and
+   *  the dedupe anchor: the down page and its recovery page both key on it. */
+  since: number;
+  /** Tier-1 escalations diverted to an issue comment while it was down. */
+  diverted: number;
+}
+
+/** What {@link Store.openOrchestratorIncident} is handed. */
+export interface OrchestratorIncidentDraft {
+  project: string;
+  mode: OrchestratorDownMode;
+  cause?: string;
+  since: number;
+}
+
+/**
+ * One durable stop/restart provenance line for the shared daemon (#378).
+ *
+ * Host-wide on purpose: the daemon serves every configured project, so this
+ * table is NOT partitioned by project — a record written by one project's CLI
+ * must be readable from another project's `status` the moment the daemon is
+ * down, exactly the situation the incidents in #378 left unattributable.
+ *
+ * Explicit fields only, and never credentials or environment values: the audit
+ * incident was a stop nobody could attribute, and a "just serialise the
+ * command line" implementation would leak tokens. Caller pid/uid and session
+ * role are captured when a conductor process carried the request; the
+ * external-signal fallback records `unattributed` because Linux exposes no
+ * sender identity for a signal, and saying which daemon and which live runs
+ * were affected is the honest capture of what the receiving process knew.
+ */
+export interface DaemonStop {
+  id: string;
+  /** Epoch ms the stop/restart was requested (mediated) or observed (fallback). */
+  at: number;
+  /**
+   * The operator-visible control path: "cli stop", "cli restart",
+   * "cli stop via systemctl" (the mediated request, as actually delivered),
+   * or "external signal" (the unattributed fallback).
+   */
+  controlPath: string;
+  /** Caller pid, when a conductor process carried the request. */
+  callerPid?: number;
+  /** Caller uid, when knowable. */
+  callerUid?: number;
+  /** The requesting session's role, when knowable (see {@link SESSION_ROLE_ENV}). */
+  role?: SessionRole;
+  /** "project" when the request originated from one project; "global" otherwise. */
+  scope: "global" | "project";
+  /** The originating project, when project-scoped. */
+  project?: string;
+  /** The daemon pid the stop/restart acted on, when one was known. */
+  daemonPid?: number;
+  /** The daemon runtime directory, when the record is the receiving daemon's
+   *  own fallback. Host paths only — the field never carries secrets. */
+  runtimeDir?: string;
+  /** Every configured project with its live-run count at request time. A
+   *  project-scoped stop of the shared daemon names the siblings here. */
+  affected: { project: string; live: number }[];
+  /** Non-secret reason; never credentials or environment values. */
+  reason: string;
+  /** True only when no mediated request existed — the external-signal fallback. */
+  unattributed: boolean;
+}
+
+/** What a caller hands {@link Store.recordDaemonStop}. The store owns `id` and `at`. */
+export type DaemonStopDraft = Omit<DaemonStop, "id" | "at">;
+
 /**
  * Baseline limits used when a project omits `caps`. Data, not behaviour: kept
  * beside the type so the defaults cannot drift out of shape with it.

package/src/upgrade.ts

@@ -311,13 +311,39 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
  * exactly this transaction's pause while leaving any per-project operator hold
  * standing.
  */
-interface UpgradeScope {
+export interface UpgradeScope {
   /** Project selectors to address the project-aware deps with, never empty. */
   selectors: readonly (string | undefined)[];
   /** The sentinel this transaction pauses and resumes. */
   pauseKey: string | undefined;
 }
 
+/**
+ * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
+ * so a caller that only wants the fence — `setup host` — can provide just
+ * these members instead of a full install-and-verify deps object. Every
+ * {@link UpgradeDeps} is a `DrainDeps`.
+ */
+export interface DrainDeps {
+  /** Live worker count per project selector; `undefined` = the one configured project. */
+  snapshot(project?: string): { liveWorkers: number };
+  /** The fleet's current dispatch/daemon layers for one project selector. */
+  layers(project?: string): FleetLayers;
+  /** Every configured project name, in config order (#389). */
+  projectNames(): readonly string[];
+  /**
+   * The daemon the transaction targets: running or not, its project when it
+   * recorded one, and a `generation` identifying the exact instance (#377).
+   */
+  daemonIdentity(): { running: boolean; project?: string; generation?: string };
+  /** The active pause sentinel as an instance, when one is readable (#377). */
+  pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
+  /** Write or clear the durable pause sentinel for the scope's project. */
+  setPaused(value: boolean, project?: string): void;
+  sleep(ms: number): Promise<void>;
+  log(message: string): void;
+}
+
 /**
  * Decides which projects a transaction covers, and refuses the requests that
  * cannot be honoured truthfully.
@@ -336,7 +362,11 @@ interface UpgradeScope {
  * A single-configured-project host keeps the historical bare selector, so its
  * behaviour is unchanged whether or not the daemon recorded its name.
  */
-function resolveScope(deps: UpgradeDeps, verb: "upgrade" | "restart", project?: string): UpgradeScope {
+export function resolveScope(
+  deps: DrainDeps,
+  verb: DrainVerb,
+  project?: string,
+): UpgradeScope {
   let configured: readonly string[] = [];
   try {
     configured = deps.projectNames();
@@ -372,8 +402,8 @@ function resolveScope(deps: UpgradeDeps, verb: "upgrade" | "restart", project?:
   return { selectors: [undefined], pauseKey: undefined };
 }
 
-async function waitForDrain(
-  deps: UpgradeDeps,
+export async function waitForDrain(
+  deps: DrainDeps,
   scope: UpgradeScope,
   deadlineAt?: number,
   stale?: () => string | undefined,
@@ -427,7 +457,7 @@ interface RestartBegun {
  * stopped by it.
  */
 function restartFenceProblem(
-  deps: UpgradeDeps,
+  deps: DrainDeps,
   scope: UpgradeScope,
   begun: RestartBegun,
 ): string | undefined {
@@ -450,6 +480,79 @@ function restartFenceProblem(
   return undefined;
 }
 
+/**
+ * Every verb {@link pauseAndDrain} accepts, declared once and used for both the
+ * `verb` parameter's type and the sentinel round-trip test — so a fourth verb
+ * that cannot be encoded as a single `source=` token (see
+ * {@link pauseSourceToken}; a verb with a space) fails the suite instead of
+ * shipping an unprovable pause (#552).
+ */
+export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
+export type DrainVerb = (typeof DRAIN_VERBS)[number];
+
+/**
+ * Pause claims and drain live workers to idle — the destructive half of the
+ * trusted restart transaction, minus the restart. Exported so `setup host`
+ * runs the same fence `upgrade` and `restart` do, then performs its own unit
+ * installs instead of {@link drainAndRestart}'s daemon restart.
+ *
+ * Generation-scoped (#377) exactly as the full transaction is: the caller
+ * captures the restart-owned pause and the daemon generation, and the drain
+ * aborts if a `resume` lifts the pause, replaces the sentinel, or a newer
+ * daemon generation appears. The fence failure path (the pause it just
+ * created cannot be read back as an instance) fails closed WITHOUT leaving
+ * that pause behind: a pause the transaction cannot prove it is still using
+ * is released, restoring the entry state, so the refusal never stops dispatch
+ * on its own (#552). A drain that outlives `timeoutMs` stays paused (no
+ * resume in a catch) — a drain cannot silently resume dispatch over a wedged
+ * daemon. `timeoutMs` absent waits indefinitely, which is the `upgrade`
+ * posture.
+ */
+export async function pauseAndDrain(
+  deps: DrainDeps,
+  verb: DrainVerb,
+  o: { project?: string; timeoutMs?: number },
+): Promise<{ scope: UpgradeScope; initialPaused: boolean }> {
+  // Host-wide by default (#389): the daemon this restarts serves every
+  // configured project, so the drain counts every project's workers and the
+  // pause covers all of them.
+  const scope = resolveScope(deps, verb, o.project);
+  const initial = deps.layers(scope.pauseKey);
+  if (!initial.paused) deps.setPaused(true, scope.pauseKey);
+  // The lock must be provable NOW. A pause that cannot be read as an instance
+  // — sentinel malformed or unreadable — would make the whole fence fail open
+  // (nothing to compare against), so the transaction refuses before waiting
+  // on anything it cannot act upon either way.
+  const pauseToken = deps.pauseState(scope.pauseKey);
+  if (pauseToken === undefined) {
+    // Fail closed — but do not abandon our own pause. It is only correct to
+    // hold a pause the transaction is still using, and a sentinel we cannot
+    // prove is unusable: restore the entry state exactly, so a verb that
+    // paused the fleet releases it (a fleet already paused stays paused)
+    // before the refusal propagates (#552).
+    if (!initial.paused) deps.setPaused(false, scope.pauseKey);
+    throw new Error(
+      `${verb} cancelled: cannot prove the ${verb}-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted`,
+    );
+  }
+  const begun: RestartBegun = {
+    pauseToken,
+    daemon: deps.daemonIdentity(),
+  };
+  const stale = () => restartFenceProblem(deps, scope, begun);
+  await waitForDrain(
+    deps,
+    scope,
+    o.timeoutMs === undefined ? undefined : Date.now() + o.timeoutMs,
+    stale,
+  );
+  // The drain completed; the world may have moved on while it did. Re-prove
+  // the pause and the generation an instant before the destructive call.
+  const cancelled = stale();
+  if (cancelled !== undefined) throw new Error(cancelled);
+  return { scope, initialPaused: initial.paused };
+}
+
 /**
  * Pause, drain, restart, restore — the trusted restart transaction, minus the
  * install/verify steps of {@link upgradeConductor}.
@@ -477,34 +580,9 @@ export async function drainAndRestart(
   deps: UpgradeDeps,
   o: { project?: string; timeoutMs: number },
 ): Promise<void> {
-  // Host-wide by default (#389): the daemon this restarts serves every
-  // configured project, so the drain counts every project's workers and the
-  // pause covers all of them.
-  const scope = resolveScope(deps, "restart", o.project);
-  const initial = deps.layers(scope.pauseKey);
-  if (!initial.paused) deps.setPaused(true, scope.pauseKey);
-  // The lock must be provable NOW. A pause that cannot be read as an instance
-  // — sentinel malformed or unreadable — would make the whole fence fail open
-  // (nothing to compare against), so the transaction refuses before waiting
-  // on anything it cannot act upon either way.
-  const pauseToken = deps.pauseState(scope.pauseKey);
-  if (pauseToken === undefined) {
-    throw new Error(
-      "restart cancelled: cannot prove the restart-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted",
-    );
-  }
-  const begun: RestartBegun = {
-    pauseToken,
-    daemon: deps.daemonIdentity(),
-  };
-  const stale = () => restartFenceProblem(deps, scope, begun);
-  await waitForDrain(deps, scope, Date.now() + o.timeoutMs, stale);
-  // The drain completed; the world may have moved on while it did. Re-prove
-  // the pause and the generation an instant before the destructive call.
-  const cancelled = stale();
-  if (cancelled !== undefined) throw new Error(cancelled);
+  const { scope, initialPaused } = await pauseAndDrain(deps, "restart", o);
   await deps.restartDaemon();
-  if (!initial.paused) deps.setPaused(false, scope.pauseKey);
+  if (!initialPaused) deps.setPaused(false, scope.pauseKey);
 }
 
 function recoveryProblem(

package/src/verbs/protocol.ts

@@ -180,18 +180,21 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
     allowedRoles: ["orchestrator"],
     description:
       "Merge one open pull request. The daemon re-reads the live head immediately before merging " +
-      "and refuses on any mismatch with headSha, and one merge is in flight per project at a time.",
+      "and refuses on any mismatch with headSha, and one merge is in flight per project at a time. " +
+      "A PR a run of this project opened merges on the holder's authority; a PR no run opened " +
+      "merges only as the orchestrator's own work (author=orchestrator), and only in a routed repo.",
     args: {
       prUrl: {
         type: "string",
         required: true,
-        description: "Full pull request URL, belonging to a run in this project.",
+        description:
+          "Full pull request URL — a run's in this project, or the orchestrator's own (author=orchestrator).",
       },
       headSha: {
         type: "string",
         required: true,
         description:
-          "The head you believe you are merging. Re-read live before the merge; a stale one is refused.",
+          "The exact head you believe you are merging. Re-read live before the merge; a stale one is refused.",
       },
       reason: {
         type: "string",
@@ -199,6 +202,16 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
         description: `Why this merge, from the closed set: ${MERGE_REASONS.join(", ")}.`,
         oneOf: MERGE_REASONS,
       },
+      author: {
+        type: "string",
+        required: false,
+        description:
+          "Optional authorship claim; only 'orchestrator'. States that no run opened this PR — it is " +
+          "the orchestrator's own work. Required for a PR no run in this project opened; it never " +
+          "unlocks a run's PR, and never bypasses the configured holder, the routed-repo scope, the " +
+          "live-head recheck, the green-checks gate, the pause or the ledger.",
+        oneOf: ["orchestrator"],
+      },
       rationale: RATIONALE_ARG,
     },
     // The exact sentence #126 asks a refused worker to be given. `authority`

package/src/verbs/server.ts

@@ -812,6 +812,12 @@ async function prMergeVerb(
   const prUrl = String(args["prUrl"]);
   const headSha = String(args["headSha"]);
   const reason = String(args["reason"]);
+  // An explicit authorship claim: `<author> = "orchestrator"` states that no
+  // run opened this PR — it is the caller's own work. The claim is the only
+  // thing that can make a run-less PR eligible, and it unlocks nothing else:
+  // the holder gate above, the routed-repo scope, the pause, the live-head
+  // recheck, the green-checks gate and the ledger all still apply below.
+  const authoredByOrchestrator = args["author"] === "orchestrator";
   const target = runForPr(deps, project.name, prUrl);
   const recovery =
     target === undefined
@@ -836,8 +842,28 @@ async function prMergeVerb(
           : "Ask the operator which session is meant to hold it."),
     );
   }
+  if (authoredByOrchestrator && target !== undefined) {
+    return refuse(
+      "pr-not-this-run",
+      `refused: ${prUrl} is a run-opened PR in ${project.name}; the orchestrator-authors claim is only for ` +
+        "pull requests no run ever opened, and a run-owned PR is not claimable as the orchestrator's own work.",
+    );
+  }
   if (target === undefined && recovery === undefined) {
-    return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
+    if (!authoredByOrchestrator) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
+          "If the orchestrator authored it (no run opened it), re-call with --arg author=orchestrator; " +
+          "a PR a run opened stays that run's.",
+      );
+    }
+    if (routedRepo === undefined) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: ${prUrl} is not in ${project.name}'s routed repositories; the orchestrator-authors path cannot widen project scope.`,
+      );
+    }
   }
   if (target === undefined && routedRepo === undefined) {
     return refuse(