omp-conductor 0.15.12 → 0.15.13

package/src/store.ts

@@ -24,6 +24,8 @@ import type {
   DecisionState,
   DigestBacklog,
   DispatchSummary,
+  DaemonStop,
+  DaemonStopDraft,
   FrictionAdmissionReason,
   FrictionKind,
   FrictionObservation,
@@ -38,6 +40,9 @@ import type {
   MaterialEventDraft,
   LabelOp,
   MergeLock,
+  OrchestratorDownMode,
+  OrchestratorIncident,
+  OrchestratorIncidentDraft,
   ReportDeliveryState,
   ReportDraft,
   ReportEnqueue,
@@ -327,6 +332,65 @@ function toIntakeItem(row: IntakeRow): IntakeItem {
   return item;
 }
 
+/** The `orchestrator_incidents` table exactly as SQLite hands it back (#288). */
+interface OrchestratorIncidentRow {
+  project: string;
+  mode: string;
+  cause: string | null;
+  since: number;
+  diverted: number;
+}
+
+/** NULL columns become absent properties, matching the other row converters. */
+function toOrchestratorIncident(row: OrchestratorIncidentRow): OrchestratorIncident {
+  return {
+    project: row.project,
+    mode: row.mode as OrchestratorDownMode,
+    ...(row.cause === null ? {} : { cause: row.cause }),
+    since: row.since,
+    diverted: row.diverted,
+  };
+}
+
+/** The `daemon_stops` table exactly as SQLite hands it back (#378). */
+interface DaemonStopRow {
+  id: string;
+  at: number;
+  controlPath: string;
+  callerPid: number | null;
+  callerUid: number | null;
+  role: string | null;
+  scope: string;
+  project: string | null;
+  daemonPid: number | null;
+  runtimeDir: string | null;
+  affected: string;
+  reason: string;
+  unattributed: number;
+}
+
+/**
+ * NULL columns become absent properties, matching the other row converters:
+ * a record read back out of the store deep-equals the one that went in.
+ */
+function toDaemonStop(row: DaemonStopRow): DaemonStop {
+  return {
+    id: row.id,
+    at: row.at,
+    controlPath: row.controlPath,
+    ...(row.callerPid === null ? {} : { callerPid: row.callerPid }),
+    ...(row.callerUid === null ? {} : { callerUid: row.callerUid }),
+    ...(row.role === null ? {} : { role: row.role as SessionRole }),
+    scope: row.scope as "global" | "project",
+    ...(row.project === null ? {} : { project: row.project }),
+    ...(row.daemonPid === null ? {} : { daemonPid: row.daemonPid }),
+    ...(row.runtimeDir === null ? {} : { runtimeDir: row.runtimeDir }),
+    affected: JSON.parse(row.affected) as { project: string; live: number }[],
+    reason: row.reason,
+    unattributed: row.unattributed === 1,
+  };
+}
+
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS runs (
   id          TEXT    PRIMARY KEY,
@@ -628,6 +692,44 @@ CREATE TABLE IF NOT EXISTS intake_items (
 );
 CREATE INDEX IF NOT EXISTS intake_items_project_state
   ON intake_items (project, state, createdAt);
+-- The embedded orchestrator's lost-liveness incident (#288): durable so a
+-- daemon restarted while its orchestrator is still down rediscovers the open
+-- incident instead of forgetting it, and the diverted counter accumulates
+-- across the outage and across restarts. One row per project; removed on
+-- recovery. since is the dedupe anchor for both the down page and its
+-- closing recovery page. The mode CHECK keeps the two causes distinct.
+CREATE TABLE IF NOT EXISTS orchestrator_incidents (
+  project TEXT    PRIMARY KEY,
+  mode    TEXT    NOT NULL CHECK (mode IN ('start-failed', 'crashed')),
+  cause   TEXT,
+  since   INTEGER NOT NULL,
+  diverted INTEGER NOT NULL DEFAULT 0
+);
+
+-- Every stop/restart of the shared daemon, and who asked for it and why
+-- (#378). Deliberately NOT partitioned by project: the daemon serves every
+-- configured project, so a record written by one project's CLI must be
+-- readable from another project's status the moment the daemon is down —
+-- that cross-project read is the whole point. Written before signalling on
+-- the mediated paths (cli stop/restart) and as an honest unattributed
+-- fallback when the daemon receives a signal with no mediated request.
+-- Explicit fields only: never raw argv, environment values or credentials.
+CREATE TABLE IF NOT EXISTS daemon_stops (
+  id           TEXT    PRIMARY KEY,
+  at           INTEGER NOT NULL,
+  controlPath  TEXT    NOT NULL,
+  callerPid    INTEGER,
+  callerUid    INTEGER,
+  role         TEXT,
+  scope        TEXT    NOT NULL,
+  project      TEXT,
+  daemonPid    INTEGER,
+  runtimeDir   TEXT,
+  affected     TEXT    NOT NULL,
+  reason       TEXT    NOT NULL,
+  unattributed INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS daemon_stops_at ON daemon_stops (at);
 `;
 
 /**
@@ -1365,6 +1467,23 @@ export function openStore(dbPath: string): Store {
   const insertNotified = db.query<unknown, [string, number]>(
     `INSERT OR IGNORE INTO notifications ("key", at) VALUES (?, ?)`,
   );
+  // Orchestrator-down incident (#288). One row per project; INSERT OR IGNORE is
+  // the dedupe so a flapping orchestrator opens (and pages) once per incident.
+  const selectOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `SELECT project, mode, cause, since, diverted FROM orchestrator_incidents WHERE project = ?`,
+  );
+  const insertOrchestratorIncident = db.query<unknown, [string, string, string | null, number]>(
+    `INSERT OR IGNORE INTO orchestrator_incidents (project, mode, cause, since)
+     VALUES (?, ?, ?, ?)`,
+  );
+  const bumpOrchestratorIncident = db.query<unknown, [number, string]>(
+    `UPDATE orchestrator_incidents SET diverted = diverted + ? WHERE project = ?`,
+  );
+  const deleteOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `DELETE FROM orchestrator_incidents
+      WHERE project = ?
+      RETURNING project, mode, cause, since, diverted`,
+  );
   const upsertDispatch = db.query<unknown, [string, string]>(
     `INSERT INTO dispatch_summaries (project, summary) VALUES (?, ?)
      ON CONFLICT(project) DO UPDATE SET summary = excluded.summary`,
@@ -1889,6 +2008,16 @@ export function openStore(dbPath: string): Store {
       ORDER BY at DESC, rowid DESC
       LIMIT ?`,
   );
+  const insertDaemonStop = db.query<unknown, SqlValue[]>(
+    `INSERT INTO daemon_stops
+       (id, at, controlPath, callerPid, callerUid, role, scope, project, daemonPid, runtimeDir, affected, reason, unattributed)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  );
+  // `rowid` breaks the tie when two requests land in the same millisecond, so
+  // "newest first" never silently hides one stop behind another.
+  const selectLatestDaemonStop = db.query<DaemonStopRow, []>(
+    `SELECT * FROM daemon_stops ORDER BY at DESC, rowid DESC LIMIT 1`,
+  );
   // Breaking a stale claim and taking it must be one transaction, or two
   // daemons both see the stale row, both delete it, and both insert.
   const breakStaleMergeLock = db.query<unknown, [string, number]>(
@@ -2223,6 +2352,58 @@ export function openStore(dbPath: string): Store {
       insertNotified.run(key, Date.now());
     },
 
+    orchestratorIncident(project: string): OrchestratorIncident | undefined {
+      const row = selectOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean {
+      return (
+        insertOrchestratorIncident.run(
+          draft.project,
+          draft.mode,
+          draft.cause ?? null,
+          draft.since,
+        ).changes === 1
+      );
+    },
+
+    bumpOrchestratorDiverted(project: string, by = 1): void {
+      // Only meaningful while an incident is open: a healthy or external
+      // orchestrator has no row, and the UPDATE touches nothing.
+      bumpOrchestratorIncident.run(by, project);
+    },
+
+    closeOrchestratorIncident(project: string, _at: number): OrchestratorIncident | undefined {
+      const row = deleteOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    recordDaemonStop(draft: DaemonStopDraft): DaemonStop {
+      const entry: DaemonStop = { ...draft, id: crypto.randomUUID(), at: Date.now() };
+      insertDaemonStop.run(
+        entry.id,
+        entry.at,
+        entry.controlPath,
+        toSql(entry.callerPid),
+        toSql(entry.callerUid),
+        toSql(entry.role),
+        entry.scope,
+        toSql(entry.project),
+        toSql(entry.daemonPid),
+        toSql(entry.runtimeDir),
+        JSON.stringify(entry.affected),
+        entry.reason,
+        entry.unattributed ? 1 : 0,
+      );
+      return entry;
+    },
+
+    latestDaemonStop(): DaemonStop | undefined {
+      const row = selectLatestDaemonStop.get();
+      return row === null ? undefined : toDaemonStop(row);
+    },
+
     recordDispatch(project: string, summary: DispatchSummary): void {
       const previous = selectDispatch.get(project);
       upsertDispatch.run(project, JSON.stringify(summary));

package/src/types.ts

@@ -1736,6 +1736,25 @@ export interface Store {
   /** Start the cooldown only after a tick carrying these signals was sent. */
   markFrictionSurfaced(project: string, kinds: readonly FrictionKind[], at: number): void;
   markNotified(key: string): void;
+  /** The live embedded-orchestrator-down incident for a project, if any. */
+  orchestratorIncident(project: string): OrchestratorIncident | undefined;
+  /** Record a new incident. `false` when one is already open for the project —
+   *  dedupe so a flapping orchestrator opens (and pages) once per incident. */
+  openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean;
+  /** Count tier-1 escalations diverted to an issue comment. No-op when no
+   *  incident is open, so a healthy or external orchestrator never accumulates. */
+  bumpOrchestratorDiverted(project: string, by?: number): void;
+  /** Close the open incident and hand back what it accumulated, for the
+   *  recovery page's downtime and diverted count. `undefined` when none. */
+  closeOrchestratorIncident(project: string, at: number): OrchestratorIncident | undefined;
+  /** Append one daemon stop/restart provenance line (#378). Host-wide — never
+   *  partitioned by project, so any project's status reads the same records. */
+  recordDaemonStop(draft: DaemonStopDraft): DaemonStop;
+  /** The newest stop/restart provenance line, or `undefined` when none was
+   *  ever recorded. Read by `status` while the daemon is down and after the
+   *  next start — the debrief line an operator gets instead of a bare
+   *  "daemon not running". */
+  latestDaemonStop(): DaemonStop | undefined;
   /** Record one observed GitHub rate-limit refusal (the tracker's hook). Rows
    *  older than 24h are pruned in the same write (#198). */
   recordGhRefusal?(at: number): void;
@@ -1952,6 +1971,98 @@ export interface HeldNoticeDraft {
   urgent?: true;
 }
 
+/**
+ * Why the embedded orchestrator is down. `start-failed` covers a session that
+ * never came up (the daemon's `startOrchestrator` threw); `crashed` covers a
+ * session that died after a healthy start (the terminal event reached the
+ * handle and it reported not alive).
+ */
+export type OrchestratorDownMode = "start-failed" | "crashed";
+
+/**
+ * The durable orchestrator-down incident, one per project, re-derived across
+ * daemon restarts so a restart while still down cannot forget it.
+ *
+ * Rows persist while the incident is open and are removed on recovery. The
+ * escalate path writes it from the store and the reconcile closes it against
+ * the same store, so a daemon killed mid-incident leaves the row for the next
+ * one — the records ledger advances only on an finished outcome, never on a
+ * process that is gone.
+ */
+export interface OrchestratorIncident {
+  project: string;
+  mode: OrchestratorDownMode;
+  /** Bounded, human-readable cause. Normally the start error or a "session
+   *  child exited N" note. */
+  cause?: string;
+  /** Epoch ms when this incident began. Stable for the life of the row, and
+   *  the dedupe anchor: the down page and its recovery page both key on it. */
+  since: number;
+  /** Tier-1 escalations diverted to an issue comment while it was down. */
+  diverted: number;
+}
+
+/** What {@link Store.openOrchestratorIncident} is handed. */
+export interface OrchestratorIncidentDraft {
+  project: string;
+  mode: OrchestratorDownMode;
+  cause?: string;
+  since: number;
+}
+
+/**
+ * One durable stop/restart provenance line for the shared daemon (#378).
+ *
+ * Host-wide on purpose: the daemon serves every configured project, so this
+ * table is NOT partitioned by project — a record written by one project's CLI
+ * must be readable from another project's `status` the moment the daemon is
+ * down, exactly the situation the incidents in #378 left unattributable.
+ *
+ * Explicit fields only, and never credentials or environment values: the audit
+ * incident was a stop nobody could attribute, and a "just serialise the
+ * command line" implementation would leak tokens. Caller pid/uid and session
+ * role are captured when a conductor process carried the request; the
+ * external-signal fallback records `unattributed` because Linux exposes no
+ * sender identity for a signal, and saying which daemon and which live runs
+ * were affected is the honest capture of what the receiving process knew.
+ */
+export interface DaemonStop {
+  id: string;
+  /** Epoch ms the stop/restart was requested (mediated) or observed (fallback). */
+  at: number;
+  /**
+   * The operator-visible control path: "cli stop", "cli restart",
+   * "cli stop via systemctl" (the mediated request, as actually delivered),
+   * or "external signal" (the unattributed fallback).
+   */
+  controlPath: string;
+  /** Caller pid, when a conductor process carried the request. */
+  callerPid?: number;
+  /** Caller uid, when knowable. */
+  callerUid?: number;
+  /** The requesting session's role, when knowable (see {@link SESSION_ROLE_ENV}). */
+  role?: SessionRole;
+  /** "project" when the request originated from one project; "global" otherwise. */
+  scope: "global" | "project";
+  /** The originating project, when project-scoped. */
+  project?: string;
+  /** The daemon pid the stop/restart acted on, when one was known. */
+  daemonPid?: number;
+  /** The daemon runtime directory, when the record is the receiving daemon's
+   *  own fallback. Host paths only — the field never carries secrets. */
+  runtimeDir?: string;
+  /** Every configured project with its live-run count at request time. A
+   *  project-scoped stop of the shared daemon names the siblings here. */
+  affected: { project: string; live: number }[];
+  /** Non-secret reason; never credentials or environment values. */
+  reason: string;
+  /** True only when no mediated request existed — the external-signal fallback. */
+  unattributed: boolean;
+}
+
+/** What a caller hands {@link Store.recordDaemonStop}. The store owns `id` and `at`. */
+export type DaemonStopDraft = Omit<DaemonStop, "id" | "at">;
+
 /**
  * Baseline limits used when a project omits `caps`. Data, not behaviour: kept
  * beside the type so the defaults cannot drift out of shape with it.

package/src/upgrade.ts

@@ -364,7 +364,7 @@ export interface DrainDeps {
  */
 export function resolveScope(
   deps: DrainDeps,
-  verb: "upgrade" | "restart" | "setup host",
+  verb: DrainVerb,
   project?: string,
 ): UpgradeScope {
   let configured: readonly string[] = [];
@@ -480,6 +480,16 @@ function restartFenceProblem(
   return undefined;
 }
 
+/**
+ * Every verb {@link pauseAndDrain} accepts, declared once and used for both the
+ * `verb` parameter's type and the sentinel round-trip test — so a fourth verb
+ * that cannot be encoded as a single `source=` token (see
+ * {@link pauseSourceToken}; a verb with a space) fails the suite instead of
+ * shipping an unprovable pause (#552).
+ */
+export const DRAIN_VERBS = ["upgrade", "restart", "setup host"] as const;
+export type DrainVerb = (typeof DRAIN_VERBS)[number];
+
 /**
  * Pause claims and drain live workers to idle — the destructive half of the
  * trusted restart transaction, minus the restart. Exported so `setup host`
@@ -489,14 +499,18 @@ function restartFenceProblem(
  * Generation-scoped (#377) exactly as the full transaction is: the caller
  * captures the restart-owned pause and the daemon generation, and the drain
  * aborts if a `resume` lifts the pause, replaces the sentinel, or a newer
- * daemon generation appears. On ANY throw after the pause the fleet stays
- * paused (no resume in a catch), so a drain that outlives `timeoutMs` cannot
- * silently resume dispatch over a wedged daemon. `timeoutMs` absent waits
- * indefinitely, which is the `upgrade` posture.
+ * daemon generation appears. The fence failure path (the pause it just
+ * created cannot be read back as an instance) fails closed WITHOUT leaving
+ * that pause behind: a pause the transaction cannot prove it is still using
+ * is released, restoring the entry state, so the refusal never stops dispatch
+ * on its own (#552). A drain that outlives `timeoutMs` stays paused (no
+ * resume in a catch) — a drain cannot silently resume dispatch over a wedged
+ * daemon. `timeoutMs` absent waits indefinitely, which is the `upgrade`
+ * posture.
  */
 export async function pauseAndDrain(
   deps: DrainDeps,
-  verb: "upgrade" | "restart" | "setup host",
+  verb: DrainVerb,
   o: { project?: string; timeoutMs?: number },
 ): Promise<{ scope: UpgradeScope; initialPaused: boolean }> {
   // Host-wide by default (#389): the daemon this restarts serves every
@@ -511,6 +525,12 @@ export async function pauseAndDrain(
   // on anything it cannot act upon either way.
   const pauseToken = deps.pauseState(scope.pauseKey);
   if (pauseToken === undefined) {
+    // Fail closed — but do not abandon our own pause. It is only correct to
+    // hold a pause the transaction is still using, and a sentinel we cannot
+    // prove is unusable: restore the entry state exactly, so a verb that
+    // paused the fleet releases it (a fleet already paused stays paused)
+    // before the refusal propagates (#552).
+    if (!initial.paused) deps.setPaused(false, scope.pauseKey);
     throw new Error(
       `${verb} cancelled: cannot prove the ${verb}-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted`,
     );

package/src/verbs/protocol.ts

@@ -180,18 +180,21 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
     allowedRoles: ["orchestrator"],
     description:
       "Merge one open pull request. The daemon re-reads the live head immediately before merging " +
-      "and refuses on any mismatch with headSha, and one merge is in flight per project at a time.",
+      "and refuses on any mismatch with headSha, and one merge is in flight per project at a time. " +
+      "A PR a run of this project opened merges on the holder's authority; a PR no run opened " +
+      "merges only as the orchestrator's own work (author=orchestrator), and only in a routed repo.",
     args: {
       prUrl: {
         type: "string",
         required: true,
-        description: "Full pull request URL, belonging to a run in this project.",
+        description:
+          "Full pull request URL — a run's in this project, or the orchestrator's own (author=orchestrator).",
       },
       headSha: {
         type: "string",
         required: true,
         description:
-          "The head you believe you are merging. Re-read live before the merge; a stale one is refused.",
+          "The exact head you believe you are merging. Re-read live before the merge; a stale one is refused.",
       },
       reason: {
         type: "string",
@@ -199,6 +202,16 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
         description: `Why this merge, from the closed set: ${MERGE_REASONS.join(", ")}.`,
         oneOf: MERGE_REASONS,
       },
+      author: {
+        type: "string",
+        required: false,
+        description:
+          "Optional authorship claim; only 'orchestrator'. States that no run opened this PR — it is " +
+          "the orchestrator's own work. Required for a PR no run in this project opened; it never " +
+          "unlocks a run's PR, and never bypasses the configured holder, the routed-repo scope, the " +
+          "live-head recheck, the green-checks gate, the pause or the ledger.",
+        oneOf: ["orchestrator"],
+      },
       rationale: RATIONALE_ARG,
     },
     // The exact sentence #126 asks a refused worker to be given. `authority`

package/src/verbs/server.ts

@@ -812,6 +812,12 @@ async function prMergeVerb(
   const prUrl = String(args["prUrl"]);
   const headSha = String(args["headSha"]);
   const reason = String(args["reason"]);
+  // An explicit authorship claim: `<author> = "orchestrator"` states that no
+  // run opened this PR — it is the caller's own work. The claim is the only
+  // thing that can make a run-less PR eligible, and it unlocks nothing else:
+  // the holder gate above, the routed-repo scope, the pause, the live-head
+  // recheck, the green-checks gate and the ledger all still apply below.
+  const authoredByOrchestrator = args["author"] === "orchestrator";
   const target = runForPr(deps, project.name, prUrl);
   const recovery =
     target === undefined
@@ -836,8 +842,28 @@ async function prMergeVerb(
           : "Ask the operator which session is meant to hold it."),
     );
   }
+  if (authoredByOrchestrator && target !== undefined) {
+    return refuse(
+      "pr-not-this-run",
+      `refused: ${prUrl} is a run-opened PR in ${project.name}; the orchestrator-authors claim is only for ` +
+        "pull requests no run ever opened, and a run-owned PR is not claimable as the orchestrator's own work.",
+    );
+  }
   if (target === undefined && recovery === undefined) {
-    return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
+    if (!authoredByOrchestrator) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: ${prUrl} is not a pull request any run in ${project.name} opened. ` +
+          "If the orchestrator authored it (no run opened it), re-call with --arg author=orchestrator; " +
+          "a PR a run opened stays that run's.",
+      );
+    }
+    if (routedRepo === undefined) {
+      return refuse(
+        "pr-not-this-run",
+        `refused: ${prUrl} is not in ${project.name}'s routed repositories; the orchestrator-authors path cannot widen project scope.`,
+      );
+    }
   }
   if (target === undefined && routedRepo === undefined) {
     return refuse(