omp-conductor 0.15.12 → 0.15.13

package/src/orchestrator-down.ts

@@ -0,0 +1,231 @@
+/**
+ * The orchestrator-down incident: how the daemon learns — and tells its
+ * operator — that the embedded orchestrator session is gone.
+ *
+ * A failed orchestrator used to degrade quietly: the daemon logged a warning
+ * and kept running, but the tier-1 escalations that then fell back to issue
+ * comments were exactly the "nobody reads it until morning" path the
+ * orchestrator exists to avoid. Nothing paged, and `status` never said the
+ * operator channel was degraded (the pending issue is #288).
+ *
+ * This module makes three facts durable and observable:
+ *
+ *  1. **The incident.** Start failure (no handle) and unexpected session death
+ *     (a handle whose `alive()` is false) each open one durable
+ *     `orchestrator-incident` row, re-derived across restarts so a daemon
+ *     restarted while still down rediscovers it. The row survives and is
+ *     removed only on recovery.
+ *  2. **One page per incident.** Open, and every subsequent down tick, page a
+ *     tier-2 urgent escalation whose key is anchored on the incident's `since`
+ *     timestamp — the notification ledger pages once, never per tick, and a
+ *     *new* incident (a flapping orchestrator recovering then dying again) gets
+ *     a new anchor and pages again.
+ *  3. **The degrade row.** While down, `status` names the mode, the since-
+ *     moment and how many tier-1 escalations were diverted to issue comments.
+ *     Recovery closes the row and pages one closing notice on the same anchor,
+ *     with downtime and the diverted count.
+ *
+ * The session liveness driving this is event-driven (the proxy's `session_exit`
+ * terminal event), never a timer: a wedged-but-alive session stays "alive"
+ * here and belongs to the `.conductor-stalled` watchdog, which this module
+ * deliberately does not touch.
+ */
+
+import type { OrchestratorHandle } from "./orchestrator.ts";
+import type {
+  Escalation,
+  OrchestratorDownMode,
+  OrchestratorIncident,
+  ProjectConfig,
+  Store,
+} from "./types.ts";
+
+/** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */
+const NO_ISSUE = 0;
+
+export interface OrchestratorDownDeps {
+  project: ProjectConfig;
+  store: Store;
+  /** The embedded session handle; `undefined` when it failed to start. */
+  orchestrator?: OrchestratorHandle;
+  /**
+   * The daemon's escalation transport. Errors are caught here (the call is
+   * best-effort and ledger-deduped), so a transport that is itself down never
+   * takes the tick down with it.
+   */
+  escalate(e: Escalation): Promise<void>;
+  /**
+   * Start-failure cause, only meaningful when a new incident is opened with no
+   * handle. Carried so the page and `status` tell the operator *why*, not just
+   * that the orchestrator is gone.
+   */
+  startCause?: string;
+  now?(): number;
+  log?(msg: string): void;
+}
+
+function iso(at: number): string {
+  return new Date(at).toISOString();
+}
+
+/** Compact, human-scale duration for downtime and ages. */
+export function formatDownDuration(ms: number): string {
+  if (ms < 60_000) return `${Math.max(0, Math.round(ms / 1_000))}s`;
+  if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m`;
+  if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h`;
+  return `${Math.round(ms / 86_400_000)}d`;
+}
+
+function warn(msg: string): void {
+  process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`);
+}
+
+function errText(e: unknown): string {
+  return e instanceof Error ? e.message : String(e);
+}
+
+/**
+ * Best-effort page, mirroring the daemon's `safeEscalate`: a transport failure
+ * is logged, never thrown. The recipient (the daemon's escalator) owns the
+ * notification-ledger dedupe, which is what makes "page once per incident" a
+ * property of the store rather than of this loop.
+ */
+async function deliver(deps: OrchestratorDownDeps, e: Escalation): Promise<boolean> {
+  try {
+    await deps.escalate(e);
+    return true;
+  } catch (err) {
+    (deps.log ?? warn)(`orchestrator-down escalation could not be delivered: ${errText(err)}`);
+    return false;
+  }
+}
+
+/**
+ * The down page. Keyed on the incident's `since` so a still-open incident never
+ * pages again (the ledger dedupes on the summary), while each *new* incident —
+ * a fresh `since` — pages once.
+ */
+export function downEscalation(project: string, incident: OrchestratorIncident): Escalation {
+  return {
+    tier: 2,
+    category: "confirmed-failure",
+    urgent: true,
+    project,
+    issue: NO_ISSUE,
+    summary:
+      `Orchestrator down since ${iso(incident.since)} — tier-1 escalations diverting to issue comments (${project})`,
+    detail: [
+      `Mode: ${incident.mode}.`,
+      ...(incident.cause === undefined ? [] : [`Cause: ${incident.cause}.`]),
+      "",
+      "Tier-1 escalations still land — as issue comments, the path this orchestrator",
+      "exists to avoid. Dispatch, settlement and release are unaffected; the operator",
+      "channel is degraded. Recovery sends one closing notice on this same incident.",
+    ].join("\n"),
+  };
+}
+
+/** The closing notice, referencing the same incident anchor. */
+export function recoveryEscalation(
+  project: string,
+  closed: OrchestratorIncident,
+  now: number,
+): Escalation {
+  const downtime = formatDownDuration(Math.max(0, now - closed.since));
+  return {
+    tier: 2,
+    category: "confirmed-failure",
+    urgent: true,
+    project,
+    issue: NO_ISSUE,
+    summary:
+      `Orchestrator recovered for ${project} ` +
+      `(down since ${iso(closed.since)}) — tier-1 escalations resume as injected prompts`,
+    detail: [
+      `Mode: ${closed.mode}. Downtime: ${downtime}.`,
+      `Tier-1 escalations diverted to issue comments while down: ${closed.diverted}.`,
+      "",
+      "The orchestrator is back and ticking. The .conductor-stalled wedge watchdog is",
+      "unchanged and still covers a session that stops draining its queue.",
+    ].join("\n"),
+  };
+}
+
+/**
+ * Reconcile the embedded orchestrator's liveness against the durable incident.
+ * Idempotent, so it is safe to call at startup (after the escalator exists) and
+ * on every tick: opening, paging, divert-counting and recovering all converge
+ * on the store.
+ *
+ * External orchestrators return early — their down state is a human-owned pane,
+ * not conductor's to page, and the stall watchdog still covers their wedge.
+ */
+export async function reconcileOrchestratorDown(deps: OrchestratorDownDeps): Promise<void> {
+  const { project, store } = deps;
+  const now = deps.now ?? Date.now;
+  const log = deps.log ?? warn;
+  if (project.escalation.orchestrator === "external") return;
+
+  const handle = deps.orchestrator;
+  const up = handle !== undefined && handle.alive();
+  const existing = store.orchestratorIncident(project.name);
+
+  if (up) {
+    if (existing === undefined) return;
+    // A session is (re)started and alive: close the incident from before and
+    // page one closing notice on the same anchor.
+    const closed = store.closeOrchestratorIncident(project.name, now());
+    if (closed === undefined) return; // raced with another daemon
+    const delivered = await deliver(deps, recoveryEscalation(project.name, closed, now()));
+    log(
+      `orchestrator recovered — closed incident from ${iso(closed.since)} ` +
+        `(${closed.diverted} diverted)${delivered ? "" : " (page unsent)"}`,
+    );
+    return;
+  }
+
+  // Down. Open once (the store's INSERT OR IGNORE is the dedupe), then page —
+  // the page itself is ledger-deduped so later ticks re-derive the world
+  // without paging again.
+  if (existing === undefined) {
+    const mode: OrchestratorDownMode = handle === undefined ? "start-failed" : "crashed";
+    if (
+      !store.openOrchestratorIncident({
+        project: project.name,
+        mode,
+        ...(mode === "start-failed"
+          ? deps.startCause === undefined
+            ? {}
+            : { cause: deps.startCause }
+          : { cause: "session died after a healthy start" }),
+        since: now(),
+      })
+    ) {
+      return; // raced with another daemon opening it
+    }
+  }
+  const incident = store.orchestratorIncident(project.name);
+  if (incident === undefined) return;
+  await deliver(deps, downEscalation(project.name, incident));
+  log(
+    `ERROR: orchestrator ${incident.mode} down since ${iso(incident.since)} — ` +
+      `tier-1 escalations diverting to issue comments`,
+  );
+}
+
+/**
+ * The first-class degrade row for `status`: mode, since-when, and how many
+ * tier-1 escalations have been diverted to issue comments. Absent entirely
+ * (empty array) when the orchestrator is healthy, so recovery drops the row.
+ */
+export function formatOrchestratorDown(row: OrchestratorIncident, now = Date.now()): string[] {
+  return [
+    `orchestrator  DEGRADED  ${row.mode} since ${iso(row.since)} ` +
+      `(${formatDownDuration(Math.max(0, now - row.since))} ago)`,
+    `  diverting  tier-1 escalations to issue comments` +
+      (row.diverted === 0
+        ? " (none yet)"
+        : ` — ${row.diverted} escalated issue${row.diverted === 1 ? "" : "s"} diverted`),
+    ...(row.cause === undefined ? [] : [`  cause      ${row.cause}`]),
+  ];
+}

package/src/orchestrator-tick.ts

@@ -71,6 +71,7 @@ import {
   releaseDriftDigestLine,
   releaseRefusal,
   releaseToolMatch,
+  sharedHostGuardDigestLine,
   type ReleaseDecision,
 } from "./release-policy.ts";
 import {
@@ -2346,6 +2347,12 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
   if (scope.projectName !== undefined) {
     const drift = releaseDriftDigestLine(scope.projectName);
     if (drift !== undefined) content = `${content}\n${drift}`;
+    // The shared-host gate's refusals are reported on their own line, apart
+    // from release drift (#562): a worker being stopped by the interlock is
+    // the guard working, never a release-policy divergence, and it must not
+    // move the drift counter or borrow the release-policy heading.
+    const guard = sharedHostGuardDigestLine(scope.projectName);
+    if (guard !== undefined) content = `${content}\n${guard}`;
     if (existsSync(dbPath())) {
       try {
         frictionStore = openStore(dbPath());

package/src/orchestrator.ts

@@ -76,6 +76,11 @@ export interface OrchestratorHandle {
    */
   deliver(e: Escalation, project: string): Promise<DeliveryReceipt>;
   busy(): boolean;
+  /** Whether the underlying session is still running. Event-driven, never a
+   *  timer: set false by the proxy's real `session_exit` terminal event, and
+   *  false once the handle has been disposed. A wedged-but-alive session still
+   *  reads alive here — that is the stall watchdog's job, not liveness. */
+  alive(): boolean;
   sessionFile(): string | undefined;
   dispose(): Promise<void>;
 }
@@ -216,6 +221,14 @@ export async function startOrchestrator(o: OrchestratorOpts): Promise<Orchestrat
   });
 
   let disposed = false;
+  // Read by the handle's `alive()`: a session that has ended is gone whether
+  // it crashed or was shut down, and from that instant nothing about it can be
+  // relied on. Driven by the proxy's `session_exit` terminal event — a real
+  // signal, not a timer, and not the `agent_end`/streaming bookkeeping above.
+  let terminated = false;
+  session.on("session_exit", () => {
+    terminated = true;
+  });
   /** Tail of the delivery queue. Always settled-or-settling, never rejected. */
   let queue: Promise<void> = Promise.resolve();
   /** Consumed by the first injection; see {@link OrchestratorOpts.brief}. */
@@ -273,6 +286,7 @@ export async function startOrchestrator(o: OrchestratorOpts): Promise<Orchestrat
       return accepted;
     },
     busy: () => streaming,
+    alive: () => !terminated && !disposed,
     // The path the harness actually opened, read live: the transcript is how a
     // human audits what the orchestrator decided on their behalf.
     sessionFile: () => session.sessionFile,

package/src/release-policy.ts

@@ -76,8 +76,10 @@ export type ReleaseDecision = { block: true; reason: string };
  * and a worker is always, whatever the config says. So it stays out of the
  * configurable grant vocabulary (`releaseShapeEnum` is drawn straight from
  * {@link RELEASE_SHAPES}); it only rides the release-policy classifier and the
- * block ledger so one seam sees it and its refusals land where the next tick
- * already reads drift.
+ * block ledger so one seam sees it. Its refusals land in the same ledger the
+ * tick reads, but the tick reports them apart from release drift (#562):
+ * a refusal here is the interlock working, never a release-policy violation
+ * and never something a grant could have permitted.
  */
 export const SHARED_HOST_SHAPE = "shared-host-gate" as const;
 export type SharedHostShape = typeof SHARED_HOST_SHAPE;
@@ -172,11 +174,66 @@ function commandSegments(command: string): string[] {
     .filter((segment) => segment.length > 0);
 }
 
+/** A leading wrapper command, matched and consumed in a chain (#558). `env`
+ *  was already stripped; `timeout`, `nice`, `stdbuf` and a shell `-c` were
+ *  not, so a whole-package run became allowed the moment it picked up a
+ *  wrapper — the incident ran `timeout 300 bun test` and the guard never
+ *  fired. Each alternative owns the arguments that belong to it (the
+ *  duration, the niceness, the option run), so stripping never swallows the
+ *  command's own words. Ordered longest-first so `timeout` is consumed
+ *  before `time` and `env` before a bare `VAR=…` chain. */
+const COMMAND_WRAPPER_PREFIX = new RegExp(
+  [
+    "^sudo(?:\\s+-[a-z][a-z0-9-]*)*\\s+",
+    "^env\\s+",
+    "^(?:[A-Za-z_][A-Za-z0-9_]*=\\S+\\s+)+",
+    "^timeout(?:\\s+--?[a-z][a-z0-9-]*(?:=\\S+|\\s+\\S+)?)*\\s+(?:inf|infinity|\\d+(?:\\.\\d+)?[smhd]?)\\s+",
+    "^nice(?:\\s+-n\\s+-?\\d+|\\s+-\\d+|\\s+--adjustment\\s*=\\s*-?\\d+)?\\s+",
+    "^ionice(?:\\s+--?[a-z][a-z0-9-]*(?:\\s+\\d+)?)*\\s+",
+    "^stdbuf(?:\\s+--?[a-z][a-z0-9-]*(?:=\\S+|[A-Za-z0-9]+|\\s+[A-Za-z0-9])?)*\\s+",
+    "^xargs(?:\\s+--?[A-Za-z][A-Za-z0-9-]*(?:=\\S+|\\s+\\S+)?)*\\s+",
+    "^time\\s+",
+  ].join("|"),
+);
+
+/** `bash -c`, `sh -c`, `zsh -c` with optional preceding flags — `-lc` and
+ *  friends parse as combined short options, so the `c` may ride in the same
+ *  token as the flags before it. A `bash -n` parse check is deliberately not
+ *  one: it executes nothing. */
+const SHELL_DASH_C = /^(?:bash|sh|zsh)\s+(?:-\S+\s+)*?-[A-Za-z]*c\b\s+/;
+
+/**
+ * The command a `bash -c '<cmd>'` / `sh -c "<cmd>"` segment executes is the
+ * quoted string, not the `bash` invocation, so a wrapper command that reaches
+ * for a shell must be resolved to the string inside the quotes before
+ * classifying. Extra `$0`.. arguments after the closing quote are ignored:
+ * they are not the command.
+ */
+function shellCommandInner(segment: string): string | undefined {
+  const shell = SHELL_DASH_C.exec(segment);
+  if (shell === null) return undefined;
+  const rest = segment.slice(shell[0].length);
+  const quote = rest[0];
+  if (quote !== "'" && quote !== '"') return undefined;
+  const close = rest.indexOf(quote, 1);
+  if (close < 0) return undefined;
+  return rest.slice(1, close);
+}
+
+/** Normalise a segment before classification: consume every leading wrapper
+ *  command (#558). The old stripper recognised `env` and `sudo` and nothing
+ *  else, so a whole-package `bun test` became allowed the moment a worker
+ *  wrapped it in `timeout`/`nice`/`bash -c`. Chains matter — `timeout 300
+ *  nice bun test` is one command with two wrappers — so the table is matched
+ *  repeatedly until the leading word belongs to the command itself.
+ */
 function stripCommandPrefix(segment: string): string {
-  return segment
-    .replace(/^env\s+/, "")
-    .replace(/^(?:[A-Za-z_][A-Za-z0-9_]*=\S+\s+)*/, "")
-    .replace(/^sudo\s+/, "");
+  let stripped = segment;
+  let match: RegExpExecArray | null;
+  while ((match = COMMAND_WRAPPER_PREFIX.exec(stripped)) !== null) {
+    stripped = stripped.slice(match[0].length);
+  }
+  return stripped;
 }
 
 /** Recognise the explicit release/deploy command shapes this policy promises to gate. */
@@ -250,6 +307,17 @@ function sharedHostScriptMatch(segment: string): string | undefined {
 function releaseCommandMatch(command: string): { shape: GateShape; matched: string } | undefined {
   for (const raw of commandSegments(command)) {
     const segment = stripCommandPrefix(raw);
+    // A `bash -c '<cmd>'` / `sh -c "<cmd>"` wrapper executes the quoted
+    // string, so classify that string as a command of its own: a whole
+    // package inside the quotes is refused (#558), a focused run inside
+    // the quotes stays a focused run, and a harmless `-c` script body
+    // (the pre-push `bash -n` parse gate) never unwraps to begin with.
+    const inner = shellCommandInner(segment);
+    if (inner !== undefined) {
+      const nested = releaseCommandMatch(inner);
+      if (nested !== undefined) return nested;
+      continue;
+    }
     // Patching the running conductor is gated like a release act: `upgrade`,
     // the detached `upgrade-install`/`upgrade-rollback` executor entries and
     // any future spelling under the upgrade family all need the install shape,
@@ -677,21 +745,21 @@ export interface ReleaseDriftSummary {
   latest: ReleaseBlock;
 }
 
-/** Aggregate today's blocked attempts for the orchestrator's daily digest. */
-export function releaseDriftToday(
-  project: string,
-  root = stateDir(),
-  now = new Date(),
-): ReleaseDriftSummary | undefined {
+/**
+ * The `release-policy-blocks.jsonl` rows recorded for `project` on `now`'s
+ * UTC day, in file order. One scan backs both day summaries, so the release
+ * drift counter and the shared-host counter can never disagree about which
+ * ledger rows are in play (#562).
+ */
+function auditRowsToday(project: string, root: string, now: Date): ReleaseBlock[] {
   let text: string;
   try {
     text = readFileSync(join(root, RELEASE_POLICY_AUDIT_FILE), "utf8");
   } catch {
-    return undefined;
+    return [];
   }
   const day = now.toISOString().slice(0, 10);
-  let count = 0;
-  let latest: ReleaseBlock | undefined;
+  const rows: ReleaseBlock[] = [];
   for (const line of text.split("\n")) {
     if (line.length === 0) continue;
     try {
@@ -703,14 +771,58 @@ export function releaseDriftToday(
         (value.source === "worker" || value.source === "orchestrator") &&
         typeof value.shape === "string"
       ) {
-        count += 1;
-        latest = value as ReleaseBlock;
+        rows.push(value as ReleaseBlock);
       }
     } catch {
       // One torn line does not hide later valid audit records.
     }
   }
-  return latest === undefined ? undefined : { count, latest };
+  return rows;
+}
+
+/**
+ * Aggregate today's blocked release/deploy attempts for the orchestrator's
+ * daily digest. The shared-host gate (#428) is deliberately excluded (#562):
+ * a guard refusal is the interlock working, never a divergence from release
+ * policy, and no grant could ever have permitted the command — so counting a
+ * refusal here would report a success as a violation. {@link
+ * sharedHostRefusalsToday} reads the same ledger and reports those rows on
+ * their own line, so nothing is dropped.
+ */
+export function releaseDriftToday(
+  project: string,
+  root = stateDir(),
+  now = new Date(),
+): ReleaseDriftSummary | undefined {
+  const drift = auditRowsToday(project, root, now).filter((row) => row.shape !== SHARED_HOST_SHAPE);
+  const latest = drift[drift.length - 1];
+  if (latest === undefined) return undefined;
+  return { count: drift.length, latest };
+}
+
+export interface SharedHostGuardSummary {
+  count: number;
+  latest: ReleaseBlock;
+}
+
+/**
+ * Aggregate today's shared-host gate refusals (#562), apart from release
+ * drift. Every row records a worker being correctly stopped — the whole
+ * package runs and host shell suites that once SIGTERMed the production
+ * daemon — so the count and the latest {@link ReleaseBlock#matched matched
+ * command} are the operator's signal when a worker keeps pushing at the gate
+ * (tonight's clustering is how #558's wrapper evasion was found), and they
+ * must never ride the release-policy line.
+ */
+export function sharedHostRefusalsToday(
+  project: string,
+  root = stateDir(),
+  now = new Date(),
+): SharedHostGuardSummary | undefined {
+  const blocked = auditRowsToday(project, root, now).filter((row) => row.shape === SHARED_HOST_SHAPE);
+  const latest = blocked[blocked.length - 1];
+  if (latest === undefined) return undefined;
+  return { count: blocked.length, latest };
 }
 
 export function releaseDriftDigestLine(
@@ -735,3 +847,36 @@ export function releaseDriftDigestLine(
     "Include this divergence from releasePolicy=none in today's digest."
   );
 }
+
+/**
+ * The tick/digest line for the shared-host guard (#562), deliberately apart
+ * from {@link releaseDriftDigestLine}: a shared-host refusal is a worker
+ * being correctly stopped, never a release-policy divergence, and naming a
+ * release grant here would misattribute an event no grant covers (#549). A
+ * working guard reads as a working guard; the guard name and the matched
+ * command are the genuinely useful parts, because a clustering of refusals
+ * is itself a signal.
+ */
+export function sharedHostGuardDigestLine(
+  project: string,
+  root = stateDir(),
+  now = new Date(),
+): string | undefined {
+  const refusal = sharedHostRefusalsToday(project, root, now);
+  if (refusal === undefined) return undefined;
+  const latest = refusal.latest;
+  const attribution = [
+    `${latest.source} ${latest.shape} at ${latest.at}`,
+    ...(latest.issue === undefined ? [] : [`issue #${latest.issue}`]),
+    ...(latest.runId === undefined ? [] : [`run ${latest.runId}`]),
+    ...(latest.tool === undefined ? [] : [`tool ${latest.tool}`]),
+    ...(latest.invocation === undefined ? [] : [`attempted "${latest.invocation}"`]),
+    ...(latest.matched === undefined ? [] : [`matched "${latest.matched}"`]),
+  ].join(", ");
+  return (
+    `The shared-host guard refused ${refusal.count} worker command(s) today ` +
+    `(latest: ${attribution}). ` +
+    "This is the shared-host interlock, not a release-policy divergence: a worker " +
+    "is refused these whatever grants it holds; the operator owns the host and CI owns the full suite."
+  );
+}