omp-conductor 0.17.1 → 0.18.1

package/src/failure-class.ts

@@ -17,9 +17,14 @@
 
 import type { Caps, FailureClass, RecoveryAction, RunRecord } from "./types.ts";
 
-/** Facts the caller fetched, each only for the rows that need it. */
+/** Facts the caller fetched, each only for the rows that need it.
+ *
+ *  `pr` can also be `"missing"`: the tracker has proven the claimed PR
+ *  definitively does not exist (#779). It is a fact, the same way `"closed"`
+ *  is — a REST 404 the adapter corroborated with a same-repository pulls-list
+ *  read — never a transient "could not tell", which stays `undefined`. */
 export interface ClassifyFacts {
-  pr?: "open" | "merged" | "closed";
+  pr?: "open" | "merged" | "closed" | "missing";
   mergeable?: "conflicting" | "clean" | "unknown";
   checks?: { name: string; state: string; link?: string }[];
   /** Full session error recovered from the transcript. Kept as a fact so a
@@ -142,6 +147,12 @@ const START_FAILURE_SIGNATURES = [
   "invalid api key",
   "authentication failed",
   "could not load its peer dependency",
+  // The pre-launch identity gate (#798/#828): the account is missing, or the
+  // harness binding a worker resolves through is not established. The daemon
+  // refuses before it spawns anything, so no attempt was spent — and the
+  // symptom this replaced (the child dying on the peer import) already read as
+  // a start failure, so charging one here would be a regression dressed as a fix.
+  "worker identity unavailable",
 ] as const;
 
 /**
@@ -188,10 +199,23 @@ export function providerCreditRefusal(error: {
 }
 
 /** Provider text that names a per-request stream fault. Deliberately narrow:
- *  a 429 is rate limiting and a 402 is credit — different remedies (#220). */
+ *  a 429 is rate limiting and a 402 is credit — different remedies (#220).
+ *
+ *  The structural `kind` that `readSessionError` marks outranks this list: a
+ *  transcript record the harness itself attributed to a provider abort is a
+ *  per-request stream fault whatever the provider called it, so no vendor
+ *  prose needs enumerating here (#743). The list remains for transcripts that
+ *  only ever carried prose, like the original "stream stalled" stall. */
 const TRANSIENT_FAULT_SIGNATURES = ["stream stalled"] as const;
 
-export function providerTransientFault(error: { status?: number; message: string }): string | undefined {
+export function providerTransientFault(error: {
+  status?: number;
+  message: string;
+  kind?: "provider-stream";
+}): string | undefined {
+  if (error.kind === "provider-stream") {
+    return error.message.split("\n")[0]?.trim() ?? error.message;
+  }
   const text = error.message.toLowerCase();
   if (!TRANSIENT_FAULT_SIGNATURES.some((s) => text.includes(s))) return undefined;
   return error.message.split("\n")[0]?.trim() ?? error.message;
@@ -499,6 +523,21 @@ export function classifyRun(
   if (run.state === "failed" && facts.pr === "open") {
     const checks = facts.checks ?? [];
     const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true);
+    if (checks.length > 0 && unresolved.length === 0) {
+      // The PR is open and every check is green: the strongest mechanical
+      // success evidence there is short of merge. The row's `failed` state is
+      // stale — whatever the worker recorded, its work is passing — so this is
+      // not a failure of any class. `none` recovery restores it to
+      // `pushed-green` in classifyAndRecover instead of escalating an
+      // `[unknown]` that charges an attempt (#766). An empty check list is
+      // "could not tell", not all-green, and still falls through to `unknown`
+      // below.
+      return {
+        cls: "unknown",
+        recovery: "none",
+        evidence: `${run.prUrl ?? "the PR"} is open and every check is green`,
+      };
+    }
     if (checks.length > 0 && unresolved.length > 0) {
       // A failed check whose *log* smells like infrastructure — a registry 429,
       // a runner shutdown, a DNS failure (#177). The check has a verdict, so

package/src/fleet.ts

@@ -50,8 +50,24 @@ import { inspectBriefLayout } from "./brief-upgrade.ts";
 import { dbPath, openStore } from "./store.ts";
 import { renderBriefForProject } from "./setup.ts";
 import { DEFAULT_ARM_PROOF, type ArmProof, type DaemonStop, type ProjectConfig, type Store } from "./types.ts";
-import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
+import { DEFAULT_DEPS, probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
 import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
+// The drain surface belongs beside hold/releaseHold on the operator surface:
+// the record is daemon state (implemented next to the pause sentinel), but the
+// later CLI and release #484 children import from here, exactly where they find
+// every other fleet control.
+export {
+  cancelDrain,
+  consumeDrain,
+  createDrain,
+  drainPath,
+  readDrain,
+  type CreateDrainOptions,
+  type DrainProblem,
+  type DrainRecord,
+  type DrainStatus,
+  type DrainVerdict,
+} from "./daemon.ts";
 import {
   healthCheck,
   isAlive,
@@ -59,6 +75,7 @@ import {
   probeUnit,
   runSystemctl,
   stopDaemon,
+  type HealthCheckResult,
   type StopResult,
 } from "./lifecycle.ts";
 import type { WorkerPausePhase } from "./worker.ts";
@@ -1053,21 +1070,15 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
   return out;
 }
 
-async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
-  const bin = deps.herdrBin ?? "herdr";
-  const session =
-    deps.herdrSession ??
-    resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
-  const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
-    encoding: "utf8",
-    timeout: 8_000,
-    env: process.env,
-  });
-  if (res.error) throw res.error;
-  if (res.status !== 0) {
-    throw new Error((res.stderr ?? res.stdout ?? `process-info exit ${String(res.status)}`).trim());
-  }
-  const raw = (res.stdout ?? "").trim();
+/**
+ * The `process_info` document out of `herdr pane process-info` output
+ * (#832): every caller that maps the fleet pane's live omp processes starts
+ * from this schema, whether the output carries herdr's CLI `result` envelope
+ * or the bare document. Throws when the output cannot be read; the caller
+ * decides what an unreadable answer means.
+ */
+export function parseHerdrProcessInfo(stdout: string, paneId: string): ProcessInfo {
+  const raw = stdout.trim();
   if (raw.length === 0) {
     throw new Error(`herdr pane process-info printed nothing for ${paneId}`);
   }
@@ -1083,10 +1094,27 @@ async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promi
   if (info === undefined) {
     throw new Error(`herdr pane process-info has no process_info for ${paneId} — unrecognized schema`);
   }
-  return ompPidsFromProcessInfo(info);
+  return info;
+}
+
+async function herdrOmpForegroundPids(paneId: string, deps: PaneStopDeps): Promise<number[]> {
+  const bin = deps.herdrBin ?? "herdr";
+  const session =
+    deps.herdrSession ??
+    resolveHerdrSessionWithBridge({ env: process.env, herdrBin: bin });
+  const res = spawnSync(bin, ["--session", session, "pane", "process-info", "--pane", paneId], {
+    encoding: "utf8",
+    timeout: 8_000,
+    env: process.env,
+  });
+  if (res.error) throw res.error;
+  if (res.status !== 0) {
+    throw new Error((res.stderr ?? res.stdout ?? `process-info exit ${String(res.status)}`).trim());
+  }
+  return ompPidsFromProcessInfo(parseHerdrProcessInfo(res.stdout ?? "", paneId));
 }
 
-interface ProcessInfo {
+export interface ProcessInfo {
   shell_pid?: number;
   foreground_processes?: ForegroundProc[];
 }
@@ -1121,6 +1149,85 @@ function isOmpProcess(proc: ForegroundProc): boolean {
   return false;
 }
 
+/**
+ * The omp processes the fleet pane claims, resolved to their start times
+ * (#832), or why they could not be read. Structurally identical to
+ * upgrade-verify's `PaneOmpProbe` — the upgrade engine and the post-restart
+ * verifier pass this straight into the pure verdicts, and the type is
+ * repeated here because the verifier leaf must not import this module (fleet
+ * reaches the daemon, which reaches the verifier).
+ */
+export type PaneProbeResult = { starts: readonly number[] } | { problem: string };
+
+/**
+ * Every omp process the fleet pane currently claims by herdr, resolved to
+ * its start time (#832): the pane is where an *external* orchestrator
+ * session lives, and the only fact that proves it reloaded is the live
+ * process's own start — a pane process still running from before the install
+ * began loaded the pre-upgrade extension. Every unreadable answer is a
+ * `problem`: absence of evidence is never a reload.
+ *
+ * `run` is the injected command runner (the upgrade's scripted seam, the
+ * daemon's `runCommand`), and `startTime` resolves one pid to its start —
+ * the /proc read lives with the verifier's other live-process facts.
+ */
+export async function herdrPaneOmpStarts(
+  run: (
+    command: string,
+    args: readonly string[],
+  ) => Promise<{ code: number; stdout: string; stderr: string }>,
+  session: string,
+  startTime: (pid: number) => number | undefined,
+): Promise<PaneProbeResult> {
+  const agents = await run("herdr", ["--session", session, "agent", "list"]);
+  if (agents.code !== 0) {
+    return {
+      problem:
+        `herdr agent list failed: ${agents.stderr.trim() || agents.stdout.trim() || `exit ${agents.code}`}`,
+    };
+  }
+  let parsed: HerdrAgent[];
+  try {
+    parsed = parseHerdrAgentList(agents.stdout);
+  } catch (err) {
+    return { problem: err instanceof Error ? err.message : String(err) };
+  }
+  const starts: number[] = [];
+  for (const agent of parsed) {
+    // A name without a live omp claim is a leftover label, not a pane process.
+    if (agent.agent === undefined) continue;
+    const info = await run("herdr", [
+      "--session",
+      session,
+      "pane",
+      "process-info",
+      "--pane",
+      agent.paneId,
+    ]);
+    if (info.code !== 0) {
+      return {
+        problem:
+          `herdr pane process-info for ${agent.paneId} failed: ` +
+          `${info.stderr.trim() || info.stdout.trim() || `exit ${info.code}`}`,
+      };
+    }
+    let pids: number[];
+    try {
+      pids = ompPidsFromProcessInfo(parseHerdrProcessInfo(info.stdout, agent.paneId));
+    } catch (err) {
+      return { problem: err instanceof Error ? err.message : String(err) };
+    }
+    for (const pid of pids) {
+      const startedAt = startTime(pid);
+      if (startedAt === undefined) {
+        return { problem: `cannot read the start time of pane process ${pid} — reload unproven` };
+      }
+      starts.push(startedAt);
+    }
+  }
+  return { starts };
+}
+
 /**
  * The project a bare read means, when the config leaves no doubt.
  *
@@ -1143,8 +1250,26 @@ function bareReadProject(): string | undefined {
 
 export function fleetLayers(projectName?: string): FleetLayers {
   const rec = livingDaemon();
+  // systemd is the authoritative liveness witness for a unit-owned daemon: a
+  // pidfile that is missing, stale, or skewed against the unit's MainPID must
+  // not declare the daemon dead (#716). Consulted only when the pidfile says
+  // dead — a healthy record needs no shell-out, and a host without systemctl
+  // answers `inactive`, so those hosts behave exactly as before. `failed`,
+  // `unknown`, and a mid-restart unit (MainPID 0, process gone) all answer
+  // undefined: a genuinely stopped daemon never reports running.
+  let unitPid: number | undefined;
+  if (rec === undefined) {
+    const ownership = probeUnit();
+    if (ownership.kind === "active" && isAlive(ownership.pid)) unitPid = ownership.pid;
+  }
+  const daemon: FleetLayers["daemon"] =
+    rec !== undefined
+      ? { running: true, pid: rec.pid, port: rec.port }
+      : unitPid !== undefined
+        ? { running: true, pid: unitPid }
+        : { running: false };
   const paused = isPaused(projectName ?? bareReadProject());
-  const dispatch: DispatchLayer = rec === undefined ? "stopped" : paused ? "paused" : "running";
+  const dispatch: DispatchLayer = daemon.running ? (paused ? "paused" : "running") : "stopped";
 
   const tick = resolveTickConfig(projectName);
   let ticks: TicksLayer;
@@ -1224,7 +1349,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
     ...(tickConfigPath === undefined ? {} : { tickConfigPath }),
     ...(haltPath === undefined ? {} : { paneHaltPath: haltPath }),
     paused,
-    daemon: rec === undefined ? { running: false } : { running: true, pid: rec.pid, port: rec.port },
+    daemon,
   };
 }
 
@@ -1337,7 +1462,7 @@ export function workerPhasesFromHealthz(
  */
 export function classifyDaemonProjectHealth(
   record: { project?: string } | undefined,
-  health: { ok: boolean; body?: string } | undefined,
+  health: HealthCheckResult | undefined,
   project: string,
 ): DaemonProjectHealth {
   if (record === undefined) return { kind: "stopped" };
@@ -1347,7 +1472,13 @@ export function classifyDaemonProjectHealth(
   if (record.project !== undefined && record.project !== project) {
     return { kind: "other-project", serves: record.project };
   }
-  if (health?.ok !== true) return { kind: "unreachable" };
+  // #685: a timed-out probe is a probe outcome, never a death verdict — the
+  // pid is up but wedged, so it must not read as `not running`. Only a
+  // refusal or other failure (nothing listening, a torn answer) stays
+  // `unreachable`.
+  if (health?.ok !== true) {
+    return health?.failure === "timeout" ? { kind: "unresponsive" } : { kind: "unreachable" };
+  }
   try {
     const payload = JSON.parse(health.body ?? "null") as unknown;
     if (payload === null || typeof payload !== "object") {
@@ -1437,7 +1568,12 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
         };
   const healthBody = projectHealth.kind === "ok" ? rawHealth?.body : undefined;
   const cached = codeGraphFromHealthz(healthBody, project.name);
-  const codeGraph = cached ?? (await probeCodeGraph(project));
+  // The runtime half of the code-graph finding (#726): the probe reads the
+  // store-backed per-run observations, so "observed" is always grounded in
+  // dispatched runs, never in the daemon's own process or in mcp.json. The
+  // store stays scoped to the same try/finally as the other reads so a
+  // throwing probe cannot leak its handle.
+  const store = openStore(dbPath());
   const workerPhases = [...workerPhasesFromHealthz(healthBody, project.name)].map(
     ([issue, phase]) => ({ issue, phase }),
   );
@@ -1446,10 +1582,16 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
   // and rendered identically from either project: the daemon_stops table is
   // deliberately not partitioned by project, because the daemon serves every
   // project and the uninvolved one must see who stopped it too.
-  const store = openStore(dbPath());
+  let codeGraph: CodeGraphHealth;
   let lastStop: DaemonStop | undefined;
   let siblings: { project: string; live: number }[] = [];
   try {
+    codeGraph =
+      cached ??
+      (await probeCodeGraph(project, {
+        ...DEFAULT_DEPS,
+        graphToolsObservations: () => store.graphToolsObservationCounts(project.name),
+      }));
     lastStop = store.latestDaemonStop();
     // Shared-daemon visibility (#545): every configured project other than the
     // one being viewed, with its live-run count.