omp-conductor 0.19.7 → 0.20.0

package/src/daemon/http.ts

@@ -0,0 +1,417 @@
+/**
+ * The loopback HTTP surface: the daemon's only in-band control plane.
+ *
+ * One module because there is exactly one authorisation rule and it must be
+ * impossible to add a route that skips it — every mutating path answers behind
+ * the bearer check here, `/healthz` is the single deliberate exception, and
+ * `daemonHttpResponse` is the one function that decides which is which.
+ *
+ * `DaemonHttpDeps` stays with the surface rather than in `deps.ts`: it is not
+ * what a tick needs, it is the much smaller set a request needs, and its
+ * `health` field types a projection from `views.ts`. Hoisting it into the shared
+ * floor would make the seam module depend on the view layer to describe a
+ * request.
+ */
+import { httpTokenPath, verifyHttpToken } from "../http-token.ts";
+import { log } from "../log.ts";
+import { LIVE_STATES } from "../store.ts";
+import type { Caps, RunState, Store } from "../types.ts";
+import type { TurnLimitRegistry, WorkerControlRegistry } from "./deps.ts";
+import type { DaemonHealth } from "./views.ts";
+
+export const TURN_OVERRIDE_STATES: ReadonlySet<RunState> = new Set([
+  "failed",
+  "killed",
+  "orphaned",
+  "blocked",
+]);
+
+export async function turnLimitResponse(
+  req: Request,
+  project: string,
+  store: Pick<Store, "latestRun" | "setTurnOverride">,
+  registry: TurnLimitRegistry,
+  caps: Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">,
+): Promise<Response | undefined> {
+  const url = new URL(req.url);
+  const match = /^\/runs\/(\d+)\/turn-limit$/.exec(url.pathname);
+  if (req.method !== "PUT" || match === null) return undefined;
+  if (!req.headers.get("content-type")?.startsWith("application/json")) {
+    return Response.json({ error: "content-type must be application/json" }, { status: 415 });
+  }
+
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
+  }
+  if (body === null || typeof body !== "object") {
+    return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
+  }
+  const requestedProject = Reflect.get(body, "project");
+  if (typeof requestedProject !== "string" || requestedProject.length === 0) {
+    return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
+  }
+  if (requestedProject !== project) {
+    return Response.json(
+      { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
+      { status: 409 },
+    );
+  }
+  const maxTurns = Reflect.get(body, "maxTurns");
+  if (!Number.isSafeInteger(maxTurns) || (maxTurns as number) < 1) {
+    return Response.json({ error: "maxTurns must be a positive integer" }, { status: 400 });
+  }
+
+  const issue = Number(match[1]);
+  if ((maxTurns as number) > caps.workerMaxTurnsCeiling) {
+    return Response.json(
+      {
+        error:
+          `#${issue} turn budget ${maxTurns as number} exceeds the ` +
+          `${caps.workerMaxTurnsCeiling}-turn caps.workerMaxTurnsCeiling`,
+      },
+      { status: 422 },
+    );
+  }
+  const outcome = registry.extend(project, issue, maxTurns as number);
+  if (outcome.kind === "extended") return Response.json(outcome);
+  if (outcome.kind === "not-increase") {
+    return Response.json(
+      { error: `#${issue} already has a ${outcome.maxTurns}-turn ceiling` },
+      { status: 409 },
+    );
+  }
+
+  const latest = store.latestRun(project, issue);
+  if (latest === undefined) {
+    return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
+  }
+  if (!TURN_OVERRIDE_STATES.has(latest.state)) {
+    return Response.json(
+      {
+        error:
+          `#${issue} has no live worker controller; its session already settled ` +
+          `or belongs to another daemon (stored state: ${latest.state})`,
+      },
+      { status: 409 },
+    );
+  }
+  if ((maxTurns as number) <= caps.workerMaxTurns) {
+    return Response.json(
+      {
+        error:
+          `#${issue} next-attempt turn budget must exceed the ` +
+          `${caps.workerMaxTurns}-turn caps.workerMaxTurns base`,
+      },
+      { status: 409 },
+    );
+  }
+  store.setTurnOverride(project, issue, maxTurns as number);
+  return Response.json({
+    kind: "next-attempt",
+    issue,
+    nextAttemptMaxTurns: maxTurns,
+    baseMaxTurns: caps.workerMaxTurns,
+  });
+}
+
+export async function workerControlResponse(
+  req: Request,
+  project: string,
+  store: Pick<Store, "latestRun" | "updateRun" | "recordMaterialEvent">,
+  registry: WorkerControlRegistry,
+  /** Journal line for every accepted pause/resume — the live half of the
+   *  audit trail #997 asks for; the durable half is the material event. */
+  log: (line: string) => void = () => {},
+  now: () => number = Date.now,
+): Promise<Response | undefined> {
+  const url = new URL(req.url);
+  const match = /^\/runs\/(\d+)\/(pause|resume|stop)$/.exec(url.pathname);
+  if (req.method !== "PUT" || match === null) return undefined;
+  if (!req.headers.get("content-type")?.startsWith("application/json")) {
+    return Response.json({ error: "content-type must be application/json" }, { status: 415 });
+  }
+
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: "request body must be valid JSON" }, { status: 400 });
+  }
+  if (body === null || typeof body !== "object") {
+    return Response.json({ error: "request body must be a JSON object" }, { status: 400 });
+  }
+  const requestedProject = Reflect.get(body, "project");
+  if (typeof requestedProject !== "string" || requestedProject.length === 0) {
+    return Response.json({ error: "project must be a non-empty string" }, { status: 400 });
+  }
+  if (requestedProject !== project) {
+    return Response.json(
+      { error: `daemon serves project "${project}", not requested project "${requestedProject}"` },
+      { status: 409 },
+    );
+  }
+  const action = match[2] as "pause" | "resume" | "stop";
+  let reason: string | undefined;
+  if (action === "stop") {
+    const rawReason = Reflect.get(body, "reason");
+    if (typeof rawReason !== "string" || rawReason.trim() === "") {
+      return Response.json({ error: "reason must be a non-empty string" }, { status: 400 });
+    }
+    reason = rawReason.trim().replace(/\s+/g, " ");
+    if (reason.length > 500) {
+      return Response.json({ error: "reason must be at most 500 characters" }, { status: 400 });
+    }
+  }
+  // Who asked. The board, the CLI and the dashboard each name themselves so a
+  // pause is attributable afterwards (#997); an older caller that sends no
+  // source is recorded as such rather than guessed at.
+  const rawSource = Reflect.get(body, "source");
+  if (rawSource !== undefined && (typeof rawSource !== "string" || rawSource.trim() === "" || rawSource.trim().length > 40)) {
+    return Response.json({ error: "source must be a non-empty string of at most 40 characters when present" }, { status: 400 });
+  }
+  const source = typeof rawSource === "string" ? rawSource.trim() : "unattributed";
+
+
+  const issue = Number(match[1]);
+  const outcome =
+    action === "pause"
+      ? await registry.pause(project, issue, source)
+      : action === "resume"
+        ? registry.resume(project, issue)
+        : await registry.stop(project, issue, reason!);
+  if (outcome.kind === "ok") {
+    // The transition really happened — write both halves of the audit trail
+    // before answering (#997): one journal line for the live log, one durable
+    // material event the digest ledger keeps after the journal rotates. #986
+    // was parked 14 minutes by an unlogged keypress and nobody could say why.
+    const verb = action === "pause" ? "paused" : "resumed";
+    log(`#${issue} worker ${verb} via ${source} (run ${outcome.runId})`);
+    try {
+      store.recordMaterialEvent({
+        project,
+        category: "worker-control",
+        summary: `#${issue} worker ${verb} via ${source}`,
+        evidence: `run ${outcome.runId}, phase ${outcome.phase}`,
+        occurredAt: now(),
+        recordedAt: now(),
+      });
+    } catch (err) {
+      // The audit must never turn a successful transition into an error
+      // answer, but a swallowed write would be a silent audit gap — log it.
+      log(`#${issue} worker-control audit write failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return Response.json({ runId: outcome.runId, phase: outcome.phase });
+  }
+  if (outcome.kind === "refused") {
+    return Response.json({ error: `#${issue}: ${outcome.error}` }, { status: 409 });
+  }
+  if (outcome.kind === "stopped") {
+    const stopped = store.latestRun(project, issue);
+    if (stopped === undefined || stopped.id !== outcome.runId) {
+      return Response.json(
+        { error: `#${issue} stopped, but its terminal run record is unavailable` },
+        { status: 500 },
+      );
+    }
+    if (stopped.state !== "stopped") {
+      return Response.json({
+        outcome: "already-terminal",
+        runId: stopped.id,
+        state: stopped.state,
+      });
+    }
+    return Response.json({
+      outcome: "stopped",
+      runId: stopped.id,
+      state: stopped.state,
+      reason: outcome.reason,
+      ...(stopped.salvageSha === undefined ? {} : { salvageSha: stopped.salvageSha }),
+      ...(stopped.salvageError === undefined ? {} : { salvageError: stopped.salvageError }),
+      worktree: stopped.worktree,
+    });
+  }
+
+
+  const latest = store.latestRun(project, issue);
+  if (latest === undefined) {
+    return Response.json({ error: `no run recorded for #${issue}` }, { status: 404 });
+  }
+  if (action === "stop" && !LIVE_STATES.includes(latest.state)) {
+    return Response.json({
+      outcome: "already-terminal",
+      runId: latest.id,
+      state: latest.state,
+    });
+  }
+  // A live-state row whose controller is absent from this process is a ghost
+  // (#431): a session controller lives in the daemon that launched it, and once
+  // ownership is exclusive (the once lease makes a drill hold the same pidfile
+  // as the long dispatcher), a row that is live but has no controller in *this*
+  // owner is a row whose owner already died — not work another live daemon is
+  // still writing to. Refuse nothing: recover it as an orphan (worktree kept
+  // for triage, labels untouched, live slot freed) so the issue can be
+  // dispatched again and the row stops counting as a live worker.
+  if (action === "stop") {
+    store.updateRun(latest.id, {
+      state: "orphaned",
+      endedAt: Date.now(),
+      lastError: reason,
+    });
+    return Response.json({
+      outcome: "stopped",
+      runId: latest.id,
+      state: "orphaned",
+      reason: reason!,
+      worktree: latest.worktree,
+    });
+  }
+  return Response.json(
+    {
+      error:
+        `#${issue} has no live worker controller; its session already settled ` +
+        `or belongs to another daemon (stored state: ${latest.state})`,
+    },
+    { status: 409 },
+  );
+}
+
+export interface DaemonHttpProjectDeps {
+  project: string;
+  store: Pick<Store, "latestRun" | "setTurnOverride" | "updateRun" | "recordMaterialEvent">;
+  caps: () => Pick<Caps, "workerMaxTurns" | "workerMaxTurnsCeiling">;
+  /** Project-scoped journal line — the live half of the pause audit (#997). */
+  log?: (line: string) => void;
+}
+
+export interface DaemonHttpDeps {
+  projects: readonly DaemonHttpProjectDeps[];
+  turnLimits: TurnLimitRegistry;
+  workerControls: WorkerControlRegistry;
+  health: () => DaemonHealth;
+  /** Request an immediate dispatch pass (`resume`'s wake, #380). */
+  wake: () => void;
+  /**
+   * The bearer token every mutating route requires (Phase 4). Minted by
+   * `runDaemon` through {@link ensureHttpToken} before this server exists, so
+   * it is a value rather than a thunk: one daemon start, one token, and no
+   * route can be answered against a token that arrived later than the request.
+   *
+   * An empty string authenticates nobody — {@link verifyHttpToken} refuses it —
+   * so a deps literal that forgets the token fails closed rather than open.
+   */
+  token: string;
+  /** Journal line for a refused request. Defaults to the daemon's own log. */
+  log?: (line: string) => void;
+}
+
+/**
+ * The whole HTTP surface, in one named function so a test can pin what is *not*
+ * on it.
+ *
+ * Four route families: the health read, the dispatch wake, turn-limit control,
+ * and live-worker pause/resume/stop control. Everything else is 404. The
+ * controls mutate only daemon-owned live sessions; tracker and repository
+ * mutations stay on the authenticated per-run channel described at the
+ * `Bun.serve` call (#126).
+ *
+ * ## Authentication, and the one deliberate asymmetry
+ *
+ * Everything except `GET /healthz` requires `Authorization: Bearer <token>`
+ * (Phase 4). The gate sits here, above route dispatch, rather than in each
+ * route: the surface is "authenticated except the health read", and a rule
+ * stated once cannot be forgotten by the next route somebody adds. An
+ * unmatched path therefore also answers 401 rather than 404 — fail closed, and
+ * an unauthenticated caller learns nothing about which paths exist.
+ *
+ * `GET /healthz` stays open, and MUST stay open: `requireDaemonControl`
+ * (`lifecycle.ts`) probes it to decide whether a live unit-owned daemon exists
+ * at all, which happens *before* any caller has a reason — or a way — to hold
+ * this token. A health read mutates nothing, so the asymmetry costs nothing.
+ * Do not "fix" it by authenticating `/healthz`: that breaks daemon discovery
+ * for every CLI command, including the ones that would then be told to
+ * authenticate.
+ */
+export async function daemonHttpResponse(req: Request, d: DaemonHttpDeps): Promise<Response> {
+  const url = new URL(req.url);
+  if (req.method === "GET" && url.pathname === "/healthz") return Response.json(d.health());
+
+  // Every remaining route mutates: /wake starts a dispatch pass, /runs/* pauses,
+  // resumes, stops or re-ceilings a live worker. Until Phase 4 they were open on
+  // the reasoning that loopback is a boundary. On a shared host it is not: any
+  // local process — including a worker session this daemon launched itself —
+  // could stop any run or raise any turn ceiling. Refuse, and say which route
+  // and why, but never the token, not even truncated: a rejected credential is
+  // an attacker's guess and a log is a file other things read.
+  const authorization = req.headers.get("authorization");
+  if (!verifyHttpToken(authorization, d.token)) {
+    (d.log ?? log)(
+      `401 ${req.method} ${url.pathname} — ` +
+        `${authorization === null ? "no Authorization header" : "bearer token rejected"} ` +
+        `(the token is ${httpTokenPath()}; the daemon mints it at start)`,
+    );
+    return Response.json(
+      {
+        error:
+          "unauthorized: this route requires the daemon's bearer token. " +
+          `Read it from ${httpTokenPath()} and send it as "Authorization: Bearer <token>".`,
+      },
+      { status: 401 },
+    );
+  }
+
+  if (req.method === "POST" && url.pathname === "/wake") {
+    // A resume wake: daemon-wide and loopback-only, like /healthz, but
+    // authenticated — waking dispatch in a loop is a denial of service any
+    // local process could previously mount. The pass it produces re-reads
+    // every project's pause file, so a project that is still held stays held —
+    // the wake shortens the wait, it never bypasses a gate.
+    d.wake();
+    return new Response(null, { status: 204 });
+  }
+
+  let selected = d.projects[0];
+  if (
+    req.method === "PUT" &&
+    /^\/runs\/\d+\/(?:turn-limit|pause|resume|stop)$/.test(url.pathname) &&
+    req.headers.get("content-type")?.startsWith("application/json")
+  ) {
+    try {
+      const body = await req.clone().json();
+      if (body !== null && typeof body === "object") {
+        const requested = Reflect.get(body, "project");
+        if (typeof requested === "string" && requested.length > 0) {
+          selected = d.projects.find(({ project }) => project === requested);
+          if (selected === undefined) {
+            return Response.json(
+              { error: `daemon does not serve requested project "${requested}"` },
+              { status: 409 },
+            );
+          }
+        }
+      }
+    } catch {
+      // The route handler below owns the public malformed-body response.
+    }
+  }
+  if (selected === undefined) return new Response("not found\n", { status: 404 });
+
+  const turnLimit = await turnLimitResponse(
+    req,
+    selected.project,
+    selected.store,
+    d.turnLimits,
+    selected.caps(),
+  );
+  if (turnLimit !== undefined) return turnLimit;
+  const workerControl = await workerControlResponse(
+    req,
+    selected.project,
+    selected.store,
+    d.workerControls,
+    selected.log,
+  );
+  return workerControl ?? new Response("not found\n", { status: 404 });
+}

package/src/daemon/integrity.ts

@@ -0,0 +1,108 @@
+/**
+ * The package tripwire: hash every file the running install is made of, once
+ * per tick, and stop the fleet the moment one of them changes underneath it.
+ *
+ * The boundary is drawn around "what was deployed", not around any pass: the
+ * manifest is a self-portrait of `src/`, and the two callers — the tick that
+ * compares it and `runDaemon` that records the baseline — want the same three
+ * pure functions and nothing else. Keeping them here means the tripwire has no
+ * `Deps` and therefore nothing a pass could accidentally teach it to ignore.
+ *
+ * `markPaged` lives here rather than in `supervision.ts` because both latches it
+ * serves — this one and the stall gate's — are the same "page once, and only
+ * after delivery is confirmed" rule, and one overload set is the only way that
+ * rule stays one rule.
+ */
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync } from "node:fs";
+import { join, relative } from "node:path";
+import { PACKAGE_SRC_DIR, type IntegrityGate, type StallGate } from "./deps.ts";
+
+/** Enough differing paths to tell a deploy from a tamper at a glance; the full
+ *  list is on the host, and the answer is always "go look at the host". */
+export const INTEGRITY_SAMPLE = 5;
+
+export interface IntegrityVerdict {
+  /** Labelled, sorted differences; empty when the package is untouched. */
+  diff: string[];
+  /** Any difference at all stops the fleet. */
+  pause: boolean;
+  /** First divergent tick only — a page every five minutes is a page nobody reads. */
+  page: boolean;
+}
+
+/**
+ * sha256 of every source file the running package is made of, keyed by path
+ * relative to `root`.
+ *
+ * `PACKAGE_SRC_DIR` is the installed `src/` of the code executing right now, so
+ * this is a self-portrait: what was actually deployed, not what some checkout
+ * on disk happens to contain. `.ts` and `.md` because both are executable in
+ * this package — the briefs under `src/briefs/` are the sessions' instructions,
+ * and rewriting one of those buys more than rewriting the dispatcher does.
+ * (A checkout also carries `*.test.ts`, which the published package excludes, so
+ * a daemon started from one is watching its tests too. That is the honest
+ * answer — its code did change — and it costs nothing on a real install.)
+ *
+ * Walking and hashing the ~30 files of this package measures 0.6 ms warm, once
+ * per five-minute tick, so a tick does it inline. No cache and no mtime
+ * shortcut on purpose: a cache is a second thing that can be wrong, and mtime
+ * is the first field anyone covering their tracks restores.
+ */
+export function packageManifest(root: string = PACKAGE_SRC_DIR): Map<string, string> {
+  const out = new Map<string, string>();
+  const walk = (dir: string): void => {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.isFile() && (e.name.endsWith(".ts") || e.name.endsWith(".md")))
+        out.set(relative(root, full), createHash("sha256").update(readFileSync(full)).digest("hex"));
+    }
+  };
+  walk(root);
+  return out;
+}
+
+/**
+ * Labelled rather than three arrays because every consumer — the log line, the
+ * page, the test — wants one readable list of what moved.
+ */
+export function manifestDiff(before: Map<string, string>, after: Map<string, string>): string[] {
+  const out: string[] = [];
+  for (const [path, hash] of before) {
+    const now = after.get(path);
+    if (now === undefined) out.push(`removed ${path}`);
+    else if (now !== hash) out.push(`changed ${path}`);
+  }
+  for (const path of after.keys()) if (!before.has(path)) out.push(`added ${path}`);
+  return out.sort();
+}
+
+/**
+ * The tick's decision, split from its effects so the once-only page is a thing
+ * a test can hold.
+ *
+ * `pause` stays true on every divergent tick, deliberately: an operator who
+ * resumes without restarting gets re-paused, because the boundary is still
+ * broken. `page` asks whether this tick should *try* — the caller latches the
+ * gate with {@link markPaged} only once a page actually went out, so a Telegram
+ * outage during the one tick that noticed does not buy permanent silence.
+ */
+export function checkIntegrity(gate: IntegrityGate, current: Map<string, string>): IntegrityVerdict {
+  const diff = manifestDiff(gate.baseline, current);
+  if (diff.length === 0) return { diff, pause: false, page: false };
+  return { diff, pause: true, page: !gate.paged };
+}
+
+/** Latch a page after delivery is confirmed, and never before. */
+export function markPaged(gate: StallGate, delivered: boolean, now: number): void;
+export function markPaged(gate: IntegrityGate, delivered: boolean): void;
+export function markPaged(
+  gate: { paged: boolean; lastPagedAt?: number },
+  delivered: boolean,
+  now?: number,
+): void {
+  if (!delivered) return;
+  gate.paged = true;
+  if (now !== undefined) gate.lastPagedAt = now;
+}