omp-conductor 0.19.7 → 0.20.0

package/src/daemon/deps.ts

@@ -0,0 +1,539 @@
+/**
+ * The seam every dispatch pass is written against.
+ *
+ * `Deps` is the reason this directory can exist at all: a pass takes the one
+ * struct and touches nothing else, so it can move to its own module without
+ * learning about its neighbours. That also makes this module the test of
+ * whether a cut is in the right place — a pass that needs something no `Deps`
+ * field can reach was split along the wrong line.
+ *
+ * Three kinds of thing live here and nothing else:
+ *
+ *  - `Deps` itself, the gate and registry interfaces it is made of, and the
+ *    intervals `runDaemon` schedules against. The gate types sit here rather
+ *    than beside their implementations so `Deps` never imports a pass, and no
+ *    pass imports another merely to name a field it already holds.
+ *  - `verbDeps` and `recordDaemonSessionSpend`: the two projections of `Deps`
+ *    that more than one pass performs. Duplicated they would be two answers to
+ *    one question; parked inside one pass they would make the others import it
+ *    sideways.
+ *  - the pure identity and classification leaves the passes share — `repoSlug`,
+ *    `knowledgeRepoKey`, `githubRepo`, `completionLastError`,
+ *    `exhaustedSessionReason`. A shared floor with no upstream of its own is
+ *    what keeps the rest of this directory acyclic.
+ *
+ * `PACKAGE_SRC_DIR` is the one declaration here that is not a move. Modules
+ * under `daemon/` sit one directory deeper than the code they came from, so
+ * every `import.meta.dir` read that meant "the installed `src/`" — the brief
+ * templates, the integrity manifest root, the two lines that print the package
+ * root — resolves through this constant rather than silently re-deriving a
+ * path one level too deep.
+ */
+import { join } from "node:path";
+import { findProject, loadConfig } from "../config.ts";
+import type { CredentialClassProbeResult } from "../credential-class.ts";
+import type { HerdrRun } from "../fleet.ts";
+import { probeRunLane, readBaseChain, type CriticalBaseProbe, type RunLaneProbe } from "../gitops.ts";
+import { errText, log } from "../log.ts";
+import type { OrchestratorHandle } from "../orchestrator.ts";
+import { isPaused, pausedAt } from "../pause.ts";
+import type { UnroutableReason } from "../routing.ts";
+import type { TelegramFreshness } from "../telegram-freshness.ts";
+import type { Caps, Escalation, FailureClass, HostConstraints, IssueSnapshot, ProjectConfig, RepoTarget, RunRecord, Store, Tracker } from "../types.ts";
+import type { InstalledSurfaces } from "../upgrade.ts";
+import type { UsageSource } from "../usage.ts";
+import { githubVerbActions } from "../verbs/actions.ts";
+import type { VerbActions, VerbDeps } from "../verbs/server.ts";
+import type { PeerReader } from "../verbs/socket.ts";
+import type { AdjudicationResult, AdjudicatorOpts, RunWorkerDeps, ToSpecScoutOpts, ToSpecScoutResult, WorkerPauseControl, WorkerPausePhase } from "../worker.ts";
+
+/**
+ * The installed `src/` — the directory `daemon.ts` used to be `import.meta.dir`
+ * of, before its passes moved one level down into `daemon/`.
+ *
+ * Three reads depend on that exact path and would break silently one directory
+ * deeper: the brief templates under `src/briefs/`, the integrity manifest root
+ * (which is the package's self-portrait, so a wrong root would hash a tenth of
+ * it and call the rest unchanged), and the two operator-facing lines that print
+ * the package root. One constant so all five sites keep the same answer.
+ */
+export const PACKAGE_SRC_DIR = join(import.meta.dir, "..");
+
+/** Long enough that the tracker is not polled raw, short enough that a human
+ *  who labels an issue sees it picked up within a coffee break. */
+export const TICK_INTERVAL_MS = 5 * 60_000;
+export const GRAPH_HEALTH_INTERVAL_MS = 60_000;
+/** Report delivery runs on its own timer, not the five-minute dispatch tick: a
+ *  report an operator is waiting on must not sit in the outbox for the length of
+ *  a poll interval, and delivery is owed even while claiming is paused (#123). */
+export const REPORT_DELIVERY_INTERVAL_MS = 30_000;
+
+export const DEFAULT_PORT = 8787;
+
+/** Fleet-wide escalations still need an issue number in the payload; 0 is the
+ *  sentinel that reads as "no issue" in every renderer. */
+export const NO_ISSUE = 0;
+
+export const UNROUTABLE_TEXT: Record<UnroutableReason, string> = {
+  "no-repo-label": "it carries no repo label",
+  "multiple-repo-labels": "it carries more than one repo label",
+  "unknown-repo": "its repo label maps to no configured repo",
+};
+
+export interface DaemonOpts {
+  once?: boolean;
+  port?: number;
+  project?: string;
+}
+
+/** Everything one tick touches. `project` and `caps` are re-resolved at each
+ *  tick boundary so an operator's config edit applies on the next tick rather
+ *  than the next daemon restart (#170); a tick and the runs it admits see one
+ *  consistent snapshot, and a mid-run edit never changes a live run's labels,
+ *  model or caps — `handleIssue` destructures them at dispatch time. The rest
+ *  are resolved once at startup so a tick never re-reads config mid-flight and
+ *  changes its own limits underneath itself. */
+export interface DrainSignal {
+  draining: boolean;
+}
+
+export interface Deps {
+  project: ProjectConfig;
+  caps: Caps;
+  /** The typed host-constraints block (#721), re-resolved at the tick
+   *  boundary like `project`/`caps`, and rendered into every brief the tick
+   *  dispatches. Optional so tests that never exercise it can omit it. */
+  host?: HostConstraints;
+  tracker: Tracker;
+  store: Store;
+  /**
+   * Set true the moment a daemon stop (SIGTERM/SIGINT) begins. The run loop
+   * only sees `stopping` between whole ticks, so a pass that was already in
+   * flight when the stop landed must re-check this flag itself — before it
+   * claims and before it launches — or it creates exactly the work the
+   * shutdown is about to wait for and then lose to the stop timeout (#374).
+   * Optional only so tests that never exercise shutdown can omit it;
+   * `runDaemon` always wires the real one.
+   */
+  drain?: DrainSignal;
+  /** False after a live config reload fails; autonomous delivery then holds
+   * fail-closed until a later tick validates the config again. */
+  deliveryPolicyValid?: boolean;
+  /** Provider-reported plan allowance, cached with a TTL. Resolved once at
+   *  startup like every other dep so a tick cannot swap its own meter. */
+  usage: UsageSource;
+  escalate(e: Escalation): Promise<void>;
+  turnLimits: TurnLimitRegistry;
+  workerControls: WorkerControlRegistry;
+  /** Session seam for lifecycle integration tests; production uses the real harness. */
+  workerDeps?: RunWorkerDeps;
+  /** Harness per-process logs used to classify terminal worker routes. */
+  harnessLogDir?: string;
+  /**
+   * The Herdr runner for the worker-representation surface (#1035). Production
+   * leaves it absent — the real `herdr` CLI answers; lifecycle tests inject a
+   * fake so convergence is proven without a terminal.
+   */
+  herdrRun?: HerdrRun;
+  integrity: IntegrityGate;
+  stall: StallGate;
+  /**
+   * Stops the wedged orchestrator by exact identity, for the bounded
+   * auto-restart ({@link restartWedgedOrchestrator}). Production leaves it
+   * absent and `stopConductorPane` answers — the same resolver `halt --pane`
+   * uses, so there is exactly one implementation of "which pane is the
+   * orchestrator". A test injects a fake, which is what makes the counter, the
+   * bound and the record-before-attempt ordering assertable without a terminal.
+   */
+  restartOrchestrator?: (
+    projectName: string,
+  ) => Promise<{ stopped: "herdr-agent" | "already-gone"; detail: string; agentName: string }>;
+  /**
+   * The embedded orchestrator session handle when one started; absent when it
+   * failed to start or the project uses an external orchestrator. Feeds the
+   * orchestrator-down reconcile ({@link reconcileOrchestratorDown}) so a
+   * crashed session pages once per incident instead of degrading quietly.
+   */
+  orchestrator?: OrchestratorHandle;
+  cleanup?: RetainedCleanupCursor;
+  /**
+   * Reads the connecting uid off a verb socket (#126). Resolved once at startup
+   * so the mechanism is logged before the first socket exists; absent on a host
+   * exposing no peer-credential call, where the `0600` socket under the
+   * daemon-owned `0711` parent is all that keeps other local accounts out, and
+   * `status` says so. It never told two sessions apart — they share the daemon's
+   * uid.
+   */
+  verbPeerReader?: PeerReader;
+  /**
+   * The privileged half the verbs call once their checks pass. Optional only so
+   * a test can build a `Deps` without a repository; `runDaemon` always wires
+   * the real one.
+   */
+  verbActions?: VerbActions;
+  /**
+   * The fleet-installs-itself verification pass (#486), run on the first tick
+   * after a detached upgrade. The default is the journal-gated
+   * {@link verifyPendingUpgradeTick}; a test injects its own to drive the tick
+   * without touching the host's packages, journal or outbox.
+   */
+  upgradeVerifier?: (d: Deps) => Promise<void>;
+  /**
+   * Answers whether a preserved continuation branch contains every configured
+   * critical-base marker, for the stale-base admission hold (#428). Wired by
+   * `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects
+   * a fake. Absent, the stale-base gate fails closed whenever a project names
+   * a marker (a safety interlock must not silently weaken).
+   */
+  probeCriticalBase?: CriticalBaseProbe;
+  /**
+   * Reads one active run's file lane for the admission file-lane interlock
+   * (#555): the union of its uncommitted worktree changes and its branch-vs-base
+   * diff. Reconciliation is not occupancy: while the run merges its base, the
+   * probe reports only files it has actually diverged on, never the files the
+   * merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed
+   * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert
+   * (no lane is ever known occupied), which is the issue's "fail open": the
+   * gate adds holds, it never refuses a well-formed issue for lack of this
+   * probe the way `criticalBase` does.
+   */
+  probeWorktreeLane?: RunLaneProbe;
+  /**
+   * Reads one provider's current credential class from the harness, out of
+   * process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test
+   * injects a fake.
+   *
+   * Absent, both fences refuse a project that declares `requireOauthProviders` —
+   * this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified
+   * credential class costs exactly what a wrong one costs. A project declaring
+   * nothing never calls it, so an unwired test dispatches as it always has.
+   */
+  probeCredentialClass?: (provider: string) => Promise<CredentialClassProbeResult>;
+  /**
+   * Reads one issue's tracker state in a repository the admission tracker is
+   * not bound to — the cross-repo Depends-on interlock (#420). Wired by
+   * `runDaemon` to a repo-scoped tracker; a test injects a fake. Absent,
+   * admission fails a routed cross-repo prerequisite closed.
+   */
+  probeIssueIn?: (repo: string, issue: number) => Promise<IssueSnapshot | undefined>;
+  /**
+   * Reads one issue's BODY in a repository the admission tracker is not bound
+   * to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a
+   * repo-scoped tracker; a test injects a fake. Absent, a routed reachable
+   * body fails that branch closed rather than synthesising a cycle.
+   */
+  probeBodyIn?: (repo: string, issue: number) => Promise<string | undefined>;
+  /**
+   * Reads the three install identities this host carries (#919). Wired by
+   * `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade`
+   * read, so nothing re-implements the probe. A test injects its own; absent,
+   * the pass records nothing and every cheap surface honestly says "not
+   * observed yet" rather than claiming agreement.
+   */
+  probeInstallSurfaces?: () => Promise<InstalledSurfaces>;
+  /** The `omp-telegram` install/daemon/published triple, read on the same
+   *  periodic pass as the surfaces above (#961). */
+  probeTelegramFreshness?: () => Promise<TelegramFreshness>;
+  /**
+   * Runs one review-ceiling adjudication (#932). Production is
+   * {@link runAdjudicator}; a test injects a fake so the assertion can be the
+   * launch arguments and the assembled brief — which is what this pass actually
+   * produces — rather than a stored flag.
+   */
+  runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise<AdjudicationResult>;
+  /**
+   * Runs one to-spec grooming pass (#1041). Production is
+   * {@link runToSpecScout}; a test injects a fake, which is what makes the
+   * whole mechanical pipeline — select, claim, persist, gate, promote, wake —
+   * assertable end to end without a session or a network.
+   */
+  runToSpecScoutImpl?: (opts: ToSpecScoutOpts) => Promise<ToSpecScoutResult>;
+  /**
+   * The dispatch wake fired after a queue-label hand-back or a mechanical
+   * promotion (#1041) — the same seam `settlement.ts` reads, so one injected
+   * spy covers every requeue path a tick runs. Production leaves it absent and
+   * the real loopback client answers: from inside the daemon that is a POST to
+   * its own `/wake`, which is deliberate rather than a shortcut — the wake is
+   * one coalescing flag with one owner, and a second in-process path to it
+   * would be a second answer to "is a pass pending".
+   */
+  wake?: (projectName: string) => Promise<string>;
+}
+
+/**
+ * The daemon-side view a verb call decides against (#126).
+ *
+ * `project` is a thunk, not the resolved value the rest of a tick uses. That is
+ * the whole "fail closed on an unreadable config" rule: a config read once at
+ * boot can never become unreadable, and it cannot pick up an operator who has
+ * just taken merge authority back either. Every verb pays one file read for
+ * the property that its answer reflects the config as it is *now*.
+ *
+ * `fleetStop` is read the same way and for the same reason — inside the verb,
+ * after the model decided to call it. `hold` and `halt` both set the pause
+ * sentinel alongside disarming ticks, so this one read covers both.
+ */
+export function verbDeps(d: Pick<Deps, "project" | "store" | "tracker" | "verbActions">): VerbDeps {
+  return {
+    project: () => findProject(loadConfig(), d.project.name),
+    projects: () => loadConfig().projects,
+    store: d.store,
+    tracker: d.tracker,
+    actions: d.verbActions ?? githubVerbActions(d.project),
+    fleetStop: () =>
+      isPaused(d.project.name)
+        ? "claiming is paused for this fleet (omp-conductor hold or stop)"
+        : undefined,
+    pausedAt: () => pausedAt(d.project.name),
+    log,
+    now: () => Date.now(),
+    chain: { readBaseChain },
+    lane: { probeRunLane },
+  };
+}
+
+/**
+ * Whether the wedged-orchestrator page has gone out, and when it last went out,
+ * for the stall currently on disk.
+ */
+export interface StallGate {
+  paged: boolean;
+  lastPagedAt?: number;
+}
+
+/**
+ * What the daemon booted with, and whether it has already paged about losing
+ * it. Lives exactly as long as one `runDaemon()` call — which is the whole
+ * trick: a restart re-records both.
+ */
+export interface IntegrityGate {
+  baseline: Map<string, string>;
+  paged: boolean;
+}
+
+/**
+ * `owner/repo` for `gh`, derived from the clone URL.
+ *
+ * ponytail: RepoTarget has no explicit slug, so it is parsed off the URL and
+ * falls back to the routing name. Upgrade path is an optional `slug` field once
+ * a non-GitHub remote actually shows up.
+ */
+export function repoSlug(repo: RepoTarget): string {
+  const m = /(?:[:/])([^/:]+\/[^/]+?)(?:\.git)?$/.exec(repo.cloneUrl);
+  return m?.[1] ?? repo.name;
+}
+
+/**
+ * The identity the per-repo knowledge overlay is filed under: the canonical
+ * `owner/repo` when the clone URL names one, else the routed name.
+ *
+ * Deliberately NOT {@link repoSlug}: that one is a loose parse whose job is to
+ * *say something* in a brief for any URL shape, so a filesystem path yields a
+ * plausible-looking pair of directory names. A knowledge file keyed off that
+ * would be keyed off where a mirror happens to live. One function, used by both
+ * the writer at settlement and the reader in the brief, so the two can never
+ * file and read under different names.
+ */
+export function knowledgeRepoKey(repo: RepoTarget): string {
+  return githubRepo(repo.cloneUrl) ?? repo.name;
+}
+
+export type ExtendTurnLimitResult =
+  | { kind: "extended"; runId: string; maxTurns: number }
+  | { kind: "not-increase"; runId: string; maxTurns: number }
+  | { kind: "not-active" };
+
+export interface TurnLimitController {
+  maxTurns(): number;
+  close(): void;
+}
+
+export interface TurnLimitRegistry {
+  open(project: string, issue: number, runId: string, maxTurns: number): TurnLimitController;
+  extend(project: string, issue: number, maxTurns: number): ExtendTurnLimitResult;
+}
+
+export type WorkerControlResult =
+  | { kind: "ok"; runId: string; phase: WorkerPausePhase }
+  | { kind: "stopped"; runId: string; reason: string }
+  | { kind: "refused"; runId: string; error: string }
+  | { kind: "not-active" };
+
+export interface WorkerControlSlot {
+  install(control: WorkerPauseControl): void;
+  /** Stop accepted before the session controller exists. */
+  requestedStop(): string | undefined;
+  close(): void;
+}
+
+export interface WorkerControlRegistry {
+  /**
+   * `onPhase` is called after a pause/resume/stop that actually changed this
+   * worker's phase (#842) — the authoritative transition, from the same control
+   * that performed it. It exists so a surface outside this registry (the run's
+   * Herdr representation) can follow the phase without polling and without
+   * inventing a second notion of "paused".
+   */
+  open(
+    project: string,
+    issue: number,
+    runId: string,
+    onPhase?: (phase: WorkerPausePhase) => void,
+  ): WorkerControlSlot;
+  /** `source` names who asked (board/cli/dashboard) — recorded as pause
+   *  provenance so "who paused this and when" is answerable later (#997). */
+  pause(project: string, issue: number, source: string): Promise<WorkerControlResult>;
+  resume(project: string, issue: number): WorkerControlResult;
+  stop(project: string, issue: number, reason: string): Promise<WorkerControlResult>;
+  /** Live runs whose phase is not `running` — what /healthz and the board
+   *  show — each carrying its pause provenance when one was recorded (#997). */
+  snapshot(project: string): {
+    issue: number;
+    runId: string;
+    phase: WorkerPausePhase;
+    source?: string;
+    pausedAtMs?: number;
+  }[];
+}
+
+// =============================================== daemon-owned session accounting
+// (Phase 4, attribution.) Grooming and adjudication are real sessions the daemon
+// launches on its own initiative: they resolve a model, take turns, and cost
+// money. Until now their `{turns, spendUsd}` was printed to the journal and
+// thrown away, so the only spend conductor could attribute was worker spend, and
+// every question of the form "what did the pipeline cost, not just the workers"
+// had no answer at all.
+//
+// They are NOT run rows. `computeStats` walks `runs` as issue journeys keyed on
+// `run.issue`, deciding merged/failed/settled from the sequence of states for
+// one issue — a groom row landing in that table would be counted as an attempt
+// on the issue and would corrupt the merge counts it feeds. Hence a dedicated
+// role-tagged table (Lane A's `recordSessionSpend`).
+
+/**
+ * Persist one daemon-owned session's accounting, telling unmetered from free.
+ *
+ * The honesty rule this exists to enforce: the harness reports `spendUsd: 0`
+ * both for a session that genuinely cost nothing and for one whose provider
+ * reports no per-request price at all (a subscription credential). Writing 0 for
+ * the second would render as "$0.00" and read as free, which is exactly the
+ * mistake `spend-telemetry.ts` was written to stop — "$0.00 spend is not proof
+ * of no spend" (#46, #984).
+ *
+ * So the discrimination is the same one {@link judgeSpendTelemetry} makes: a
+ * session that took turns is a *working* session, and a working session
+ * reporting zero is UNMETERED — the column is omitted and the stats lane renders
+ * it as unknown. A session that took no turns did not run (an unresolvable role,
+ * a harness that would not start), and zero is then the true cost.
+ *
+ * Recorded before any of the caller's verdict branching, so every early return
+ * still accounts for the money that was already spent.
+ */
+export function recordDaemonSessionSpend(
+  d: Pick<Deps, "project" | "store">,
+  row: {
+    role: "groom" | "adjudicator";
+    issue?: number;
+    /** The role/model pattern conductor asked for. */
+    model?: string;
+    /** What actually ran it, when the session reported back. */
+    resolvedModel?: string;
+    turns: number;
+    spendUsd: number;
+    at: number;
+  },
+): void {
+  const metered = row.turns === 0 || row.spendUsd > 0;
+  try {
+    d.store.recordSessionSpend({
+      project: d.project.name,
+      role: row.role,
+      ...(row.issue === undefined ? {} : { issue: row.issue }),
+      ...(row.model === undefined ? {} : { model: row.model }),
+      ...(row.resolvedModel === undefined ? {} : { resolvedModel: row.resolvedModel }),
+      turns: row.turns,
+      ...(metered ? { spendUsd: row.spendUsd } : {}),
+      at: row.at,
+    });
+  } catch (err) {
+    // Accounting must never lose the session it accounts for: the verdict this
+    // run produced is worth more than its price tag.
+    log(`${row.role} session spend not recorded: ${errText(err)}`);
+  }
+}
+
+/**
+ * The terminal classes that mean "the transcript this round would resume
+ * cannot usefully take another turn".
+ *
+ * Enumerated rather than inferred: each of these is a *cap or a provider
+ * refusal*, not a logic failure, so resuming buys a session that dies the same
+ * way. The value is the clause the log line and the correction brief read, so
+ * the reason a fresh session exists is never "the daemon decided so".
+ */
+export const EXHAUSTED_SESSION_CLASSES: Partial<Record<FailureClass, string>> = {
+  "turn-cap-progress": "it was killed by the turns cap",
+  "turn-cap-spinning": "it was killed by the turns cap while spinning",
+  "wall-clock-cap-progress": "it was killed by the wall-clock cap",
+  "wall-clock-cap-spinning": "it was killed by the wall-clock cap while spinning",
+  "model-empty-stop": "the provider returned empty turns until the harness retry cap ended it",
+};
+
+/**
+ * Why the session recorded on this run cannot be resumed, or `undefined` when
+ * it can (#1047).
+ *
+ * Two facts, both durable on the row and both readable without opening a
+ * transcript:
+ *
+ *  - the row's own terminal classification, when it names a cap or the
+ *    provider empty-stop; and
+ *  - the turn ledger: a session at or past its recorded ceiling has no turns
+ *    left whatever ended it. This is the fact that catches the rounds #1035
+ *    actually burnt — the review path raises the ceiling by one round's
+ *    allowance per round (#747), so a round that spends its whole allowance
+ *    leaves `turns >= maxTurns` on the row even after the claim has cleared
+ *    the classification.
+ *
+ * Shared with the answered-block continuation path, which needs the same
+ * question answered about the same kind of row: resuming an exhausted session
+ * is exactly as useless there.
+ */
+export function exhaustedSessionReason(
+  run: Pick<RunRecord, "failureClass" | "turns" | "maxTurns">,
+): string | undefined {
+  const classed = run.failureClass === undefined ? undefined : EXHAUSTED_SESSION_CLASSES[run.failureClass];
+  if (classed !== undefined) return `${classed} (${run.failureClass})`;
+  if (run.turns >= run.maxTurns) {
+    return `it has spent its whole turn ceiling (${run.turns}/${run.maxTurns} turns)`;
+  }
+  return undefined;
+}
+
+/** Canonical `owner/repo` identity from a configured network clone URL. */
+export function githubRepo(cloneUrl: string): string | undefined {
+  const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, "");
+  const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec(
+    normalized,
+  );
+  return match?.[1];
+}
+
+export interface RetainedCleanupCursor {
+  next: number;
+}
+
+/** A provider refusal a session recorded before dying, or undefined for none. */
+export interface SessionError {
+  status?: number;
+  message: string;
+}
+
+export function completionLastError(
+  providerCredit: string | undefined,
+  providerTransient: string | undefined,
+  verifiedReason: string | undefined,
+  sessionErr: SessionError | undefined,
+): string | undefined {
+  return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message;
+}