catherd-cli 0.2.1 → 1.0.0

package/src/services/lane-service.ts

@@ -0,0 +1,248 @@
+import { existsSync, readFileSync } from "node:fs";
+import { relative, sep } from "node:path";
+import { CatherdError } from "../domain/errors.ts";
+import { assertId, ID_PATTERN, parseRung } from "../domain/ids.ts";
+import type { Difficulty, Kind } from "../domain/lane.ts";
+import type { Role } from "../domain/roles.ts";
+import {
+  type ClimbReason,
+  currentRoute,
+  laneOutcome,
+  nextRung,
+  type RouteJev,
+  type RouteSource,
+} from "../domain/route.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { commitExists } from "../infra/git.ts";
+import { budgetOf } from "./budget.ts";
+import type { Deps, Verdict } from "./ports.ts";
+import {
+  appendLedger,
+  appendOutcome,
+  appendRoute,
+  findRun,
+  knowledgeFile,
+  readRoutes,
+  type Run,
+  runFile,
+  runPaths,
+} from "./run-store.ts";
+import { type Notes, type NotesPatch, refreshState } from "./state.ts";
+import { appendPrivate } from "../infra/store.ts";
+
+const withHints = (hints: string[]) => (hints.length ? { hints } : {});
+
+export interface RouteResult {
+  lane: string | null;
+  role: Role;
+  rung: string;
+  ladder: string[];
+  source: RouteSource;
+  kind: Kind | null;
+  difficulty: Difficulty | null;
+  backend: string;
+  /** the native agent to run a `claude:` rung as */
+  agent: string | null;
+  /** the Jev question set asked, and what it said, when Jev was asked */
+  questionSet: string | null;
+  jev: RouteJev | null;
+}
+
+function readLaneFile(run: Run, path: string): { lane: string; text: string } {
+  const file = runFile(run, path, "read");
+  const [folder, name, ...deeper] = relative(run.dir, file).split(sep);
+  const lane = folder === "lanes" && deeper.length === 0 && name?.endsWith(".md") ? name.slice(0, -3) : null;
+  if (!lane || !ID_PATTERN.test(lane))
+    throw new CatherdError("E_LANE_INVALID", `a lane file lives at lanes/<id>.md, not ${path}`, {
+      fix: "pass lane_file as lanes/Mx.Ly.md",
+    });
+  if (!existsSync(file))
+    throw new CatherdError("E_LANE_INVALID", `no lane file ${path}`, {
+      fix: "write it with write_run_file first",
+    });
+  return { lane, text: readFileSync(file, "utf8") };
+}
+
+/** Spec §5.4 through the routing port; a lane's route is recorded in routes.jsonl. */
+export async function route(
+  deps: Deps,
+  i: { run: string; laneFile?: string; role: Role },
+): Promise<RouteResult> {
+  const run = findRun(i.run);
+  const lane = i.laneFile === undefined ? null : readLaneFile(run, i.laneFile);
+  const profile = deps.profiles.forRepo(run.meta.repo);
+  const a = await deps.routing.route({
+    runDir: run.dir,
+    repo: run.meta.repo,
+    profile,
+    role: i.role,
+    lane: lane?.lane ?? null,
+    laneText: lane?.text ?? null,
+    spentFraction: budgetOf(run, profile.budget, deps.now())?.fraction ?? 0,
+  });
+  if (lane)
+    appendRoute(run, {
+      at: new Date(deps.now()).toISOString(),
+      lane: lane.lane,
+      role: i.role,
+      rung: a.rung,
+      ladder: a.ladder,
+      source: "route",
+      decidedBy: a.source,
+      from: null,
+      reason: null,
+      kind: a.kind,
+      difficulty: a.difficulty,
+      questionSet: a.questionSet,
+      jev: a.jev,
+    });
+  return {
+    lane: lane?.lane ?? null,
+    role: i.role,
+    ...a,
+    backend: parseRung(a.rung).backend,
+    agent: deps.profiles.agentFor(run.meta.repo, i.role, a.rung),
+  };
+}
+
+/** Spec §4.5: one rung up the lane's ladder, on a fresh thread; the reason goes to routes.jsonl. */
+export async function climb(
+  deps: Deps,
+  i: { run: string; lane: string; reason: ClimbReason; evidence?: string; env?: boolean },
+): Promise<{
+  lane: string;
+  rung: string;
+  top: boolean;
+  backend: string;
+  agent: string | null;
+  hints?: string[];
+}> {
+  const run = findRun(i.run);
+  assertId("lane", i.lane);
+  const { cur, next } = await withFileLock(runPaths(run.dir).routes, () => {
+    const cur = currentRoute(readRoutes(run), i.lane);
+    if (!cur)
+      throw new CatherdError("E_LANE_INVALID", `lane ${i.lane} was never routed`, {
+        fix: `route(run, "lanes/${i.lane}.md") first`,
+      });
+    const next = nextRung(cur.ladder, cur.rung);
+    appendRoute(run, {
+      ...cur,
+      at: new Date(deps.now()).toISOString(),
+      source: "climb",
+      from: cur.rung,
+      rung: next ?? cur.rung,
+      reason: i.evidence ? `${i.reason}: ${i.evidence}` : i.reason,
+      env: i.env === true,
+    });
+    // spec §5.6: a climb past the top rung ends the lane open
+    if (!next) {
+      const o = laneOutcome(readRoutes(run), i.lane, false, new Date(deps.now()).toISOString());
+      if (o) appendOutcome(run, o);
+    }
+    return { cur, next };
+  });
+  const { hints } = await refreshState(run, {
+    next: next
+      ? `dispatch ${i.lane} at ${next} on a fresh thread`
+      : `${i.lane} failed on its top rung: ask finding, then the architect or the report`,
+  });
+  const rung = next ?? cur.rung;
+  return {
+    lane: i.lane,
+    rung,
+    top: next === null,
+    backend: parseRung(rung).backend,
+    agent: next ? deps.profiles.agentFor(run.meta.repo, cur.role, next) : null,
+    ...withHints(hints),
+  };
+}
+
+const cell = (s: string) => s.replace(/[|\n]/g, "/").replace(/\s+/g, " ").trim();
+
+/**
+ * Spec §4.7: the full five-column ledger row, with the minutes since the previous landing (or the
+ * run's start), and `learned` appended to the repo's knowledge.md.
+ */
+export async function land(
+  deps: Deps,
+  i: {
+    run: string;
+    milestone: string;
+    what: string;
+    commit: string;
+    evidence: string;
+    next: string;
+    learned?: string;
+  },
+): Promise<{ ledger: string; minutes: number; hints?: string[] }> {
+  const run = findRun(i.run);
+  // commitExists throws E_IO_UNEXPECTED on a timeout, which reaches the caller as is
+  if (!/^[0-9a-f]{7,40}$/.test(i.commit) || !(await commitExists(run.meta.repo, i.commit)))
+    throw new CatherdError("E_RUN_COMMIT", `no commit ${i.commit} in ${run.meta.repo}`, {
+      fix: "commit the milestone first, then pass its hash",
+    });
+  const now = new Date(deps.now());
+  let row = "";
+  let minutes = 0;
+  const landRow = (notes: Notes): NotesPatch => {
+    minutes = Math.max(
+      0,
+      Math.round((now.getTime() - Date.parse(notes.lastLandedAt ?? run.meta.createdAt)) / 60_000),
+    );
+    row = [i.milestone, i.what, i.commit, String(minutes), i.evidence].map(cell).join(" | ");
+    appendLedger(run, row);
+    return { lastCheck: cell(i.evidence), next: i.next, lastLandedAt: now.toISOString() };
+  };
+  // on a failed refresh the notes still reach state.json, so the next landing counts its minutes from this one
+  const { hints } = await refreshState(run, landRow);
+  // spec §5.6: every routed lane of the milestone lands with it
+  // under the routes lock, so a racing climb cannot slip between the read and the rows
+  let routed: string[] = [];
+  let landed = 0;
+  await withFileLock(runPaths(run.dir).routes, () => {
+    const routes = readRoutes(run);
+    routed = [...new Set(routes.map((r) => r.lane))];
+    for (const lane of routed)
+      if (lane.startsWith(`${i.milestone}.`)) {
+        landed++;
+        const o = laneOutcome(routes, lane, true, now.toISOString());
+        if (o) appendOutcome(run, o);
+      }
+  });
+  // a milestone name no routed lane starts with is most likely a typo: say so rather than record nothing
+  if (routed.length > 0 && landed === 0)
+    hints.push(
+      `land: no routed lane is in milestone "${i.milestone}" (routed: ${routed.slice(0, 5).join(", ")}${routed.length > 5 ? ", …" : ""}); check its name: no lane outcome was recorded`,
+    );
+  if (i.learned) {
+    appendPrivate(
+      knowledgeFile(run.meta.repo),
+      `- ${now.toISOString().slice(0, 10)} ${run.meta.title} ${i.milestone}: ${cell(i.learned)}\n`,
+    );
+  }
+  return { ledger: row, minutes, ...withHints(hints) };
+}
+
+/** Jev's `finding` or `same-defect` answer, through the routing port. */
+export async function ask(
+  deps: Deps,
+  i: { run: string; question: "finding" | "same-defect"; state: Record<string, string> },
+): Promise<Verdict<string>> {
+  const run = findRun(i.run);
+  const need = (k: string): string => {
+    const v = i.state[k];
+    if (!v)
+      throw new CatherdError("E_INPUT_INVALID", `ask ${i.question} needs state.${k}`, {
+        fix: i.question === "finding" ? "pass state { lane_file, finding }" : "pass state { before, after }",
+      });
+    return v;
+  };
+  // spec §5.5: a profile with Jev off never asks it, for routing or for these verdicts
+  const use = deps.profiles.forRepo(run.meta.repo).jev.use;
+  if (i.question === "finding") {
+    const { text } = readLaneFile(run, need("lane_file"));
+    return deps.routing.finding(run.dir, text, need("finding"), use);
+  }
+  return deps.routing.sameDefect(run.dir, need("before"), need("after"), use);
+}

package/src/services/ports.ts

@@ -0,0 +1,132 @@
+import type { Budget } from "../domain/budget.ts";
+import type { BillingMode } from "../domain/cost.ts";
+import type { Verdict } from "../domain/jev.ts";
+import type { Difficulty, Kind } from "../domain/lane.ts";
+import type { Access } from "../domain/record.ts";
+import type { Role } from "../domain/roles.ts";
+import type { Change, ProfilePatch } from "../domain/profile.ts";
+import type { Issue } from "../domain/profile-rules.ts";
+import type { RouteJev, RouteSource } from "../domain/route.ts";
+
+/** What the run lifecycle reads from a profile. Rungs are `backend:model#effort`; a role's in ladder order. */
+export interface ProfileView {
+  name: string;
+  objective: "cost" | "speed";
+  /** `defaultRung`, when set, is where a lane starts without a kind and difficulty (spec §5.4) */
+  roles: Partial<Record<Role, { enabled: boolean; access: Access; rungs: string[]; defaultRung?: string }>>;
+  /** per billing key (spec §7.1); a missing key bills as DEFAULT_BILLING */
+  billing: Partial<Record<string, BillingMode>>;
+  jev: { use: "auto" | "off" };
+  /** per backend id: run its harness isolated (spec §7.1 `harness`) */
+  isolated: Partial<Record<string, boolean>>;
+  /** rung → its stand-in on a usage limit (spec §4.5) */
+  failover: Record<string, string>;
+  budget: Budget;
+  timeouts: { idleMin: number; wallMin: number };
+  preflight: { confirm: boolean };
+  heavy: number | "cpus/2";
+  notify: string[];
+}
+
+/** What a profile write returns (spec §7.3): whether it saved, why not, what changed, and the agents it touched. */
+export interface ProfileSaved {
+  saved: boolean;
+  errors: Issue[];
+  warnings: Issue[];
+  diff: Change[];
+  linked: string[];
+  pruned: string[];
+  /** agents whose link is new or whose file changed: they apply from the next Claude Code session */
+  newSessionNeededFor: string[];
+}
+
+export interface ProfilePort {
+  /** The profile bound to `repo` (a git toplevel), else the active one; null asks for the active one. */
+  forRepo(repo: string | null): ProfileView;
+  /** Without a name (get, validate, set): the profile `repo`, a repository's toplevel, runs on. */
+  get(
+    name?: string,
+    repo?: string | null,
+  ): {
+    /** the global active profile */
+    active: string;
+    /** the profile `repo` runs on: its binding, else the active one */
+    here: string;
+    profiles: string[];
+    profile: ProfileView;
+    /** per role: whether its backend holds it to its access mode (spec D10) */
+    enforcement: Partial<Record<Role, "enforced" | "advisory">>;
+  };
+  validate(name?: string, repo?: string | null): { valid: boolean; errors: Issue[]; warnings: Issue[] };
+  set(name: string | undefined, patch: ProfilePatch, repo?: string | null): ProfileSaved;
+  /** The native agent that runs `rung` for `role` under `repo`'s profile; null unless it is a `claude:` rung. */
+  agentFor(repo: string | null, role: Role, rung: string): string | null;
+}
+
+export interface RouteRequest {
+  runDir: string;
+  repo: string;
+  profile: ProfileView;
+  role: Role;
+  /** the lane's id, for jev.jsonl; null when routing a role without a lane */
+  lane: string | null;
+  laneText: string | null;
+  spentFraction: number;
+}
+
+export interface RouteAnswer {
+  rung: string;
+  ladder: string[];
+  source: RouteSource;
+  kind: Kind | null;
+  difficulty: Difficulty | null;
+  /** the Jev question set asked, and its summed probabilities, for outcomes.jsonl (spec §5.6) */
+  questionSet: string | null;
+  jev: RouteJev | null;
+}
+
+export type { Verdict };
+
+export interface CatalogFilter {
+  role?: Role;
+  backend?: string;
+  text?: string;
+  scoredOnly: boolean;
+  limit: number;
+  /** the git toplevel whose per-repository listings (opencode's) to read; none reads the global ones */
+  repo?: string;
+}
+
+export interface RoutingPort {
+  route(req: RouteRequest): Promise<RouteAnswer>;
+  /** `use` is the repo profile's `jev.use`: "off" answers with the rule's default and never asks Jev */
+  finding(
+    runDir: string,
+    laneText: string,
+    finding: string,
+    use: "auto" | "off",
+  ): Promise<Verdict<"design" | "code" | "unclear">>;
+  sameDefect(
+    runDir: string,
+    before: string,
+    after: string,
+    use: "auto" | "off",
+  ): Promise<Verdict<"yes" | "no">>;
+  /** `billing` prices the rungs (spec §5.3); absent keys bill as DEFAULT_BILLING */
+  catalog(
+    filter: CatalogFilter,
+    billing?: Partial<Record<string, BillingMode>>,
+  ): { total: number; models: unknown[] };
+}
+
+/** Everything a service needs from outside it; the entry layer builds one, tests build fakes. */
+export interface Deps {
+  profiles: ProfilePort;
+  routing: RoutingPort;
+  version: string;
+  /** how often the supervisor and the dispatch wait poll, in ms */
+  pollMs: number;
+  /** how often a running dispatch reports progress, in ms (spec §4.4: 30 s) */
+  tickMs: number;
+  now: () => number;
+}

package/src/services/preflight.ts

@@ -0,0 +1,207 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseLaneHeader } from "../domain/lane.ts";
+import { checkEnv } from "../infra/env.ts";
+import { heavySlots, withHeavySlot } from "../infra/heavy-lock.ts";
+import { killGroup } from "../infra/proc.ts";
+import type { Deps } from "./ports.ts";
+import { TAIL_LINES } from "./run-debug.ts";
+import { findRun, type Run, runPaths } from "./run-store.ts";
+
+/** Spec §4.7. Only `cannot-start` blocks the run. */
+export type PreflightOutcome = "pass" | "fails-as-expected" | "skipped" | "cannot-start";
+
+interface PreflightResult {
+  lane: string;
+  check: string | null;
+  outcome: PreflightOutcome;
+  exitCode: number | null;
+  tail: string[];
+  note: string | null;
+}
+
+export type PreflightReport =
+  | { needsConfirmation: true; commands: { lane: string; check: string | null }[] }
+  | {
+      needsConfirmation: false;
+      commands: { lane: string; check: string | null }[];
+      results: PreflightResult[];
+      blocked: boolean;
+    };
+
+const CHECK_TIMEOUT_MS = 120_000;
+
+/** Whose uid preflight runs under; tests stand in for root with it. */
+export const preflightUser = { uid: (): number => process.getuid?.() ?? -1 };
+
+/**
+ * Spec §10.4: a lane's check never runs as root. A machine that says it is disposable (IS_SANDBOX, the
+ * rule claude itself applies to root) is the one exception.
+ */
+const refusesRoot = (): boolean => preflightUser.uid() === 0 && !process.env.IS_SANDBOX;
+const AS_ROOT =
+  "not run: catherd runs as root, and preflight never runs a lane's check as root (spec §10.4); run catherd as a normal user, or set IS_SANDBOX=1 on a disposable machine";
+
+interface LaneCheck {
+  lane: string;
+  check: string | null;
+  owns: string[];
+  problem: string | null;
+}
+
+function laneChecks(run: Run): LaneCheck[] {
+  const dir = runPaths(run.dir).lanes;
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .filter((f) => f.endsWith(".md"))
+    .sort()
+    .map((f) => {
+      const lane = f.slice(0, -".md".length);
+      try {
+        const h = parseLaneHeader(readFileSync(join(dir, f), "utf8"));
+        return {
+          lane,
+          check: h.fastCheck,
+          owns: h.owns,
+          problem: h.fastCheck ? null : `lanes/${f} has no Fast check: line`,
+        };
+      } catch (e) {
+        return { lane, check: null, owns: [], problem: (e as Error).message };
+      }
+    });
+}
+
+/** How long the pipes may stay open after the check's own process exits. */
+export const DRAIN_MS = 500;
+
+/** Reads a pipe chunk by chunk, so what arrived is kept when the reading stops early. */
+export function collect(stream: ReadableStream<Uint8Array>): {
+  done: Promise<void>;
+  text: () => string;
+  stop: () => void;
+} {
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+  let text = "";
+  const done = (async () => {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) return;
+      text += decoder.decode(value, { stream: true });
+    }
+  })().catch(() => {
+    // a pipe that breaks ends the reading; what arrived is kept
+  });
+  // cancelling a pipe that is already closed has nothing to report
+  return { done, text: () => text, stop: () => void reader.cancel().catch(() => {}) };
+}
+
+/**
+ * Runs `check` with `sh -c` in its own process group, with an allowlisted env (no credentials), killing the
+ * group on timeout. A process the check leaves behind (even in a new session) may hold the pipes open: once
+ * the check exits, the pipes get DRAIN_MS to close, then the reading stops with what arrived.
+ */
+export async function runCheck(
+  repo: string,
+  check: string,
+  timeoutMs: number,
+): Promise<{ code: number | null; timedOut: boolean; tail: string[] }> {
+  const p = Bun.spawn(["sh", "-c", check], {
+    cwd: repo,
+    env: checkEnv(process.env, repo),
+    stdin: "ignore",
+    stdout: "pipe",
+    stderr: "pipe",
+    detached: true,
+  });
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    killGroup(p.pid, "SIGKILL");
+  }, timeoutMs);
+  const out = collect(p.stdout);
+  const err = collect(p.stderr);
+  try {
+    // the timeout kills the group, so the check's own process always exits
+    const code = await p.exited;
+    clearTimeout(timer); // the check is done: a slow drain is not a timeout
+    const drained = await Promise.race([
+      Promise.all([out.done, err.done]).then(() => true),
+      Bun.sleep(DRAIN_MS).then(() => false),
+    ]);
+    if (!drained) killGroup(p.pid, "SIGKILL"); // leftovers still in the check's group
+    const tail = `${out.text()}${err.text()}`
+      .split("\n")
+      .filter((l) => l.trim())
+      .slice(-TAIL_LINES);
+    return { code: timedOut ? null : code, timedOut, tail };
+  } finally {
+    clearTimeout(timer);
+    out.stop();
+    err.stop();
+  }
+}
+
+export function classify(r: { code: number | null; timedOut: boolean; tail: string[] }): PreflightOutcome {
+  if (r.timedOut || r.code === 126 || r.code === 127) return "cannot-start";
+  if (r.code === 0) return "pass";
+  return r.tail.some((l) => /command not found/i.test(l)) ? "cannot-start" : "fails-as-expected";
+}
+
+/**
+ * Spec §4.7: each lane's fast check once, on the base tree, behind the heavy lock. A check on a file the
+ * lane itself creates is skipped. With `profile.preflight.confirm`, the first call only lists the commands.
+ */
+export async function preflight(
+  deps: Deps,
+  i: { run: string; confirmed?: boolean; timeoutMs?: number },
+): Promise<PreflightReport> {
+  const run = findRun(i.run);
+  const profile = deps.profiles.forRepo(run.meta.repo);
+  const lanes = laneChecks(run);
+  const commands = lanes.map((l) => ({ lane: l.lane, check: l.check }));
+  if (profile.preflight.confirm && !i.confirmed) return { needsConfirmation: true, commands };
+  const timeoutMs = i.timeoutMs ?? CHECK_TIMEOUT_MS;
+  const results: PreflightResult[] = [];
+  for (const l of lanes) {
+    const check = l.check;
+    if (!check) {
+      results.push({
+        lane: l.lane,
+        check,
+        outcome: "cannot-start",
+        exitCode: null,
+        tail: [],
+        note: l.problem,
+      });
+      continue;
+    }
+    if (refusesRoot()) {
+      results.push({ lane: l.lane, check, outcome: "skipped", exitCode: null, tail: [], note: AS_ROOT });
+      continue;
+    }
+    const unborn = l.owns.filter(
+      (p) => check.includes(p.replace(/\/$/, "")) && !existsSync(join(run.meta.repo, p)),
+    );
+    if (unborn.length) {
+      const note = `${unborn.join(", ")} does not exist yet; the lane creates it`;
+      results.push({ lane: l.lane, check, outcome: "skipped", exitCode: null, tail: [], note });
+      continue;
+    }
+    const r = await withHeavySlot(heavySlots(profile.heavy), () => runCheck(run.meta.repo, check, timeoutMs));
+    results.push({
+      lane: l.lane,
+      check,
+      outcome: classify(r),
+      exitCode: r.code,
+      tail: r.tail,
+      note: r.timedOut ? `timed out after ${timeoutMs / 1000} s` : null,
+    });
+  }
+  return {
+    needsConfirmation: false,
+    commands,
+    results,
+    blocked: results.some((r) => r.outcome === "cannot-start"),
+  };
+}