catherd-cli 0.2.1 → 1.1.0

package/src/services/run-store.ts

@@ -0,0 +1,334 @@
+import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync } from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { CatherdError } from "../domain/errors.ts";
+import { assertId } from "../domain/ids.ts";
+import { type RunRecord, RunRecordSchema } from "../domain/record.ts";
+import type { OutcomeRow, RouteRow } from "../domain/route.ts";
+import { slug } from "../domain/util.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { dataDir, repoDir, runsDir } from "../infra/paths.ts";
+import {
+  appendJsonl,
+  ensureJsonlHeader,
+  ensurePrivateDir,
+  PRIVATE_DIR,
+  appendPrivate,
+  readJsonl,
+  readVersioned,
+  writeJsonAtomic,
+  writeTextAtomic,
+} from "../infra/store.ts";
+
+/** Spec §3.3: the Claude Code session that started a run; absent on 1.0 runs and outside Claude Code. */
+const StartedBySchema = z.object({
+  sessionId: z.string(),
+  hostSessionId: z.string().nullable(),
+  name: z.string().nullable(),
+});
+export type StartedBy = z.infer<typeof StartedBySchema>;
+
+const RunMetaSchema = z.looseObject({
+  schema: z.literal(1),
+  id: z.string(),
+  repo: z.string(),
+  title: z.string(),
+  aLines: z.array(z.string()),
+  createdAt: z.string(),
+  catherdVersion: z.string(),
+  startedBy: StartedBySchema.nullable().optional(),
+});
+type RunMeta = z.infer<typeof RunMetaSchema>;
+
+export interface Run {
+  id: string;
+  dir: string;
+  meta: RunMeta;
+}
+
+/** Spec §4.1. */
+export function runPaths(dir: string) {
+  return {
+    meta: join(dir, "meta.json"),
+    plan: join(dir, "plan.md"),
+    lanes: join(dir, "lanes"),
+    ledger: join(dir, "ledger.md"),
+    state: join(dir, "state.md"),
+    stateJson: join(dir, "state.json"),
+    runs: join(dir, "runs.jsonl"),
+    routes: join(dir, "routes.jsonl"),
+    jev: join(dir, "jev.jsonl"),
+    outcomes: join(dir, "outcomes.jsonl"),
+    agents: join(dir, "agents.jsonl"),
+    harness: join(dir, "harness.jsonl"),
+    /** every session that has owned the run, in order (spec §3.3) */
+    sessions: join(dir, "sessions.jsonl"),
+    roles: join(dir, "roles"),
+    shots: join(dir, "shots"),
+    /** the admission lock's target: `admission.lock` guards dispatch admission */
+    admission: join(dir, "admission"),
+  };
+}
+
+export const LEDGER_HEADER = "milestone | what | commit | minutes | evidence";
+
+const runSlug = (title: string) => slug(title).slice(0, 40) || "run";
+const stamp = (d: Date) => d.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
+
+/** Creates a run folder under the repo's runs dir. `repo` is a git toplevel; meta.json is written last. */
+export function createRun(o: {
+  repo: string;
+  title: string;
+  aLines: string[];
+  version: string;
+  now?: Date;
+  startedBy?: StartedBy | null;
+}): Run {
+  const now = o.now ?? new Date();
+  const root = runsDir(o.repo);
+  ensurePrivateDir(root);
+  const base = `${stamp(now)}-${runSlug(o.title)}`;
+  // findRun looks an id up across every repo, so an id is claimed across every repo too: the claim
+  // folder is atomic between processes, the scan covers runs made before claims existed
+  const claims = join(dataDir(), "run-ids");
+  ensurePrivateDir(claims);
+  const repos = join(dataDir(), "repos");
+  const taken = (id: string) =>
+    existsSync(repos) && readdirSync(repos).some((r) => existsSync(join(repos, r, "runs", id)));
+  let id = base;
+  for (let n = 2; ; n++) {
+    try {
+      if (!taken(id)) {
+        mkdirSync(join(claims, id), { mode: PRIVATE_DIR });
+        mkdirSync(join(root, id), { mode: PRIVATE_DIR });
+        break;
+      }
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+    }
+    id = `${base}-${n}`;
+  }
+  const dir = join(root, id);
+  const p = runPaths(dir);
+  for (const d of [p.lanes, p.roles, p.shots]) ensurePrivateDir(d);
+  writeTextAtomic(p.ledger, `${LEDGER_HEADER}\n`);
+  ensureJsonlHeader(p.runs, "runs");
+  ensureJsonlHeader(p.routes, "routes");
+  ensureJsonlHeader(p.agents, "agents");
+  const meta: RunMeta = {
+    schema: 1,
+    id,
+    repo: o.repo,
+    title: o.title,
+    aLines: o.aLines,
+    createdAt: now.toISOString(),
+    catherdVersion: o.version,
+    ...(o.startedBy ? { startedBy: o.startedBy } : {}),
+  };
+  writeJsonAtomic(p.meta, meta);
+  return { id, dir, meta };
+}
+
+export interface RunListing {
+  runs: Run[];
+  /** run folders whose meta.json cannot be read: listed as warnings, never fatal (audit C10) */
+  corrupt: { id: string; dir: string; reason: string }[];
+}
+
+/** Every run of every repo, newest first. */
+export function listRuns(): RunListing {
+  const root = join(dataDir(), "repos");
+  const out: RunListing = { runs: [], corrupt: [] };
+  if (!existsSync(root)) return out;
+  for (const repo of readdirSync(root, { withFileTypes: true })) {
+    const runs = join(root, repo.name, "runs");
+    if (!repo.isDirectory() || !existsSync(runs)) continue;
+    for (const e of readdirSync(runs, { withFileTypes: true })) {
+      if (!e.isDirectory()) continue;
+      const dir = join(runs, e.name);
+      try {
+        out.runs.push({ id: e.name, dir, meta: readVersioned(runPaths(dir).meta, RunMetaSchema, 1) });
+      } catch (err) {
+        out.corrupt.push({ id: e.name, dir, reason: (err as Error).message.split("\n")[0] ?? "unreadable" });
+      }
+    }
+  }
+  out.runs.sort((a, b) => b.meta.createdAt.localeCompare(a.meta.createdAt));
+  return out;
+}
+
+/**
+ * Every run's records on a backend's thread, oldest first: a role may resume a thread an earlier catherd
+ * run started. `run` counts even when a listing would miss it.
+ */
+export function recordsOnThread(run: Run, backend: string, thread: string): RunRecord[] {
+  const runs = listRuns().runs;
+  return (runs.some((r) => r.dir === run.dir) ? runs : [run, ...runs])
+    .flatMap((r) => readRecords(r).records)
+    .filter((r) => r.backend === backend && r.thread === thread)
+    .sort((a, b) => a.endedAt.localeCompare(b.endedAt));
+}
+
+/** The fix of an unknown run id, in the MCP tools' words; the CLI prints its own (entry/cli-kit.ts). */
+export const RUN_NOT_FOUND_FIX = "status() lists the runs";
+
+export function findRun(id: string): Run {
+  assertId("run", id);
+  const { runs, corrupt } = listRuns();
+  const run = runs.find((r) => r.id === id);
+  if (run) return run;
+  const bad = corrupt.find((r) => r.id === id);
+  if (bad)
+    throw new CatherdError("E_RUN_CORRUPT", `run ${id} cannot be read: ${bad.reason}`, {
+      fix: `fix or delete ${bad.dir}`,
+    });
+  throw new CatherdError("E_RUN_NOT_FOUND", `no run "${id}"`, { fix: RUN_NOT_FOUND_FIX });
+}
+
+/** The run's records, one per dispatch; rows that are torn, invalid or repeated are skipped and counted. */
+export function readRecords(run: Run): { records: RunRecord[]; corrupt: number } {
+  const { rows, corrupt } = readJsonl<unknown>(runPaths(run.dir).runs);
+  const seen = new Set<string>();
+  const records: RunRecord[] = [];
+  let bad = corrupt;
+  for (const row of rows) {
+    const r = RunRecordSchema.safeParse(row);
+    if (!r.success) bad++;
+    else if (!seen.has(r.data.dispatchId)) {
+      seen.add(r.data.dispatchId);
+      records.push(r.data);
+    }
+  }
+  return { records, corrupt: bad };
+}
+
+/** Appends `r` unless its dispatch already has a record, and returns the record that stands (audit C2). */
+export function appendRecord(run: Run, r: RunRecord): Promise<RunRecord> {
+  const file = runPaths(run.dir).runs;
+  return withFileLock(file, () => {
+    const existing = readRecords(run).records.find((x) => x.dispatchId === r.dispatchId);
+    if (existing) return existing;
+    ensureJsonlHeader(file, "runs");
+    appendJsonl(file, RunRecordSchema.parse(r));
+    return r;
+  });
+}
+
+export function readRoutes(run: Run): RouteRow[] {
+  return readJsonl<RouteRow>(runPaths(run.dir).routes).rows.filter(
+    (r) => typeof r?.lane === "string" && Array.isArray(r.ladder) && typeof r.rung === "string",
+  );
+}
+
+export function appendRoute(run: Run, row: RouteRow): void {
+  const file = runPaths(run.dir).routes;
+  ensureJsonlHeader(file, "routes");
+  appendJsonl(file, row);
+}
+
+/** Appends; a lane may be written more than once and its last row wins (see `latestOutcomes`). */
+export function appendOutcome(run: Run, row: OutcomeRow): void {
+  const file = runPaths(run.dir).outcomes;
+  ensureJsonlHeader(file, "outcomes");
+  appendJsonl(file, row);
+}
+
+export function readOutcomes(run: Run): OutcomeRow[] {
+  return readJsonl<OutcomeRow>(runPaths(run.dir).outcomes).rows.filter((r) => typeof r?.lane === "string");
+}
+
+const AgentRunSchema = z.looseObject({
+  at: z.string(),
+  name: z.string(),
+  role: z.string(),
+  rung: z.string(),
+  agent: z.string().nullable(),
+  totalTokens: z.number().nonnegative(),
+  costUsd: z.number().nullable(),
+  secs: z.number().nullable(),
+  status: z.enum(["ok", "failed", "cancelled"]),
+  /** the lane the subagent worked, so catalog timings count it under that lane's kind; rows before it lack it */
+  lane: z.string().nullable().optional(),
+});
+export type AgentRun = z.infer<typeof AgentRunSchema>;
+
+/** Native Claude subagent runs, as the orchestrator reported them through record_agent_run. */
+export function readAgentRuns(run: Run): AgentRun[] {
+  return readJsonl<unknown>(runPaths(run.dir).agents).rows.flatMap((row) => {
+    const r = AgentRunSchema.safeParse(row);
+    return r.success ? [r.data] : [];
+  });
+}
+
+export function appendAgentRun(run: Run, a: AgentRun): void {
+  const file = runPaths(run.dir).agents;
+  ensureJsonlHeader(file, "agents");
+  appendJsonl(file, AgentRunSchema.parse(a));
+}
+
+export function appendLedger(run: Run, row: string): void {
+  appendPrivate(runPaths(run.dir).ledger, `${row}\n`);
+}
+
+/** Spec §4.7: what past runs of a repo learned, keyed by its git toplevel. */
+export const knowledgeFile = (toplevel: string): string => join(repoDir(toplevel), "knowledge.md");
+
+const SERVER_OWNED = new Set([
+  "meta.json",
+  "state.md",
+  "state.json",
+  "ledger.md",
+  "runs.jsonl",
+  "routes.jsonl",
+  "jev.jsonl",
+  "agents.jsonl",
+  "harness.jsonl",
+  "verifier.jsonl",
+  "questions.jsonl",
+  "outcomes.jsonl",
+  "sessions.jsonl",
+]);
+
+const present = (p: string) => {
+  try {
+    lstatSync(p);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * The absolute path of `path` inside the run folder. Refuses a path, or a symlink on its way, that
+ * leaves the folder; for writes, also catherd's own files, compared case-folded (audit S7).
+ */
+export function runFile(run: Run, path: string, mode: "read" | "write"): string {
+  const full = resolve(run.dir, path);
+  const rel = relative(run.dir, full);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || resolve(run.dir, rel) !== full)
+    throw new CatherdError("E_IO_PATH", `"${path}" is outside the run folder`, {
+      fix: "pass a path relative to the run folder, like lanes/M1.L1.md",
+    });
+  const folded = rel.toLowerCase();
+  if (
+    mode === "write" &&
+    (SERVER_OWNED.has(folded) ||
+      folded === "roles" ||
+      folded.startsWith(`roles${sep}`) ||
+      folded === "digests" ||
+      folded.startsWith(`digests${sep}`) ||
+      folded.endsWith(".lock"))
+  )
+    throw new CatherdError("E_IO_PATH", `${rel} is written by catherd itself`, {
+      fix: "write plan.md, lanes/<id>.md, a dossier or notes instead",
+    });
+  let probe = full;
+  while (!present(probe)) probe = dirname(probe);
+  const root = realpathSync(run.dir);
+  const real = realpathSync(probe);
+  if (real !== root && !real.startsWith(`${root}${sep}`))
+    throw new CatherdError("E_IO_PATH", `"${path}" resolves outside the run folder`, {
+      fix: "remove the symlink that leaves the run folder",
+    });
+  return full;
+}

package/src/services/runs-page.ts

@@ -0,0 +1,254 @@
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { budgetStatus } from "../domain/budget.ts";
+import { ID_PATTERN } from "../domain/ids.ts";
+import type { RunRecord } from "../domain/record.ts";
+import { dispatchPaths } from "../infra/dispatch-dir.ts";
+import { nonBlankLines } from "../infra/store.ts";
+import { spendOf } from "./budget.ts";
+import { dispatchState, listDispatches, liveDispatches } from "./dispatches.ts";
+import { lastActivity } from "./finalize.ts";
+import type { Deps } from "./ports.ts";
+import { findRun, listRuns, readRecords, type Run, runPaths } from "./run-store.ts";
+import { groupRuns, type SessionGroup } from "./session-view.ts";
+
+/**
+ * Spec §4: what the Runs tab shows. The top level is the sessions; a session's screen holds its runs, each with its
+ * milestones and every role (live ones first); a role's screen holds its brief, reply and record.
+ */
+
+export interface SessionRow {
+  /** the session id; null for "earlier runs" */
+  key: string | null;
+  name: string;
+  live: boolean;
+  runs: number;
+  liveRoles: number;
+  landed: number;
+  /** ISO time of its newest activity */
+  lastActivity: string;
+}
+
+export interface RoleRow {
+  run: string;
+  dispatchId: string;
+  name: string;
+  role: string;
+  rung: string;
+  /** running | starting while live; once finished, the record's status (`finished` until it has one) */
+  status: string;
+  live: boolean;
+  /** a live role's admission time: the page ticks its elapsed time from it */
+  since: string;
+  /** a finished role's seconds, from its record */
+  secs: number | null;
+  /** a live role's last command, file edit or message line */
+  lastEvent: string | null;
+  replyStatus: string | null;
+}
+
+export interface Milestone {
+  name: string;
+  landed: boolean;
+  /** the ledger's "what" once landed */
+  what: string;
+}
+
+export interface SessionRun {
+  id: string;
+  title: string;
+  repo: string;
+  createdAt: string;
+  continued: "here" | "elsewhere" | null;
+  continuedIn: string | null;
+  /** the budget's spent fraction, null without a cap */
+  budget: number | null;
+  milestones: Milestone[];
+  roles: RoleRow[];
+}
+
+export interface SessionDetail {
+  session: SessionRow;
+  runs: SessionRun[];
+  /** the run folders to watch for changes */
+  dirs: string[];
+}
+
+export interface RoleDetail {
+  run: string;
+  runTitle: string;
+  dispatchId: string;
+  name: string;
+  role: string;
+  rung: string;
+  /** running | starting | finished */
+  state: string;
+  brief: string;
+  reply: string;
+  record: RunRecord | null;
+}
+
+export interface MilestoneDetail {
+  run: string;
+  runTitle: string;
+  name: string;
+  landed: boolean;
+  what: string;
+  /** R/digests/<name>.md; null when land has not written one (not landed, or landed by 1.0) or it is empty */
+  digest: string | null;
+}
+
+const keyOf = (g: SessionGroup): string | null => g.session?.sessionId ?? null;
+
+/** The runs that live on in a session now (it owns them): each run's live roles and landings count once. */
+const ownRuns = (g: SessionGroup): Run[] => g.runs.filter((x) => x.current).map((x) => x.run);
+
+function rowOf(deps: Deps, g: SessionGroup): SessionRow {
+  const own = ownRuns(g);
+  return {
+    key: keyOf(g),
+    name: g.session?.name ?? "earlier runs",
+    live: g.session?.live ?? false,
+    runs: g.runs.length,
+    liveRoles: own.reduce((n, r) => n + liveDispatches(r, deps.now()).length, 0),
+    landed: own.reduce((n, r) => n + Math.max(0, nonBlankLines(runPaths(r.dir).ledger).length - 1), 0),
+    lastActivity: new Date(g.lastActivity).toISOString(),
+  };
+}
+
+/** The Runs tab's top level: every session, newest activity first, "earlier runs" last. */
+export function sessionRows(deps: Deps): { rows: SessionRow[]; warnings: string[] } {
+  const { runs, corrupt } = listRuns();
+  return {
+    rows: groupRuns(runs).map((g) => rowOf(deps, g)),
+    warnings: corrupt.map((c) => `skipped run ${c.id}: ${c.reason}`),
+  };
+}
+
+/** The ledger's landed milestones, then each milestone a lane file names that has not landed. */
+function milestonesOf(run: Run): Milestone[] {
+  const landed = nonBlankLines(runPaths(run.dir).ledger)
+    .slice(1)
+    .map((l) => l.split(" | "))
+    .map(([name, what]) => ({ name: (name ?? "").trim(), landed: true, what: (what ?? "").trim() }));
+  const lanes = existsSync(runPaths(run.dir).lanes) ? readdirSync(runPaths(run.dir).lanes) : [];
+  const open = [...new Set(lanes.map((f) => /^(M\d+)\./.exec(f)?.[1]).filter((m): m is string => !!m))]
+    .filter((m) => !landed.some((l) => l.name === m))
+    .map((name) => ({ name, landed: false, what: "" }));
+  const n = (m: string) => Number(/\d+/.exec(m)?.[0] ?? 0);
+  return [...landed, ...open.sort((a, b) => n(a.name) - n(b.name))];
+}
+
+/** Each role's latest dispatch, live ones first, then the newest finished first. */
+function rolesOf(deps: Deps, run: Run): RoleRow[] {
+  const now = deps.now();
+  const records = new Map(readRecords(run).records.map((r) => [r.dispatchId, r]));
+  const latest = new Map<string, ReturnType<typeof listDispatches>[number]>();
+  for (const d of listDispatches(run)) latest.set(d.admit.name, d);
+  const rows = [...latest.values()].map((d): RoleRow => {
+    const r = records.get(d.admit.dispatchId);
+    const state = r ? "finished" : dispatchState(d, now);
+    const live = state !== "finished";
+    return {
+      run: run.id,
+      dispatchId: d.admit.dispatchId,
+      name: d.admit.name,
+      role: d.admit.role,
+      rung: d.admit.rung,
+      status: r?.status ?? state,
+      live,
+      since: d.admit.admittedAt,
+      secs: r?.secs ?? null,
+      lastEvent: live ? lastActivity(d) : null,
+      replyStatus: r?.replyStatus ?? null,
+    };
+  });
+  return rows.sort((a, b) =>
+    a.live !== b.live ? (a.live ? -1 : 1) : b.dispatchId.localeCompare(a.dispatchId),
+  );
+}
+
+function sessionRun(deps: Deps, x: SessionGroup["runs"][number]): SessionRun {
+  const run = x.run;
+  let budget: number | null = null;
+  try {
+    const b = budgetStatus(
+      spendOf(run, readRecords(run).records, liveDispatches(run, deps.now()), deps.now()),
+      deps.profiles.forRepo(run.meta.repo).budget,
+    );
+    budget = b?.fraction ?? null;
+  } catch {
+    // an unreadable profile: no bar, the rest of the page stands
+  }
+  return {
+    id: run.id,
+    title: run.meta.title,
+    repo: run.meta.repo,
+    createdAt: run.meta.createdAt,
+    continued: x.continued,
+    continuedIn: x.continuedIn,
+    budget,
+    milestones: milestonesOf(run),
+    roles: rolesOf(deps, run),
+  };
+}
+
+/** A session's screen: its runs, newest first, each with its milestones and roles. */
+export function sessionDetail(deps: Deps, key: string | null): SessionDetail {
+  const g = groupRuns(listRuns().runs).find((x) => keyOf(x) === key);
+  if (!g) throw new Error(key === null ? "no earlier runs" : `no session "${key}" in the runs`);
+  return {
+    session: rowOf(deps, g),
+    runs: g.runs.map((x) => sessionRun(deps, x)),
+    dirs: g.runs.map((x) => x.run.dir),
+  };
+}
+
+const text = (file: string): string => {
+  try {
+    return readFileSync(file, "utf8");
+  } catch {
+    return "";
+  }
+};
+
+/** A role's screen: its brief, its reply and its record. */
+export function roleDetail(deps: Deps, runId: string, dispatchId: string): RoleDetail {
+  const run = findRun(runId);
+  const d = listDispatches(run).find((x) => x.admit.dispatchId === dispatchId);
+  if (!d) throw new Error(`no dispatch ${dispatchId} in run ${runId}`);
+  const record = readRecords(run).records.find((r) => r.dispatchId === dispatchId) ?? null;
+  const p = dispatchPaths(d.dir);
+  return {
+    run: run.id,
+    runTitle: run.meta.title,
+    dispatchId,
+    name: d.admit.name,
+    role: d.admit.role,
+    rung: d.admit.rung,
+    state: record ? "finished" : dispatchState(d, deps.now()),
+    brief: text(p.brief),
+    reply: text(p.reply),
+    record,
+  };
+}
+
+/** Spec 1.1 §10: a milestone's screen, its digest. */
+export function milestoneDetail(runId: string, name: string): MilestoneDetail {
+  const run = findRun(runId);
+  // the id rule land writes digests under (assertId): never a path out of the run folder
+  const valid = ID_PATTERN.test(name) && !name.includes("..");
+  const m = valid ? milestonesOf(run).find((x) => x.name === name) : undefined;
+  if (!m) throw new Error(`no milestone "${name}" in run ${runId}`);
+  const file = join(run.dir, "digests", `${name}.md`);
+  const text = existsSync(file) ? readFileSync(file, "utf8") : "";
+  return {
+    run: run.id,
+    runTitle: run.meta.title,
+    name,
+    landed: m.landed,
+    what: m.what,
+    // an empty file is no digest either: land writes the whole digest at once
+    digest: text.trim() ? text : null,
+  };
+}