catherd-cli 0.2.0 → 1.0.0

package/src/services/run-store.ts

@@ -0,0 +1,316 @@
+import { existsSync, lstatSync, mkdirSync, readdirSync, realpathSync } from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+import { CatherdError } from "../domain/errors.ts";
+import { assertId } from "../domain/ids.ts";
+import { type RunRecord, RunRecordSchema } from "../domain/record.ts";
+import type { OutcomeRow, RouteRow } from "../domain/route.ts";
+import { slug } from "../domain/util.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { dataDir, repoDir, runsDir } from "../infra/paths.ts";
+import {
+  appendJsonl,
+  ensureJsonlHeader,
+  ensurePrivateDir,
+  PRIVATE_DIR,
+  appendPrivate,
+  readJsonl,
+  readVersioned,
+  writeJsonAtomic,
+  writeTextAtomic,
+} from "../infra/store.ts";
+
+const RunMetaSchema = z.looseObject({
+  schema: z.literal(1),
+  id: z.string(),
+  repo: z.string(),
+  title: z.string(),
+  aLines: z.array(z.string()),
+  createdAt: z.string(),
+  catherdVersion: z.string(),
+});
+type RunMeta = z.infer<typeof RunMetaSchema>;
+
+export interface Run {
+  id: string;
+  dir: string;
+  meta: RunMeta;
+}
+
+/** Spec §4.1. */
+export function runPaths(dir: string) {
+  return {
+    meta: join(dir, "meta.json"),
+    plan: join(dir, "plan.md"),
+    lanes: join(dir, "lanes"),
+    ledger: join(dir, "ledger.md"),
+    state: join(dir, "state.md"),
+    stateJson: join(dir, "state.json"),
+    runs: join(dir, "runs.jsonl"),
+    routes: join(dir, "routes.jsonl"),
+    jev: join(dir, "jev.jsonl"),
+    outcomes: join(dir, "outcomes.jsonl"),
+    agents: join(dir, "agents.jsonl"),
+    harness: join(dir, "harness.jsonl"),
+    roles: join(dir, "roles"),
+    shots: join(dir, "shots"),
+    /** the admission lock's target: `admission.lock` guards dispatch admission */
+    admission: join(dir, "admission"),
+  };
+}
+
+export const LEDGER_HEADER = "milestone | what | commit | minutes | evidence";
+
+const runSlug = (title: string) => slug(title).slice(0, 40) || "run";
+const stamp = (d: Date) => d.toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15);
+
+/** Creates a run folder under the repo's runs dir. `repo` is a git toplevel; meta.json is written last. */
+export function createRun(o: {
+  repo: string;
+  title: string;
+  aLines: string[];
+  version: string;
+  now?: Date;
+}): Run {
+  const now = o.now ?? new Date();
+  const root = runsDir(o.repo);
+  ensurePrivateDir(root);
+  const base = `${stamp(now)}-${runSlug(o.title)}`;
+  // findRun looks an id up across every repo, so an id is claimed across every repo too: the claim
+  // folder is atomic between processes, the scan covers runs made before claims existed
+  const claims = join(dataDir(), "run-ids");
+  ensurePrivateDir(claims);
+  const repos = join(dataDir(), "repos");
+  const taken = (id: string) =>
+    existsSync(repos) && readdirSync(repos).some((r) => existsSync(join(repos, r, "runs", id)));
+  let id = base;
+  for (let n = 2; ; n++) {
+    try {
+      if (!taken(id)) {
+        mkdirSync(join(claims, id), { mode: PRIVATE_DIR });
+        mkdirSync(join(root, id), { mode: PRIVATE_DIR });
+        break;
+      }
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+    }
+    id = `${base}-${n}`;
+  }
+  const dir = join(root, id);
+  const p = runPaths(dir);
+  for (const d of [p.lanes, p.roles, p.shots]) ensurePrivateDir(d);
+  writeTextAtomic(p.ledger, `${LEDGER_HEADER}\n`);
+  ensureJsonlHeader(p.runs, "runs");
+  ensureJsonlHeader(p.routes, "routes");
+  ensureJsonlHeader(p.agents, "agents");
+  const meta: RunMeta = {
+    schema: 1,
+    id,
+    repo: o.repo,
+    title: o.title,
+    aLines: o.aLines,
+    createdAt: now.toISOString(),
+    catherdVersion: o.version,
+  };
+  writeJsonAtomic(p.meta, meta);
+  return { id, dir, meta };
+}
+
+export interface RunListing {
+  runs: Run[];
+  /** run folders whose meta.json cannot be read: listed as warnings, never fatal (audit C10) */
+  corrupt: { id: string; dir: string; reason: string }[];
+}
+
+/** Every run of every repo, newest first. */
+export function listRuns(): RunListing {
+  const root = join(dataDir(), "repos");
+  const out: RunListing = { runs: [], corrupt: [] };
+  if (!existsSync(root)) return out;
+  for (const repo of readdirSync(root, { withFileTypes: true })) {
+    const runs = join(root, repo.name, "runs");
+    if (!repo.isDirectory() || !existsSync(runs)) continue;
+    for (const e of readdirSync(runs, { withFileTypes: true })) {
+      if (!e.isDirectory()) continue;
+      const dir = join(runs, e.name);
+      try {
+        out.runs.push({ id: e.name, dir, meta: readVersioned(runPaths(dir).meta, RunMetaSchema, 1) });
+      } catch (err) {
+        out.corrupt.push({ id: e.name, dir, reason: (err as Error).message.split("\n")[0] ?? "unreadable" });
+      }
+    }
+  }
+  out.runs.sort((a, b) => b.meta.createdAt.localeCompare(a.meta.createdAt));
+  return out;
+}
+
+/**
+ * Every run's records on a backend's thread, oldest first: a role may resume a thread an earlier catherd
+ * run started. `run` counts even when a listing would miss it.
+ */
+export function recordsOnThread(run: Run, backend: string, thread: string): RunRecord[] {
+  const runs = listRuns().runs;
+  return (runs.some((r) => r.dir === run.dir) ? runs : [run, ...runs])
+    .flatMap((r) => readRecords(r).records)
+    .filter((r) => r.backend === backend && r.thread === thread)
+    .sort((a, b) => a.endedAt.localeCompare(b.endedAt));
+}
+
+/** The fix of an unknown run id, in the MCP tools' words; the CLI prints its own (entry/cli-kit.ts). */
+export const RUN_NOT_FOUND_FIX = "status() lists the runs";
+
+export function findRun(id: string): Run {
+  assertId("run", id);
+  const { runs, corrupt } = listRuns();
+  const run = runs.find((r) => r.id === id);
+  if (run) return run;
+  const bad = corrupt.find((r) => r.id === id);
+  if (bad)
+    throw new CatherdError("E_RUN_CORRUPT", `run ${id} cannot be read: ${bad.reason}`, {
+      fix: `fix or delete ${bad.dir}`,
+    });
+  throw new CatherdError("E_RUN_NOT_FOUND", `no run "${id}"`, { fix: RUN_NOT_FOUND_FIX });
+}
+
+/** The run's records, one per dispatch; rows that are torn, invalid or repeated are skipped and counted. */
+export function readRecords(run: Run): { records: RunRecord[]; corrupt: number } {
+  const { rows, corrupt } = readJsonl<unknown>(runPaths(run.dir).runs);
+  const seen = new Set<string>();
+  const records: RunRecord[] = [];
+  let bad = corrupt;
+  for (const row of rows) {
+    const r = RunRecordSchema.safeParse(row);
+    if (!r.success) bad++;
+    else if (!seen.has(r.data.dispatchId)) {
+      seen.add(r.data.dispatchId);
+      records.push(r.data);
+    }
+  }
+  return { records, corrupt: bad };
+}
+
+/** Appends `r` unless its dispatch already has a record, and returns the record that stands (audit C2). */
+export function appendRecord(run: Run, r: RunRecord): Promise<RunRecord> {
+  const file = runPaths(run.dir).runs;
+  return withFileLock(file, () => {
+    const existing = readRecords(run).records.find((x) => x.dispatchId === r.dispatchId);
+    if (existing) return existing;
+    ensureJsonlHeader(file, "runs");
+    appendJsonl(file, RunRecordSchema.parse(r));
+    return r;
+  });
+}
+
+export function readRoutes(run: Run): RouteRow[] {
+  return readJsonl<RouteRow>(runPaths(run.dir).routes).rows.filter(
+    (r) => typeof r?.lane === "string" && Array.isArray(r.ladder) && typeof r.rung === "string",
+  );
+}
+
+export function appendRoute(run: Run, row: RouteRow): void {
+  const file = runPaths(run.dir).routes;
+  ensureJsonlHeader(file, "routes");
+  appendJsonl(file, row);
+}
+
+/** Appends; a lane may be written more than once and its last row wins (see `latestOutcomes`). */
+export function appendOutcome(run: Run, row: OutcomeRow): void {
+  const file = runPaths(run.dir).outcomes;
+  ensureJsonlHeader(file, "outcomes");
+  appendJsonl(file, row);
+}
+
+export function readOutcomes(run: Run): OutcomeRow[] {
+  return readJsonl<OutcomeRow>(runPaths(run.dir).outcomes).rows.filter((r) => typeof r?.lane === "string");
+}
+
+const AgentRunSchema = z.looseObject({
+  at: z.string(),
+  name: z.string(),
+  role: z.string(),
+  rung: z.string(),
+  agent: z.string().nullable(),
+  totalTokens: z.number().nonnegative(),
+  costUsd: z.number().nullable(),
+  secs: z.number().nullable(),
+  status: z.enum(["ok", "failed", "cancelled"]),
+  /** the lane the subagent worked, so catalog timings count it under that lane's kind; rows before it lack it */
+  lane: z.string().nullable().optional(),
+});
+export type AgentRun = z.infer<typeof AgentRunSchema>;
+
+/** Native Claude subagent runs, as the orchestrator reported them through record_agent_run. */
+export function readAgentRuns(run: Run): AgentRun[] {
+  return readJsonl<unknown>(runPaths(run.dir).agents).rows.flatMap((row) => {
+    const r = AgentRunSchema.safeParse(row);
+    return r.success ? [r.data] : [];
+  });
+}
+
+export function appendAgentRun(run: Run, a: AgentRun): void {
+  const file = runPaths(run.dir).agents;
+  ensureJsonlHeader(file, "agents");
+  appendJsonl(file, AgentRunSchema.parse(a));
+}
+
+export function appendLedger(run: Run, row: string): void {
+  appendPrivate(runPaths(run.dir).ledger, `${row}\n`);
+}
+
+/** Spec §4.7: what past runs of a repo learned, keyed by its git toplevel. */
+export const knowledgeFile = (toplevel: string): string => join(repoDir(toplevel), "knowledge.md");
+
+const SERVER_OWNED = new Set([
+  "meta.json",
+  "state.md",
+  "state.json",
+  "ledger.md",
+  "runs.jsonl",
+  "routes.jsonl",
+  "jev.jsonl",
+  "agents.jsonl",
+  "harness.jsonl",
+  "outcomes.jsonl",
+]);
+
+const present = (p: string) => {
+  try {
+    lstatSync(p);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * The absolute path of `path` inside the run folder. Refuses a path, or a symlink on its way, that
+ * leaves the folder; for writes, also catherd's own files, compared case-folded (audit S7).
+ */
+export function runFile(run: Run, path: string, mode: "read" | "write"): string {
+  const full = resolve(run.dir, path);
+  const rel = relative(run.dir, full);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || resolve(run.dir, rel) !== full)
+    throw new CatherdError("E_IO_PATH", `"${path}" is outside the run folder`, {
+      fix: "pass a path relative to the run folder, like lanes/M1.L1.md",
+    });
+  const folded = rel.toLowerCase();
+  if (
+    mode === "write" &&
+    (SERVER_OWNED.has(folded) ||
+      folded === "roles" ||
+      folded.startsWith(`roles${sep}`) ||
+      folded.endsWith(".lock"))
+  )
+    throw new CatherdError("E_IO_PATH", `${rel} is written by catherd itself`, {
+      fix: "write plan.md, lanes/<id>.md, a dossier or notes instead",
+    });
+  let probe = full;
+  while (!present(probe)) probe = dirname(probe);
+  const root = realpathSync(run.dir);
+  const real = realpathSync(probe);
+  if (real !== root && !real.startsWith(`${root}${sep}`))
+    throw new CatherdError("E_IO_PATH", `"${path}" resolves outside the run folder`, {
+      fix: "remove the symlink that leaves the run folder",
+    });
+  return full;
+}

package/src/services/setup.ts

@@ -0,0 +1,87 @@
+import { existsSync, readdirSync, readFileSync, renameSync } from "node:fs";
+import { basename, join } from "node:path";
+import type { Issue } from "../domain/profile-rules.ts";
+import { configDir } from "../infra/paths.ts";
+import { type Refreshed, refreshDiscovery } from "./catalog-service.ts";
+import type { Synced } from "./agent-links.ts";
+import { activate, resetProfile, withProfilesLock } from "./profile-service.ts";
+import { configFile, profilesDir, projectsFile } from "./profile-store.ts";
+import { ensurePrivateDir } from "../infra/store.ts";
+
+/** True when profile `name` has a file (`default` exists as a name even before it has one). */
+export const hasProfileFile = (name: string): boolean => existsSync(join(profilesDir(), `${name}.json`));
+
+/** A JSON file written by catherd 0.x: readable JSON with no `schema` field. */
+function isLegacy(file: string): boolean {
+  try {
+    const v = JSON.parse(readFileSync(file, "utf8")) as { schema?: unknown } | null;
+    return typeof v === "object" && v !== null && v.schema === undefined;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Spec D2, a clean break: 1.0 never reads a 0.x config.json, projects.json or profile, so `init` moves
+ * them to `<config>/0.x-backup-<stamp>/`, under the profiles lock. Returns the files moved.
+ */
+export function moveLegacy(now: Date = new Date()): string[] {
+  return withProfilesLock(() => moveLegacyUnlocked(now));
+}
+
+function moveLegacyUnlocked(now: Date): string[] {
+  const candidates = [
+    configFile(),
+    projectsFile(),
+    ...(existsSync(profilesDir())
+      ? readdirSync(profilesDir())
+          .filter((f) => f.endsWith(".json"))
+          .map((f) => join(profilesDir(), f))
+      : []),
+  ].filter((f) => existsSync(f) && isLegacy(f));
+  if (candidates.length === 0) return [];
+  const backup = join(configDir(), `0.x-backup-${now.toISOString().replace(/[:.]/g, "-")}`);
+  ensurePrivateDir(join(backup, "profiles"));
+  return candidates.map((f) => {
+    const to = join(backup, f.startsWith(profilesDir()) ? join("profiles", basename(f)) : basename(f));
+    renameSync(f, to);
+    return to;
+  });
+}
+
+export interface InitResult {
+  moved: string[];
+  profile: string;
+  /** true when the default profile was written under `profile` */
+  created: boolean;
+  /** why the default profile was not written: it does not validate against this machine's catalog */
+  errors: Issue[];
+  /** false when nothing was made active: the profile was not written and has no file */
+  active: boolean;
+  synced: Synced | null;
+  refreshed: Refreshed[];
+}
+
+/**
+ * Spec §8 `catherd init`, the setup half: moves 0.x files aside, writes the default profile (spec §7.2)
+ * unless a 1.0 one exists and `overwrite` is not set, makes it active and links its agents, and lists
+ * every backend's models. It never throws for a default profile that does not validate here: it writes
+ * nothing, returns the errors, and activates the profile only when it has a file.
+ */
+export async function initSetup(
+  o: { profile?: string; overwrite?: boolean; now?: Date } = {},
+): Promise<InitResult> {
+  const moved = moveLegacy(o.now);
+  const profile = o.profile ?? "default";
+  let created = false;
+  let errors: Issue[] = [];
+  if (!hasProfileFile(profile) || o.overwrite === true) {
+    const saved = resetProfile(profile);
+    created = saved.saved;
+    errors = saved.errors;
+  }
+  const active = hasProfileFile(profile);
+  const synced = active ? activate(profile) : null;
+  const refreshed = await refreshDiscovery();
+  return { moved, profile, created, errors, active, synced, refreshed };
+}

package/src/services/state.ts

@@ -0,0 +1,104 @@
+import { relative } from "node:path";
+import { z } from "zod";
+import { errorMessage, isCatherdError } from "../domain/errors.ts";
+import { overlaps } from "../domain/lane.ts";
+import { renderState } from "../domain/state.ts";
+import { dispatchPaths } from "../infra/dispatch-dir.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { gitHead, statusSnapshot } from "../infra/git.ts";
+import { readVersioned, writeJsonAtomic, writeTextAtomic } from "../infra/store.ts";
+import { liveDispatches } from "./dispatches.ts";
+import { type Run, runPaths } from "./run-store.ts";
+
+/** state.json: the orchestrator's notes that state.md shows beside the live facts. */
+const NotesSchema = z.looseObject({
+  schema: z.literal(1),
+  next: z.string(),
+  lastCheck: z.string().nullable(),
+  lastLandedAt: z.string().nullable(),
+});
+export type Notes = z.infer<typeof NotesSchema>;
+export type NotesPatch = Partial<Omit<Notes, "schema">>;
+
+const FRESH: Notes = { schema: 1, next: "plan the milestones", lastCheck: null, lastLandedAt: null };
+
+export function readNotes(run: Run): Notes {
+  try {
+    return readVersioned(runPaths(run.dir).stateJson, NotesSchema, 1);
+  } catch (e) {
+    if (isCatherdError(e) && e.code === "E_CONFIG_NEWER_SCHEMA") throw e;
+    return { ...FRESH };
+  }
+}
+
+/**
+ * Rewrites state.json and state.md under the run's state lock, so concurrent writers never lose a
+ * field (audit C11). `change` may be a function of the current notes; it runs inside the lock.
+ * Throws E_IO_UNEXPECTED, writing nothing, when git fails: state.md must not show a broken git as clean.
+ */
+export function updateState(run: Run, change: NotesPatch | ((n: Notes) => NotesPatch) = {}): Promise<string> {
+  const p = runPaths(run.dir);
+  return withFileLock(p.stateJson, async () => {
+    // git first: a failed snapshot throws before either file is written, so they never disagree
+    const [head, snap] = await Promise.all([gitHead(run.meta.repo), statusSnapshot(run.meta.repo)]);
+    const notes = readNotes(run);
+    const next: Notes = { ...notes, ...(typeof change === "function" ? change(notes) : change) };
+    writeJsonAtomic(p.stateJson, next);
+    const live = liveDispatches(run);
+    const text = renderState({
+      title: run.meta.title,
+      head: head ?? "none",
+      dirty: Object.keys(snap)
+        .sort()
+        .map((path) => ({
+          path,
+          owner: live.find((d) => overlaps([path], d.admit.owns).length > 0)?.admit.name ?? null,
+        })),
+      running: live.map((d) => ({
+        name: d.admit.name,
+        rung: d.admit.rung,
+        thread: d.admit.thread,
+        since: d.admit.admittedAt.slice(11, 16),
+        brief: relative(run.dir, dispatchPaths(d.dir).brief),
+      })),
+      lastCheck: next.lastCheck,
+      next: next.next,
+    });
+    writeTextAtomic(p.state, text);
+    return text;
+  });
+}
+
+/**
+ * Ruling (b): a failed state.md refresh (git broken) never fails the call. The notes still go to state.json
+ * under the state lock (state.md stays as it was), and the message comes back as a hint.
+ */
+export async function refreshState(
+  run: Run,
+  change: NotesPatch | ((n: Notes) => NotesPatch) = {},
+): Promise<{ text: string | null; hints: string[] }> {
+  let applied = false;
+  const apply = (n: Notes): NotesPatch => {
+    applied = true;
+    return typeof change === "function" ? change(n) : change;
+  };
+  try {
+    return { text: await updateState(run, apply), hints: [] };
+  } catch (e) {
+    const hints = [`state.md not refreshed: ${errorMessage(e)}`];
+    // git fails before updateState reaches the notes: keep them, so a later refresh still shows them
+    if (!applied) {
+      const stateJson = runPaths(run.dir).stateJson;
+      try {
+        await withFileLock(stateJson, () => {
+          const notes = readNotes(run);
+          writeJsonAtomic(stateJson, { ...notes, ...apply(notes) });
+        });
+      } catch (e2) {
+        // the lock outwaited (a slow git under another refresh): still never fail the call
+        hints.push(`notes not saved: ${errorMessage(e2)}`);
+      }
+    }
+    return { text: null, hints };
+  }
+}