catherd-cli 1.5.0 → 2.0.0-beta.20261022113309

package/src/services/run-events-store.ts

@@ -0,0 +1,174 @@
+import { closeSync, existsSync, fstatSync, ftruncateSync, openSync, readFileSync, readSync } from "node:fs";
+import { join } from "node:path";
+import { CatherdError, UPGRADE } from "../domain/errors.ts";
+import { EVENTS_SCHEMA, RunEventSchema, type RunEvent, type RunEventInput } from "../domain/run-events.ts";
+import { type RunState, replay } from "../domain/run-state.ts";
+import { withFileLock } from "../infra/filelock.ts";
+import { log } from "../infra/log.ts";
+import { appendJsonlBatch } from "../infra/store.ts";
+import type { Run } from "./run-store.ts";
+
+/** `<run>/events.jsonl`: one fact per line, dense `seq`, the only state a 2.0 run keeps. */
+export const eventsFile = (run: Run): string => join(run.dir, "events.jsonl");
+
+/** A 2.0 run has an events.jsonl; a run without one is a 1.5 run, and the daemon never drives it. */
+export const isDaemonRun = (run: Run): boolean => existsSync(eventsFile(run));
+
+const corrupt = (run: Run, after: number, why: string): CatherdError =>
+  new CatherdError("E_RUN_CORRUPT", `${eventsFile(run)} is corrupt after seq ${after}: ${why}`, {
+    fix: `restore events.jsonl from a backup, or finish run ${run.id} with catherd 1.x`,
+  });
+
+const tooOld = (run: Run, v: unknown): CatherdError =>
+  new CatherdError(
+    "E_RUNTIME_TOO_OLD",
+    `${eventsFile(run)} holds events of version ${String(v)}, newer than this catherd (${EVENTS_SCHEMA})`,
+    { fix: `upgrade catherd: ${UPGRADE}` },
+  );
+
+/** One line as an event: a newer `v` is too old a catherd, anything else invalid is corrupt (null here). */
+function decode(run: Run, line: string): RunEvent | null {
+  let raw: unknown;
+  try {
+    raw = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  const v = (raw as { v?: unknown } | null)?.v;
+  if (typeof v === "number" && v > EVENTS_SCHEMA) throw tooOld(run, v);
+  const r = RunEventSchema.safeParse(raw);
+  return r.success ? r.data : null;
+}
+
+/**
+ * The run's events from `fromSeq` (1 reads them all and checks every line). A last line without its newline
+ * that does not parse is a torn append: skipped and logged. Any other line that does not parse, or a seq that
+ * is not the line's number, refuses the read (E_RUN_CORRUPT, naming the seq before it); a newer `v`, too.
+ * Lines before `fromSeq` are skipped unread: the caller replayed them already.
+ */
+export function readEvents(run: Run, fromSeq = 1): RunEvent[] {
+  let text: string;
+  try {
+    text = readFileSync(eventsFile(run), "utf8");
+  } catch {
+    return [];
+  }
+  const lines = text.split("\n");
+  const tail = lines.pop() ?? "";
+  if (tail) lines.push(tail);
+  const out: RunEvent[] = [];
+  for (const [i, line] of lines.entries()) {
+    if (i + 1 < fromSeq || !line.trim()) continue;
+    const e = decode(run, line);
+    if (e === null) {
+      if (i === lines.length - 1 && line === tail) {
+        log("warn", "events", { run: run.id, torn: i + 1 });
+        break;
+      }
+      throw corrupt(run, i, "a line is not an event");
+    }
+    if (e.seq !== i + 1) throw corrupt(run, i, `seq ${e.seq} stands where ${i + 1} belongs`);
+    out.push(e);
+  }
+  return out;
+}
+
+export const replayRun = (run: Run): RunState => replay(readEvents(run), run.id);
+
+/** The last seq of the file; a torn tail is cut off first, so the next line starts clean. Under the file's lock. */
+function lastSeq(run: Run): number {
+  const file = eventsFile(run);
+  if (!existsSync(file)) return 0;
+  const fd = openSync(file, "r+");
+  try {
+    const size = fstatSync(fd).size;
+    let chunk = 64 * 1024;
+    for (;;) {
+      const start = Math.max(0, size - chunk);
+      const buf = Buffer.alloc(size - start);
+      readSync(fd, buf, 0, buf.length, start);
+      const text = buf.toString("utf8");
+      const cut = text.lastIndexOf("\n");
+      const tail = text.slice(cut + 1);
+      const body = text.slice(0, cut + 1).trimEnd();
+      // the whole last complete line is in the chunk, or the chunk is the whole file
+      if (start > 0 && !body.includes("\n")) {
+        chunk *= 4;
+        continue;
+      }
+      if (tail && decode(run, tail) === null) {
+        // a torn append: not an event, never will be one
+        ftruncateSync(fd, size - Buffer.byteLength(tail));
+        log("warn", "events", { run: run.id, tornTail: Buffer.byteLength(tail) });
+      }
+      const line = tail && decode(run, tail) !== null ? tail : body.slice(body.lastIndexOf("\n") + 1);
+      if (!line) return 0;
+      const last = decode(run, line);
+      if (!last) throw corrupt(run, 0, "its last line is not an event");
+      return last.seq;
+    }
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/** Runs `fn` under the events file's lock, handing it an append that does not take the lock again. */
+export function withEventsLock<T>(
+  run: Run,
+  fn: (add: (e: RunEventInput, now?: Date) => RunEvent) => T | Promise<T>,
+): Promise<T> {
+  return withFileLock(eventsFile(run), () => fn((e, now) => writeEvents(run, [e], now)[0] as RunEvent));
+}
+
+/** The events with the next dense seqs, validated all before the one write that appends them. */
+function writeEvents(run: Run, es: RunEventInput[], now: Date = new Date()): RunEvent[] {
+  const first = lastSeq(run) + 1;
+  const out = es.map((e, i) =>
+    RunEventSchema.parse({ v: EVENTS_SCHEMA, seq: first + i, at: now.toISOString(), ...e }),
+  );
+  appendJsonlBatch(eventsFile(run), out);
+  return out;
+}
+
+/** Appends one event under the file's lock, with the next dense `seq`, and returns it as written. */
+export async function appendEvent(run: Run, e: RunEventInput, now: Date = new Date()): Promise<RunEvent> {
+  return withEventsLock(run, (add) => add(e, now));
+}
+
+/** Appends events that belong together in one write under the file's lock: all of them or, on a crash, a torn tail. */
+export function appendEvents(run: Run, es: RunEventInput[], now: Date = new Date()): Promise<RunEvent[]> {
+  return withFileLock(eventsFile(run), () => writeEvents(run, es, now));
+}
+
+/**
+ * Calls `onEvent` with each event after `fromSeq - 1`, in order, as they are appended, until the returned stop
+ * is called. A poll that finds the file unreadable waits for the next one.
+ */
+export function tailEvents(
+  run: Run,
+  fromSeq: number,
+  onEvent: (e: RunEvent) => void,
+  o: { pollMs?: number } = {},
+): () => void {
+  let next = fromSeq;
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  const poll = (): void => {
+    if (stopped) return;
+    try {
+      for (const e of readEvents(run, next)) {
+        if (stopped) return;
+        next = e.seq + 1;
+        onEvent(e);
+      }
+    } catch (err) {
+      log("warn", "events", { run: run.id, tail: err instanceof Error ? err.message : String(err) });
+    }
+    if (!stopped) timer = setTimeout(poll, o.pollMs ?? 500);
+  };
+  poll();
+  return () => {
+    stopped = true;
+    if (timer) clearTimeout(timer);
+  };
+}

package/src/services/run-evidence.ts

@@ -17,7 +17,7 @@ export interface Evidence {
   fails: number;
 }
 
-/** `<canonical rung>|<kind or *>` → its evidence. Shown by route, catalog list and the TUI; never routes. */
+/** `<canonical rung>|<kind or *>` → its evidence. Shown by route and catalog list; never routes. */
 export type EvidenceTable = Record<string, Evidence>;
 
 const VERDICT_FAIL = /^VERDICT: FAIL\b/;

package/src/services/run-read.ts

@@ -0,0 +1,221 @@
+import { z } from "zod";
+import { errorMessage } from "../domain/errors.ts";
+import { PARK_REASONS, describeEvent } from "../domain/run-events.ts";
+import { type RunState, liveDispatches } from "../domain/run-state.ts";
+import { log } from "../infra/log.ts";
+import { daemonAlive, ensureDaemon, readHeartbeat } from "./daemon.ts";
+import { liveDispatches as liveDispatchesOf } from "./dispatches.ts";
+import { landedMilestones } from "./milestones.ts";
+import type { Deps } from "./ports.ts";
+import { openQuestions } from "./questions.ts";
+import { isDaemonRun, readEvents, replayRun, tailEvents } from "./run-events-store.ts";
+import { findRun, listRuns, type Run } from "./run-store.ts";
+
+/**
+ * The read side of a run, for the panel (plan 31) and the session's `status`: each shape has a zod
+ * schema, which is its contract. A 1.5 run reads as `engine: "1.5"`, with what its folder says.
+ */
+export const ParkedViewSchema = z.object({
+  run: z.string(),
+  runTitle: z.string(),
+  unit: z.string(),
+  question: z.string(),
+  reason: z.enum(PARK_REASONS),
+  at: z.string(),
+  /** the command that answers it */
+  command: z.string(),
+});
+export type ParkedView = z.infer<typeof ParkedViewSchema>;
+
+export const TimelineEntrySchema = z.object({
+  seq: z.number().int(),
+  at: z.string(),
+  type: z.string(),
+  text: z.string(),
+});
+export type TimelineEntry = z.infer<typeof TimelineEntrySchema>;
+
+export const RunViewSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  repo: z.string(),
+  engine: z.enum(["2.0", "1.5"]),
+  phase: z.enum(["planning", "approval", "running", "ended"]).nullable(),
+  outcome: z.enum(["done", "cancelled", "failed"]).nullable(),
+  daemon: z.object({
+    alive: z.boolean(),
+    pid: z.number().int().nullable(),
+    beatAgeSec: z.number().nullable(),
+  }),
+  milestones: z.array(
+    z.object({
+      id: z.string(),
+      title: z.string(),
+      status: z.string(),
+      round: z.number().int(),
+      commit: z.string().nullable(),
+    }),
+  ),
+  live: z.array(
+    z.object({ name: z.string(), role: z.string(), rung: z.string(), unit: z.string(), since: z.string() }),
+  ),
+  parked: z.array(ParkedViewSchema),
+  holds: z.array(z.object({ quota: z.string(), until: z.string() })),
+  /** the latest event, in one line */
+  last: z.string().nullable(),
+  startedAt: z.string(),
+  updatedAt: z.string(),
+});
+export type RunView = z.infer<typeof RunViewSchema>;
+
+export { tailEvents };
+
+const answerCommand = (run: string, unit: string): string => `catherd answer ${run} ${unit} "<answer>"`;
+
+function parkedOf(run: Run, s: RunState): ParkedView[] {
+  return Object.values(s.questions).map((q) => ({
+    run: run.id,
+    runTitle: run.meta.title,
+    unit: q.unit,
+    question: q.question,
+    reason: q.reason,
+    at: q.at,
+    command: answerCommand(run.id, q.unit),
+  }));
+}
+
+/** A daemon of an unfinished 2.0 run is started again when it is dead; a failure to start never fails a read. */
+function revive(run: Run): void {
+  try {
+    ensureDaemon(run);
+  } catch (e) {
+    log("warn", "daemon", { run: run.id, revive: errorMessage(e) });
+  }
+}
+
+function view15(deps: Deps, run: Run): RunView {
+  const now = deps.now();
+  return {
+    id: run.id,
+    title: run.meta.title,
+    repo: run.meta.repo,
+    engine: "1.5",
+    phase: null,
+    outcome: null,
+    daemon: { alive: false, pid: null, beatAgeSec: null },
+    milestones: landedMilestones(run).map((id) => ({
+      id,
+      title: id,
+      status: "landed",
+      round: 0,
+      commit: null,
+    })),
+    live: liveDispatchesOf(run, now).map((d) => ({
+      name: d.admit.name,
+      role: d.admit.role,
+      rung: d.admit.rung,
+      unit: d.admit.lane ?? d.admit.name,
+      since: d.admit.admittedAt,
+    })),
+    parked: openQuestions(run).map((q) => ({
+      run: run.id,
+      runTitle: run.meta.title,
+      unit: q.milestone,
+      question: q.question,
+      reason: "needs_user" as const,
+      at: q.at,
+      command: `answer(run: "${run.id}", milestone: "${q.milestone}", answer: "<answer>")`,
+    })),
+    holds: [],
+    last: null,
+    startedAt: run.meta.createdAt,
+    updatedAt: run.meta.createdAt,
+  };
+}
+
+/** One run at a glance. Reading an unfinished 2.0 run starts its daemon again if it died. */
+export function runView(deps: Deps, runId: string): RunView {
+  const run = findRun(runId);
+  if (!isDaemonRun(run)) return view15(deps, run);
+  revive(run);
+  const s = replayRun(run);
+  const beat = readHeartbeat(run);
+  const events = readEvents(run);
+  const last = events.at(-1);
+  return {
+    id: run.id,
+    title: run.meta.title,
+    repo: run.meta.repo,
+    engine: "2.0",
+    phase: s.phase,
+    outcome: s.outcome,
+    daemon: {
+      alive: daemonAlive(run),
+      pid: beat?.pid ?? null,
+      beatAgeSec: beat ? Math.max(0, Math.round((deps.now() - Date.parse(beat.beatAt)) / 1000)) : null,
+    },
+    milestones: s.milestones.map((m) => ({
+      id: m.id,
+      title: m.title,
+      status: m.status,
+      round: m.round,
+      commit: m.commit,
+    })),
+    live: liveDispatches(s).map((d) => ({
+      name: d.name,
+      role: d.role,
+      rung: d.rung,
+      unit: d.unit,
+      since: d.startedAt,
+    })),
+    parked: parkedOf(run, s),
+    holds: Object.entries(s.holds).map(([quota, until]) => ({ quota, until })),
+    last: last ? describeEvent(last, (id) => s.dispatches[id]?.rung) : null,
+    startedAt: run.meta.createdAt,
+    updatedAt: last?.at ?? run.meta.createdAt,
+  };
+}
+
+/** Every run, newest first (one repo's with `repo`), at most `limit` (default 50). */
+export function listRunViews(deps: Deps, o: { repo?: string; limit?: number } = {}): RunView[] {
+  const runs = listRuns(o.repo ? [o.repo] : undefined).runs.slice(0, o.limit ?? 50);
+  const out: RunView[] = [];
+  for (const r of runs) {
+    try {
+      out.push(runView(deps, r.id));
+    } catch (e) {
+      log("warn", "read", { run: r.id, error: errorMessage(e) });
+    }
+  }
+  return out;
+}
+
+/** The run's events from `fromSeq`, one line each (never a reply's body), at most `limit` (default 200). */
+export function runTimeline(runId: string, fromSeq = 1, limit = 200): TimelineEntry[] {
+  const run = findRun(runId);
+  if (!isDaemonRun(run)) return [];
+  const s = replayRun(run);
+  return readEvents(run, fromSeq)
+    .slice(0, limit)
+    .map((e) => ({
+      seq: e.seq,
+      at: e.at,
+      type: e.type,
+      text: describeEvent(e, (id) => s.dispatches[id]?.rung),
+    }));
+}
+
+/** Every unfinished 2.0 run's open questions (one run's with `run`), each with the command that answers it. */
+export function parkedQuestions(o: { run?: string } = {}): ParkedView[] {
+  const runs = o.run ? [findRun(o.run)] : listRuns().runs;
+  return runs.flatMap((run) => {
+    try {
+      if (!isDaemonRun(run)) return [];
+      const s = replayRun(run);
+      return s.phase === "ended" ? [] : parkedOf(run, s);
+    } catch (e) {
+      log("warn", "read", { run: run.id, error: errorMessage(e) });
+      return [];
+    }
+  });
+}

package/src/services/run-report.ts

@@ -0,0 +1,176 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { JudgeDecision, RuleChange } from "../domain/judge.ts";
+import { type Finding, issueLine, type RunEvent } from "../domain/run-events.ts";
+import { findingKey, type RunState } from "../domain/run-state.ts";
+import { writeTextAtomic } from "../infra/store.ts";
+import { readChecks } from "./milestone-check.ts";
+import { readRecords, type Run } from "./run-store.ts";
+
+const read = (file: string): string => {
+  try {
+    return readFileSync(file, "utf8").trim();
+  } catch {
+    return "";
+  }
+};
+
+/** What a judge decided, in one line: the verb's own payload. */
+function said(d: JudgeDecision): string {
+  switch (d.verb) {
+    case "route":
+      return d.routes.map((r) => `${r.finding} -> ${r.to}`).join(", ");
+    case "answer":
+      return d.answer;
+    case "ask_owner":
+    case "park":
+      return d.question;
+    case "rules":
+      return `${d.changes.length} change(s) proposed`;
+    default:
+      return d.why;
+  }
+}
+
+const quote = (text: string): string => `'${text.replace(/'/g, "'\\''")}'`;
+
+/**
+ * The `## Rules review` section (plan 34): what the finished run's judge proposed, each change followed by the
+ * command that applies it. The run applies none; null when the run had no review (cancelled, failed).
+ */
+function rulesSection(failed: boolean, proposals: RuleChange[] | undefined): string[] | null {
+  if (failed) return ["## Rules review", "", "No rules review: the judge gave no readable answer.", ""];
+  if (!proposals) return null;
+  if (proposals.length === 0)
+    return ["## Rules review", "", "The judge proposed no change to the standing rules.", ""];
+  return [
+    "## Rules review",
+    "",
+    "Standing rules to review: apply any of these with the command under it.",
+    ...proposals.flatMap((c, i) => [
+      `${i + 1}. ${c.op === "add" ? "ADD" : "REMOVE"} ${c.text}${c.why ? ` — ${c.why}` : ""}`,
+      `   \`catherd knowledge ${c.op === "add" ? "add" : "rm"} ${quote(c.text)}\``,
+    ]),
+    "",
+  ];
+}
+
+/** The "fixed" part of a milestone's `Rounds:` line shows at most this many findings. */
+export const FIXED_LINES = 15;
+
+/**
+ * The findings of each review round of milestone `m`: a `commit.made` opens a round, and every ok reviewer end
+ * (`reviewer`, `ui-reviewer`, the partial pass `reviewer-<M>-2`) adds to it. A round with no reviewer end is left out.
+ */
+export function reviewRounds(events: RunEvent[], m: string): Finding[][] {
+  const reviewers = new Set<string>();
+  const rounds: Finding[][] = [];
+  let open: Finding[] | null = null;
+  for (const e of events) {
+    if (e.type === "dispatch.started") {
+      if ((e.role === "reviewer" || e.role === "ui-reviewer") && e.unit === m) reviewers.add(e.dispatchId);
+    } else if (e.type === "commit.made" && e.milestone === m) open = null;
+    else if (
+      e.type === "dispatch.ended" &&
+      reviewers.has(e.dispatchId) &&
+      e.status === "ok" &&
+      e.replyStatus !== "blocked" &&
+      e.replyStatus !== "refused"
+    ) {
+      if (!open) rounds.push((open = []));
+      open.push(...e.findings);
+    }
+  }
+  return rounds;
+}
+
+/** `Rounds: 3 review round(s) · fixed: <the BLOCKER and BUG lines an earlier round named and the last did not repeat>`. */
+export function roundsLine(events: RunEvent[], m: string): string {
+  const rounds = reviewRounds(events, m);
+  const last = new Set((rounds.at(-1) ?? []).map((f) => findingKey(issueLine(f))));
+  const seen = new Set<string>();
+  const fixed: string[] = [];
+  for (const f of rounds.slice(0, -1).flat()) {
+    const key = findingKey(issueLine(f));
+    if (f.label === "NIT" || last.has(key) || seen.has(key)) continue;
+    seen.add(key);
+    fixed.push(issueLine(f));
+  }
+  const head = `Rounds: ${rounds.length} review round(s)`;
+  return fixed.length ? `${head} · fixed: ${fixed.slice(0, FIXED_LINES).join(" · ")}` : head;
+}
+
+/** Per command, the latest check record of this run and milestone that is no baseline; "Gate: not recorded" when none. */
+export function gateLines(run: Run, m: string): string[] {
+  const latest = new Map<string, ReturnType<typeof readChecks>[number]>();
+  try {
+    for (const r of readChecks(run.meta.repo))
+      if (r.run === run.id && r.milestone === m && !r.baseline) latest.set(r.command, r);
+  } catch {
+    // a check store that cannot be read records nothing
+  }
+  return latest.size
+    ? [...latest.values()].map(
+        (r) =>
+          `Gate: \`${r.command}\` exit ${r.exit} on ${r.head.slice(0, 12)} · ${Math.round(r.durationMs / 1000)} s · secret scan: ${r.secrets ?? "not run"}`,
+      )
+    : ["Gate: not recorded"];
+}
+
+const oneLine = (s: string): string => s.replace(/\s*\n\s*/g, " ").trim();
+
+/**
+ * `<run>/report.md`, the body of the merge request: the run's title, each landed milestone's digest, the known
+ * issues verbatim by milestone, and the audit: every judge decision, and what each milestone's check passed on
+ * only through the base or a rerun (`audits`). Returns its path.
+ */
+export function writeRunReport(
+  run: Run,
+  s: RunState,
+  events: RunEvent[],
+  audits: Record<string, string[]> = {},
+  proposals?: RuleChange[],
+): string {
+  const landed = events.filter((e) => e.type === "milestone.landed");
+  const decided = events.filter((e) => e.type === "judge.decided");
+  const lines = [`# ${s.title || run.meta.title}`, ""];
+  for (const e of landed) {
+    const title = s.milestones.find((m) => m.id === e.milestone)?.title;
+    const digest = read(e.digestPath.startsWith("/") ? e.digestPath : join(run.dir, e.digestPath));
+    lines.push(
+      `## ${e.milestone}${title && title !== e.milestone ? `: ${title}` : ""}`,
+      "",
+      digest || `Landed at ${e.commit.slice(0, 12)}.`,
+      "",
+      roundsLine(events, e.milestone),
+      ...gateLines(run, e.milestone),
+      "",
+    );
+  }
+  if (landed.length === 0) lines.push("No milestone landed.", "");
+  lines.push("## Known issues", "");
+  const issues = landed.flatMap((e) => e.knownIssues.map((k) => `- ${e.milestone}: ${k}`));
+  lines.push(...(issues.length ? issues : ["None."]), "");
+  // plan 34: the latest lint of each milestone's preflight
+  const lint = [
+    ...new Set(events.flatMap((e) => (e.type === "preflight.done" ? [e.milestone] : []))),
+  ].flatMap((m) => {
+    const latest = events.findLast((e) => e.type === "preflight.done" && e.milestone === m);
+    return latest?.type === "preflight.done" ? latest.lint.map((l) => `- ${m}: ${l}`) : [];
+  });
+  if (lint.length) lines.push("## Plan lint", "", ...lint, "");
+  lines.push("## Audit", "");
+  const audit = [
+    ...decided.map((e) => `- ${e.unit} (${e.kind}): ${e.decision.verb}: ${oneLine(said(e.decision))}`),
+    ...Object.entries(audits).flatMap(([m, lines]) => lines.map((l) => `- ${m} (check): ${oneLine(l)}`)),
+    // a worker's conflict with its plan task, in record order
+    ...readRecords(run).records.flatMap((r) =>
+      (r.design ?? []).map((d) => `- ${r.name} DESIGN: ${oneLine(d)}`),
+    ),
+  ];
+  lines.push(...(audit.length ? audit : ["No judge decision."]), "");
+  lines.push(...(rulesSection(s.rulesReview?.failed === true, proposals) ?? []));
+  const file = join(run.dir, "report.md");
+  writeTextAtomic(file, lines.join("\n"));
+  return file;
+}