@dsdenes/devboard 0.1.560 → 0.1.562

package/server/journal.test.ts

@@ -0,0 +1,350 @@
+import { pendingRecord } from "./fixtures/adr-source";
+import { afterAll, describe, expect, test } from "bun:test";
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, readFileSync, readdirSync, statSync, unlinkSync, utimesSync } from "node:fs";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import type { Server } from "bun";
+import { createApp } from "./app";
+import {
+  JOURNAL_MAX_BYTES,
+  openJournal,
+  pruneJournal,
+  type Journal,
+  type JournalRecord,
+} from "./journal";
+
+/**
+ * New tests keep their temporary directories inside the repository. The app
+ * fixture proves the journal is written by a real run; the rest prove rotation,
+ * pruning, and the size bound against a fixture clock.
+ */
+
+const harnessRoot = resolve(import.meta.dir, "..");
+const repositoryRoot = resolve(harnessRoot, "..");
+const testRoot = join(repositoryRoot, ".devboard", ".tmp");
+const STUB_AGENT = join(import.meta.dir, "fixtures", "stub-agent.ts");
+const temporaryDirectories: string[] = [];
+
+async function temporaryDirectory() {
+  await mkdir(testRoot, { recursive: true });
+  const directory = await mkdtemp(join(testRoot, `test-journal-${randomUUID()}-`));
+  temporaryDirectories.push(directory);
+  return directory;
+}
+
+afterAll(async () => {
+  await Promise.all(
+    temporaryDirectories.map((directory) => rm(directory, { recursive: true, force: true })),
+  );
+});
+
+function journalRecords(stateDir: string, repoDir: string): JournalRecord[] {
+  const path = join(stateDir, "journal", `${createKey(repoDir)}.jsonl`);
+  return readFileSync(path, "utf8")
+    .split("\n")
+    .filter((line) => line.length > 0)
+    .map((line) => JSON.parse(line) as JournalRecord);
+}
+
+function createKey(repoDir: string): string {
+  return createHash("sha256").update(repoDir).digest("hex").slice(0, 24);
+}
+
+interface TestClient {
+  socket: WebSocket;
+  send(command: unknown): void;
+  waitFor(
+    predicate: (message: Record<string, unknown>) => boolean,
+  ): Promise<Record<string, unknown>>;
+}
+
+function connect(url: string): Promise<TestClient> {
+  const { promise, resolve: settle, reject } = Promise.withResolvers<TestClient>();
+  const socket = new WebSocket(`ws://${url}/ws`);
+  const buffered: Record<string, unknown>[] = [];
+  const waiters: {
+    predicate: (message: Record<string, unknown>) => boolean;
+    resolve: (message: Record<string, unknown>) => void;
+  }[] = [];
+
+  socket.onopen = () => {
+    settle({
+      socket,
+      send(command) {
+        socket.send(JSON.stringify(command));
+      },
+      waitFor(predicate) {
+        const index = buffered.findIndex(predicate);
+        if (index >= 0) {
+          const message = buffered.splice(index, 1)[0];
+          if (message !== undefined) return Promise.resolve(message);
+        }
+        const { promise: waiting, resolve: next } =
+          Promise.withResolvers<Record<string, unknown>>();
+        waiters.push({ predicate, resolve: next });
+        return waiting;
+      },
+    });
+  };
+  socket.onmessage = (event) => {
+    let message: Record<string, unknown>;
+    try {
+      message = JSON.parse(String(event.data)) as Record<string, unknown>;
+    } catch {
+      return;
+    }
+    const index = waiters.findIndex((waiter) => waiter.predicate(message));
+    if (index >= 0) {
+      const waiter = waiters.splice(index, 1)[0];
+      if (waiter !== undefined) waiter.resolve(message);
+    } else {
+      buffered.push(message);
+    }
+  };
+  socket.onerror = () => reject(new Error("websocket failed"));
+  return promise;
+}
+
+const PENDING = pendingRecord("0001-first.md", "First decision", "First.", "Choose the first.");
+
+async function startServer(options: {
+  stateDir: string;
+  journal?: Journal;
+  agentCommand: string;
+}): Promise<{ server: Server<unknown>; url: string; repoDir: string }> {
+  const root = options.stateDir;
+  const adrDir = join(root, "adrs");
+  const staticDir = join(root, "static");
+  await mkdir(adrDir, { recursive: true });
+  await mkdir(staticDir, { recursive: true });
+  await writeFile(join(adrDir, "0001-first.md"), PENDING);
+  const server = createApp({
+    adrDir,
+    staticDir,
+    hostname: "127.0.0.1",
+    port: 0,
+    repoDir: root,
+    stateDir: join(root, "state"),
+    agentCommand: options.agentCommand,
+    ...(options.journal === undefined ? {} : { journal: options.journal }),
+  });
+  return { server, url: `127.0.0.1:${server.port}`, repoDir: root };
+}
+
+describe("journal", () => {
+  test(
+    "records a run's activity, question, start, and stop",
+    async () => {
+      const stateDir = await temporaryDirectory();
+      const logPath = join(stateDir, "agent.log");
+      const { server, url, repoDir } = await startServer({
+        stateDir,
+        agentCommand: `bun ${STUB_AGENT} --log=${logPath} --questions=1`,
+      });
+      const client = await connect(url);
+      try {
+        client.send({ type: "startRun" });
+        await client.waitFor((message) => message.type === "runStarted");
+        const question = await client.waitFor((message) => message.type === "question");
+        client.send({ type: "answer", id: question.id, value: "Alpha" });
+        await client.waitFor((message) => message.type === "activity");
+        await client.waitFor((message) => message.type === "runStopped");
+      } finally {
+        client.socket.close();
+        server.stop(true);
+      }
+
+      const records = journalRecords(join(stateDir, "state"), repoDir);
+      const kinds = new Set(records.map((record) => record.kind));
+      expect(kinds.has("activity")).toBe(true);
+      expect(kinds.has("question")).toBe(true);
+      expect(kinds.has("run-start")).toBe(true);
+      expect(kinds.has("run-stop")).toBe(true);
+      expect(records.find((record) => record.kind === "question")?.scope).toBe(
+        "run:0001-first.md",
+      );
+      for (const record of records) {
+        expect(record.repo).toBe(repoDir);
+        expect(record.pid).toBe(process.pid);
+        expect(typeof record.port).toBe("number");
+        expect(Number.isFinite(Date.parse(record.ts))).toBe(true);
+      }
+    },
+    20000,
+  );
+
+  test(
+    "reports a write failure and still finishes the run",
+    async () => {
+      const stateDir = await temporaryDirectory();
+      const failures: string[] = [];
+      const journal = openJournal({
+        stateDir: join(stateDir, "state"),
+        repoDir: stateDir,
+        port: 0,
+        append() {
+          throw new Error("disk full");
+        },
+        reportError(message) {
+          failures.push(message);
+        },
+      });
+      const logPath = join(stateDir, "agent.log");
+      const { server, url } = await startServer({
+        stateDir,
+        journal,
+        agentCommand: `bun ${STUB_AGENT} --log=${logPath} --questions=0`,
+      });
+      const client = await connect(url);
+      try {
+        client.send({ type: "startRun" });
+        await client.waitFor((message) => message.type === "runStarted");
+        expect(await client.waitFor((message) => message.type === "runStopped")).toMatchObject({
+          resolved: true,
+        });
+      } finally {
+        client.socket.close();
+        server.stop(true);
+      }
+      expect(failures.length).toBe(1);
+      expect(failures[0]).toContain("journal: could not write");
+    },
+    20000,
+  );
+
+  test("rotates the live file instead of truncating it", async () => {
+    const stateDir = await temporaryDirectory();
+    const journal = openJournal({
+      stateDir,
+      repoDir: "/fixture/project",
+      port: 8080,
+      rotateBytes: 200,
+    });
+    for (let index = 0; index < 6; index += 1) {
+      journal.write("activity", `line ${index} ${"x".repeat(40)}`);
+    }
+    const files = readdirSync(join(stateDir, "journal")).sort();
+    expect(files.length).toBeGreaterThan(1);
+    expect(files.some((name) => name.endsWith(".jsonl"))).toBe(true);
+    const rotated = files.filter((name) => !name.endsWith(".jsonl"));
+    expect(rotated.length).toBeGreaterThan(0);
+    const content = rotated
+      .map((name) => readFileSync(join(stateDir, "journal", name), "utf8"))
+      .join("");
+    expect(content).toContain("line 0");
+    const live = files.find((name) => /^[0-9a-f]{24}\.jsonl$/.test(name)) as string;
+    expect(readFileSync(join(stateDir, "journal", live), "utf8")).toContain("line 5");
+  });
+
+  test("prunes journals rotated by the live writer past seven days", async () => {
+    const stateDir = await temporaryDirectory();
+    const journal = openJournal({
+      stateDir,
+      repoDir: "/fixture/project",
+      port: 8080,
+      rotateBytes: 200,
+    });
+    for (let index = 0; index < 3; index += 1) {
+      journal.write("activity", `line ${index} ${"x".repeat(80)}`);
+    }
+    const directory = join(stateDir, "journal");
+    const rotated = readdirSync(directory)
+      .filter((name) => name !== basename(journal.path))
+      .sort();
+    const [oldName, freshName] = rotated;
+    if (oldName === undefined || freshName === undefined) {
+      throw new Error("rotation did not produce two journals");
+    }
+    const now = new Date("2026-10-24T12:00:00.000Z");
+    const eightDaysAgo = new Date(now.getTime() - 8 * 24 * 60 * 60 * 1000);
+    const oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000);
+    const old = join(directory, oldName);
+    const fresh = join(directory, freshName);
+    utimesSync(old, eightDaysAgo, eightDaysAgo);
+    utimesSync(fresh, oneDayAgo, oneDayAgo);
+
+    const removed = pruneJournal(stateDir, {
+      now,
+      windowStart: new Date(now.getTime() - 10 * 60 * 1000),
+    });
+
+    expect(removed).toEqual([oldName]);
+    expect(existsSync(old)).toBe(false);
+    expect(existsSync(fresh)).toBe(true);
+  });
+
+  test("keeps an old live journal in retention and restores it after deletion", async () => {
+    const stateDir = await temporaryDirectory();
+    const failures: string[] = [];
+    const journal = openJournal({
+      stateDir,
+      repoDir: "/fixture/project",
+      port: 8080,
+      rotateBytes: 200,
+      reportError: (message) => failures.push(message),
+    });
+    journal.write("activity", `line 0 ${"z".repeat(120)}`);
+    const now = new Date("2026-10-24T12:00:00.000Z");
+    const eightDaysAgo = new Date(now.getTime() - 8 * 24 * 60 * 60 * 1000);
+    utimesSync(journal.path, eightDaysAgo, eightDaysAgo);
+
+    const removed = pruneJournal(stateDir, {
+      now,
+      windowStart: new Date(now.getTime() - 10 * 60 * 1000),
+    });
+
+    expect(removed).toEqual([]);
+    expect(existsSync(journal.path)).toBe(true);
+
+    // The writer keeps its stale size; the next write crosses rotateBytes.
+    unlinkSync(journal.path);
+    journal.write("activity", `line 1 ${"z".repeat(120)}`);
+
+    expect(failures).toEqual([]);
+    expect(readFileSync(journal.path, "utf8")).toContain("line 1");
+  });
+
+  test("counts rotated journals toward the size bound outside the live window", async () => {
+    const stateDir = await temporaryDirectory();
+    const journal = openJournal({
+      stateDir,
+      repoDir: "/fixture/project",
+      port: 8080,
+      rotateBytes: 200,
+    });
+    for (let index = 0; index < 4; index += 1) {
+      journal.write("activity", `line ${index} ${"y".repeat(80)}`);
+    }
+    const directory = join(stateDir, "journal");
+    const names = readdirSync(directory);
+    const rotated = names.filter((name) => name !== basename(journal.path)).sort();
+    const [coldName, ...inside] = rotated;
+    if (coldName === undefined || inside.length === 0) {
+      throw new Error("rotation did not produce enough journals");
+    }
+    const now = new Date("2026-10-24T12:00:00.000Z");
+    const windowStart = new Date(now.getTime() - 10 * 60 * 1000);
+    const cold = join(directory, coldName);
+    const coldTime = new Date(now.getTime() - 60 * 60 * 1000);
+    utimesSync(cold, coldTime, coldTime);
+    for (const name of [...inside, basename(journal.path)]) {
+      utimesSync(join(directory, name), now, now);
+    }
+    const total = names.reduce((sum, name) => sum + statSync(join(directory, name)).size, 0);
+    const coldSize = statSync(cold).size;
+    expect(coldSize).toBeGreaterThan(1);
+
+    const removed = pruneJournal(stateDir, {
+      now,
+      windowStart,
+      maxBytes: total - coldSize + 1,
+    });
+
+    expect(removed).toEqual([coldName]);
+    expect(existsSync(cold)).toBe(false);
+    for (const name of inside) expect(existsSync(join(directory, name))).toBe(true);
+    expect(existsSync(journal.path)).toBe(true);
+    expect(JOURNAL_MAX_BYTES).toBe(1024 * 1024 * 1024);
+  });
+});

package/server/journal.ts

@@ -0,0 +1,329 @@
+import { createHash } from "node:crypto";
+import {
+  appendFileSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+} from "node:fs";
+import { basename, dirname, join } from "node:path";
+
+/**
+ * The dashboard journal is the machine-readable record the watchdog reads.
+ * One file per project per harness, appended as JSON lines so a truncated tail
+ * costs at most one record. Writes never interrupt a run: a full disk or a
+ * read-only state directory only produces a line on the server's stderr.
+ */
+
+export const JOURNAL_KINDS = [
+  "activity",
+  "question",
+  "run-start",
+  "run-stop",
+  "lane-start",
+  "lane-stop",
+  "lifecycle",
+  "acp-stderr",
+  "server-error",
+  "update",
+  "update-error",
+] as const;
+
+export type JournalKind = (typeof JOURNAL_KINDS)[number];
+
+/** A live journal rotates at this size instead of growing without bound. */
+export const JOURNAL_ROTATE_BYTES = 8 * 1024 * 1024;
+/** Rotated journals older than this leave at cycle start. */
+export const JOURNAL_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+/** Total journal bytes the watchdog keeps, unless the live window is larger. */
+export const JOURNAL_MAX_BYTES = 1024 * 1024 * 1024;
+
+export interface JournalRecord {
+  /** UTC ISO 8601 timestamp of the record. */
+  ts: string;
+  /** Absolute project path the server serves. */
+  repo: string;
+  port: number;
+  pid: number;
+  kind: JournalKind;
+  text: string;
+  /** Logical run, lane, or peer the record belongs to, when one is known. */
+  scope?: string;
+}
+
+export interface JournalOptions {
+  /** Harness state directory; the journal lives in its `journal` child. */
+  stateDir: string;
+  repoDir: string;
+  port: number;
+  pid?: number;
+  /** Injectable clock for tests. */
+  now?: () => Date;
+  /** Size at which the live file is rotated aside. */
+  rotateBytes?: number;
+  /** Injectable append, used to simulate a write failure. */
+  append?: (path: string, data: string) => void;
+  /** Injectable failure sink; defaults to the server's stderr. */
+  reportError?: (message: string) => void;
+}
+
+export interface Journal {
+  path: string;
+  write(kind: JournalKind, text: string, scope?: string): void;
+}
+
+/** The 24-hex project key that names this project's journal. */
+export function journalKey(repoDir: string): string {
+  return createHash("sha256").update(repoDir).digest("hex").slice(0, 24);
+}
+
+export function journalFile(stateDir: string, repoDir: string): string {
+  return join(stateDir, "journal", `${journalKey(repoDir)}.jsonl`);
+}
+
+/**
+ * A journal filename: `<24-hex project key>.jsonl`, rotated aside as
+ * `<key>.jsonl.<rotation instant>-<counter>`. Rotation and pruning share this
+ * one definition so a name the writer produces is a name the pruner accepts.
+ */
+const JOURNAL_NAME = /^([0-9a-f]{24})\.jsonl(?:\.(\d+)-(\d+))?$/;
+
+interface JournalName {
+  /** The project key that names the journal. */
+  key: string;
+  /** Rotation instant and counter; absent on the live journal. */
+  rotation?: { ms: number; counter: number };
+}
+
+/** Parses a journal filename; `null` for any other file in the directory. */
+function parseJournalName(name: string): JournalName | null {
+  const match = JOURNAL_NAME.exec(name);
+  if (match === null) return null;
+  const key = match[1];
+  const ms = match[2];
+  const counter = match[3];
+  if (key === undefined) return null;
+  if (ms === undefined || counter === undefined) return { key };
+  return { key, rotation: { ms: Number(ms), counter: Number(counter) } };
+}
+
+/**
+ * Every journal file of one project: rotated journals oldest first, then the
+ * live file. A project that never rotated yields only its live path, and a
+ * missing journal directory yields that path so the read reports nothing.
+ *
+ * `since` is the oldest instant a caller needs. A rotated file last written
+ * before it cannot hold a later record, so it is skipped instead of parsed:
+ * a quiet dashboard keeps up to `JOURNAL_MAX_BYTES` of old rotations.
+ */
+export function journalFiles(livePath: string, since?: Date): string[] {
+  const directory = dirname(livePath);
+  const live = parseJournalName(basename(livePath));
+  let names: string[];
+  try {
+    names = readdirSync(directory);
+  } catch {
+    return [livePath];
+  }
+  if (live === null) return [livePath];
+  const cutoff = since?.getTime();
+  const rotated: { path: string; ms: number; counter: number; name: string }[] = [];
+  for (const name of names) {
+    const parsed = parseJournalName(name);
+    if (parsed === null || parsed.rotation === undefined) continue;
+    if (parsed.key !== live.key) continue;
+    const { ms, counter } = parsed.rotation;
+    const path = join(directory, name);
+    if (cutoff !== undefined) {
+      try {
+        if (statSync(path).mtimeMs < cutoff) continue;
+      } catch {
+        // A journal disappearing mid-scan is already gone.
+        continue;
+      }
+    }
+    rotated.push({ path, ms, counter, name });
+  }
+  rotated.sort(
+    (left, right) =>
+      left.ms - right.ms || left.counter - right.counter || left.name.localeCompare(right.name),
+  );
+  return [...rotated.map((file) => file.path), livePath];
+}
+
+/** One JSONL line per event, truncated so a single huge stream cannot flood it. */
+export function journalLine(record: JournalRecord): string {
+  const text = record.text.length > 2000 ? `${record.text.slice(0, 2000)}…` : record.text;
+  return `${JSON.stringify({ ...record, text })}\n`;
+}
+
+export function openJournal(options: JournalOptions): Journal {
+  const path = journalFile(options.stateDir, options.repoDir);
+  const append = options.append ?? ((target: string, data: string) => appendFileSync(target, data));
+  const reportError =
+    options.reportError ??
+    ((message: string) => {
+      process.stderr.write(`${message}\n`);
+    });
+  const rotateBytes = options.rotateBytes ?? JOURNAL_ROTATE_BYTES;
+  const pid = options.pid ?? process.pid;
+  let opened = false;
+  let size = 0;
+  let rotations = 0;
+  let reported = false;
+
+  // A dashboard that has not emitted an event yet is still instrumented.
+  try {
+    mkdirSync(dirname(path), { recursive: true });
+  } catch {
+    // The first write reports the failure through the configured sink.
+  }
+
+  function open() {
+    if (opened) return;
+    mkdirSync(dirname(path), { recursive: true });
+    try {
+      size = statSync(path).size;
+    } catch {
+      size = 0;
+    }
+    opened = true;
+  }
+
+  /** Moves the live file aside so no record is ever truncated away. */
+  function rotate() {
+    rotations += 1;
+    try {
+      // A counter keeps two rotations in the same millisecond from colliding.
+      renameSync(path, `${path}.${Date.now()}-${rotations}`);
+    } catch (error) {
+      // The live file can disappear under a running writer: an external delete
+      // or a retention sweep of an older build. Rotation then starts a fresh
+      // live file instead of throwing the record away.
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+    size = 0;
+  }
+
+  return {
+    path,
+    write(kind: JournalKind, text: string, scope?: string) {
+      const record: JournalRecord = {
+        ts: (options.now?.() ?? new Date()).toISOString(),
+        repo: options.repoDir,
+        port: options.port,
+        pid,
+        kind,
+        text,
+        ...(scope === undefined ? {} : { scope }),
+      };
+      const line = journalLine(record);
+      const bytes = Buffer.byteLength(line);
+      try {
+        open();
+        if (size > 0 && size + bytes > rotateBytes) rotate();
+        append(path, line);
+        size += bytes;
+      } catch (error) {
+        if (reported) return;
+        reported = true;
+        const detail = error instanceof Error ? error.message : String(error);
+        reportError(`journal: could not write ${path}: ${detail}`);
+      }
+    },
+  };
+}
+
+export interface PruneOptions {
+  /** Cycle start; the live evidence window ends here. */
+  now: Date;
+  /** Oldest timestamp the watchdog still needs. */
+  windowStart: Date;
+  retentionMs?: number;
+  maxBytes?: number;
+}
+
+interface JournalFile {
+  path: string;
+  name: string;
+  size: number;
+  mtimeMs: number;
+  /** The live file the writer appends to; never a deletion candidate. */
+  live: boolean;
+}
+
+/**
+ * Drops rotated journals past the retention limit and enforces the total bound
+ * by deleting the oldest complete files first. A file that still holds records
+ * inside the live window is never deleted for the bound.
+ */
+export function pruneJournal(stateDir: string, options: PruneOptions): string[] {
+  const directory = join(stateDir, "journal");
+  const retentionMs = options.retentionMs ?? JOURNAL_RETENTION_MS;
+  const maxBytes = options.maxBytes ?? JOURNAL_MAX_BYTES;
+  let names: string[];
+  try {
+    names = readdirSync(directory);
+  } catch {
+    return [];
+  }
+
+  const files: JournalFile[] = [];
+  for (const name of names) {
+    const parsed = parseJournalName(name);
+    if (parsed === null) continue;
+    const path = join(directory, name);
+    try {
+      const info = statSync(path);
+      if (!info.isFile()) continue;
+      files.push({
+        path,
+        name,
+        size: info.size,
+        mtimeMs: info.mtimeMs,
+        live: parsed.rotation === undefined,
+      });
+    } catch {
+      // A journal disappearing mid-scan is already gone.
+    }
+  }
+
+  const removed: string[] = [];
+  const cutoff = options.now.getTime() - retentionMs;
+  const windowStart = options.windowStart.getTime();
+  const remaining = new Set(files);
+
+  for (const file of [...files].sort((left, right) => left.mtimeMs - right.mtimeMs)) {
+    if (!remaining.has(file)) continue;
+    // Only rotated evidence expires: the live file belongs to the writer.
+    if (file.live) continue;
+    if (file.mtimeMs >= cutoff) continue;
+    try {
+      unlinkSync(file.path);
+      remaining.delete(file);
+      removed.push(file.name);
+    } catch {
+      // A concurrent reader may already have removed it.
+    }
+  }
+
+  let total = [...remaining].reduce((sum, file) => sum + file.size, 0);
+  for (const file of [...remaining].sort((left, right) => left.mtimeMs - right.mtimeMs)) {
+    if (total <= maxBytes) break;
+    // The live file still counts toward the bound but is never a candidate.
+    if (file.live) continue;
+    // A file still inside the window is evidence the next cycle needs.
+    if (file.mtimeMs >= windowStart) continue;
+    try {
+      unlinkSync(file.path);
+      remaining.delete(file);
+      total -= file.size;
+      removed.push(file.name);
+    } catch {
+      // Already gone.
+    }
+  }
+
+  return removed;
+}