@cueloop/daemon 0.1.0-alpha.8 → 0.1.0-alpha.81

package/src/store.ts

@@ -1,68 +1,298 @@
 /**
- * Session persistence (#14): one JSON document per session, every write
- * through a temp file + atomic rename so a crash mid-write can never leave
- * a corrupt record. Recovery is a read-only scan; records that fail to
- * parse are skipped and reported, never deleted.
+ * Thread persistence: one append-only JSONL log per thread, under a per-project bucket keyed by the
+ * repo's durable root commit (a shared bucket for repo-less threads). Each mutation appends a
+ * snapshot line and the last valid line wins, so a crash's torn tail is simply skipped on read; the
+ * log is compacted to one line on the first write, once it grows long, and when the thread resolves.
+ * Boot migrates the old whole-record JSON files into this layout and parks the originals. Records
+ * that fail to parse are skipped and reported, never deleted; records from before histories existed
+ * are given one on read.
+ *
+ * `ThreadRepository` is the contract every adapter satisfies; the conformance
+ * suite in ./testing/store-conformance.ts pins it for the file store and the
+ * in-memory store alike.
  */
 
-import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import type { ReviewSession } from "@cueloop/schema";
-import { validateSessionRecord } from "./validate";
-import { sessionsDir } from "./paths";
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+import { historyFromLinear, withShareLinks, type Thread } from "@cueloop/schema";
+import { migratedSessionsDir, sessionsDir, threadBucket, threadsDir } from "./paths";
+import { validateThreadRecord } from "./validate";
+
+/** A thread's log is rewritten to a single snapshot line once it grows past this many, so an
+ *  active thread's file never bloats unbounded while writes stay cheap appends in between. */
+const COMPACT_THRESHOLD = 40;
 
 export interface RecoveryReport {
   recovered: string[];
   skipped: { file: string; error: string }[];
 }
 
-export class SessionStore {
-  private sessions = new Map<string, ReviewSession>();
-  private readonly dir: string;
+/** What the daemon needs from session storage. */
+export interface ThreadRepository {
+  /** Load what is stored; called once on boot. */
+  recover(): RecoveryReport;
+  get(id: string): Thread | undefined;
+  /** Every session, oldest first. */
+  list(): Thread[];
+  upsert(session: Thread): void;
+  /** True when a session was removed. */
+  delete(id: string): boolean;
+}
+
+/**
+ * A record as it is read: a history is derived for records written without
+ * one. A record with no revision has no head to derive from and keeps
+ * reading without a history - migration never loses a record.
+ */
+export function withHistory(session: Thread): Thread {
+  // migrate a legacy single share into shares[] on the same read that fills history
+  const migrated = withShareLinks(session);
+
+  if (migrated.history || migrated.revisions.length === 0) return migrated;
+
+  return { ...migrated, history: historyFromLinear(migrated) };
+}
+
+/** Records in the order `list()` promises: oldest first. */
+function oldestFirst(sessions: Iterable<Thread>): Thread[] {
+  return [...sessions].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+}
 
-  constructor(home: string) {
-    this.dir = sessionsDir(home);
-    mkdirSync(this.dir, { recursive: true });
+export class ThreadStore implements ThreadRepository {
+  private sessions = new Map<string, Thread>();
+  /** id -> its JSONL path (the bucket depends on the record's root commit, so it is tracked). */
+  private files = new Map<string, string>();
+  /** id -> lines in its log, so upsert knows when to compact instead of append. */
+  private lineCounts = new Map<string, number>();
+
+  constructor(private readonly home: string) {
+    mkdirSync(threadsDir(home), { recursive: true });
+    // the legacy dir is where a pre-upgrade daemon wrote; ensure it exists so its scan is uniform
+    mkdirSync(sessionsDir(home), { recursive: true });
   }
 
-  /** Read-only scan of the state directory; called once on boot. */
   recover(): RecoveryReport {
     const report: RecoveryReport = { recovered: [], skipped: [] };
-    for (const file of readdirSync(this.dir)) {
+
+    this.migrateLegacy(report);
+    const root = threadsDir(this.home);
+
+    for (const bucket of readdirSync(root)) {
+      const bucketPath = join(root, bucket);
+
+      if (!statSync(bucketPath).isDirectory()) continue;
+      for (const file of readdirSync(bucketPath)) {
+        if (!file.endsWith(".jsonl")) continue;
+        const filePath = join(bucketPath, file);
+        try {
+          const { session, lines, torn } = readThread(filePath);
+
+          this.sessions.set(session.id, session);
+          this.files.set(session.id, filePath);
+          this.lineCounts.set(session.id, lines);
+          report.recovered.push(session.id);
+          // a torn trailing fragment would fuse with the next append and lose that update, so rewrite
+          // the log to a single clean snapshot line now that the last valid record is in hand
+          if (torn) this.writeWhole(session);
+        } catch (error) {
+          report.skipped.push({
+            file,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    }
+
+    return report;
+  }
+
+  /** One-time upgrade: write each old whole-record JSON as a thread log and park the original, so
+   *  the threads scan below recovers it. Invalid records are reported and left in place, never lost. */
+  private migrateLegacy(report: RecoveryReport): void {
+    const legacyDir = sessionsDir(this.home);
+
+    if (!existsSync(legacyDir)) return;
+    const parked = migratedSessionsDir(this.home);
+
+    for (const file of readdirSync(legacyDir)) {
       if (!file.endsWith(".json")) continue;
+      const legacyPath = join(legacyDir, file);
+      let session: Thread;
+
       try {
-        const raw = readFileSync(join(this.dir, file), "utf8");
-        const parsed = validateSessionRecord(JSON.parse(raw));
+        const parsed = validateThreadRecord(JSON.parse(readFileSync(legacyPath, "utf8")));
+
         if (!parsed.ok) throw new Error(`invalid record - ${parsed.error}`);
-        const session = parsed.value as ReviewSession;
-        this.sessions.set(session.id, session);
-        report.recovered.push(session.id);
-      } catch (err) {
-        report.skipped.push({ file, error: err instanceof Error ? err.message : String(err) });
+        session = withHistory(parsed.value);
+      } catch (error) {
+        report.skipped.push({
+          file,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        continue;
       }
+      const targetPath = join(
+        threadBucket(session.workspace.rootCommit, this.home),
+        `${session.id}.jsonl`,
+      );
+
+      // after a rollback a newer JSONL can already hold this thread; never overwrite it with the
+      // older legacy snapshot - the existing log stays authoritative and the threads scan recovers it
+      if (!existsSync(targetPath)) this.writeWhole(session);
+      // forget the in-memory tracking migration seeded: the threads scan is the one recovery path
+      this.files.delete(session.id);
+      this.lineCounts.delete(session.id);
+      mkdirSync(parked, { recursive: true });
+      renameSync(legacyPath, join(parked, file));
     }
-    return report;
   }
 
-  get(id: string): ReviewSession | undefined {
+  /** Rewrite the thread to a single snapshot line, atomically - the first write and every compaction. */
+  private writeWhole(session: Thread): void {
+    const filePath =
+      this.files.get(session.id) ??
+      join(threadBucket(session.workspace.rootCommit, this.home), `${session.id}.jsonl`);
+
+    mkdirSync(dirname(filePath), { recursive: true });
+    const tempPath = filePath + ".tmp";
+
+    writeFileSync(tempPath, JSON.stringify(session) + "\n");
+    renameSync(tempPath, filePath);
+    this.files.set(session.id, filePath);
+    this.lineCounts.set(session.id, 1);
+  }
+
+  get(id: string): Thread | undefined {
     return this.sessions.get(id);
   }
 
-  list(): ReviewSession[] {
-    return [...this.sessions.values()].sort((a, b) => a.createdAt.localeCompare(b.createdAt));
+  list(): Thread[] {
+    return oldestFirst(this.sessions.values());
   }
 
-  upsert(session: ReviewSession): void {
+  upsert(session: Thread): void {
     this.sessions.set(session.id, session);
-    const path = join(this.dir, `${session.id}.json`);
-    const tmp = path + ".tmp";
-    writeFileSync(tmp, JSON.stringify(session, null, 2));
-    renameSync(tmp, path);
+    const filePath = this.files.get(session.id);
+    const lines = this.lineCounts.get(session.id) ?? 0;
+
+    // the first write, an over-long log, and a resolved (now immutable) thread all compact to one
+    // line; every other mutation is a cheap append that a torn tail on crash simply drops on read
+    if (filePath === undefined || lines >= COMPACT_THRESHOLD || session.status === "resolved") {
+      this.writeWhole(session);
+
+      return;
+    }
+    appendFileSync(filePath, JSON.stringify(session) + "\n");
+    this.lineCounts.set(session.id, lines + 1);
   }
 
   delete(id: string): boolean {
     if (!this.sessions.delete(id)) return false;
-    rmSync(join(this.dir, `${id}.json`), { force: true });
+    const filePath = this.files.get(id);
+
+    if (filePath) rmSync(filePath, { force: true });
+    this.files.delete(id);
+    this.lineCounts.delete(id);
+
     return true;
   }
 }
+
+/**
+ * Read a thread log: the last valid snapshot line wins, so a crash's torn trailing line is skipped.
+ * `torn` is true when that winning line was not the last physical line - the caller rewrites the log
+ * so the next append starts on a clean boundary instead of fusing onto the torn fragment.
+ */
+function readThread(filePath: string) {
+  const allLines = readFileSync(filePath, "utf8")
+    .split("\n")
+    .filter((line) => line.trim().length > 0);
+
+  for (let index = allLines.length - 1; index >= 0; index--) {
+    let record: unknown;
+
+    try {
+      record = JSON.parse(allLines[index]!);
+    } catch {
+      continue;
+    }
+    const parsed = validateThreadRecord(record);
+
+    if (parsed.ok) {
+      return {
+        session: withHistory(parsed.value),
+        lines: allLines.length,
+        torn: index !== allLines.length - 1,
+      };
+    }
+  }
+  throw new Error("no valid record line");
+}
+
+/**
+ * The in-memory adapter: the same contract with nothing on disk. `seed` stands
+ * in for what a file store finds on recovery, so validation and migration are
+ * exercised the same way.
+ */
+export class MemoryThreadStore implements ThreadRepository {
+  private sessions = new Map<string, Thread>();
+
+  constructor(private readonly seed: unknown[] = []) {}
+
+  recover(): RecoveryReport {
+    const report: RecoveryReport = { recovered: [], skipped: [] };
+
+    this.seed.forEach((record, index) => {
+      const parsed = validateThreadRecord(record);
+
+      if (!parsed.ok) {
+        report.skipped.push({ file: `seed[${index}]`, error: `invalid record - ${parsed.error}` });
+
+        return;
+      }
+      const session = withHistory(parsed.value);
+
+      this.sessions.set(session.id, session);
+      report.recovered.push(session.id);
+    });
+
+    return report;
+  }
+
+  get(id: string): Thread | undefined {
+    return this.sessions.get(id);
+  }
+
+  list(): Thread[] {
+    return oldestFirst(this.sessions.values());
+  }
+
+  upsert(session: Thread): void {
+    this.sessions.set(session.id, session);
+  }
+
+  delete(id: string): boolean {
+    return this.sessions.delete(id);
+  }
+}
+
+/** @deprecated use ThreadRepository */
+export type SessionRepository = ThreadRepository;
+/** @deprecated use ThreadStore */
+export const SessionStore = ThreadStore;
+/** @deprecated use ThreadStore */
+export type SessionStore = ThreadStore;
+/** @deprecated use MemoryThreadStore */
+export const MemorySessionStore = MemoryThreadStore;
+/** @deprecated use MemoryThreadStore */
+export type MemorySessionStore = MemoryThreadStore;

package/src/testing/store-conformance.ts

@@ -0,0 +1,194 @@
+/**
+ * The session repository contract as executable cases. Every adapter runs the
+ * same suite: what a store returns, in which order, what survives a restart,
+ * what a bad record does, and how a record without a history reads.
+ *
+ * `restart` gives back a fresh adapter over the same persisted state (or the
+ * same seed) so recovery is tested through the contract, never past it.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { SCHEMA_VERSION, type Thread } from "@cueloop/schema";
+import type { ThreadRepository } from "../store";
+
+export interface StoreHarness {
+  /** A fresh, empty adapter that will recover `records` on `recover()`. */
+  open: (records: unknown[]) => ThreadRepository;
+  /** A new adapter over whatever `store` persisted, as after a daemon restart. */
+  restart: (store: ThreadRepository) => ThreadRepository;
+}
+
+export function createTestThreadRecord(
+  id: string,
+  createdAt: string,
+  overrides: Partial<Thread> = {},
+): Thread {
+  return {
+    schemaVersion: SCHEMA_VERSION,
+    id,
+    workspace: { repoRoot: "/repo", branch: "main" },
+    artifact: { type: "plan", content: `# ${id}`, meta: {} },
+    revisions: [{ revision: 1, content: `# ${id}`, submittedAt: createdAt }],
+    annotations: [],
+    message: null,
+    status: "pending",
+    createdAt,
+    ...overrides,
+  };
+}
+
+export function runThreadStoreConformance(name: string, harness: StoreHarness): void {
+  describe(`${name} - session repository contract`, () => {
+    test("a stored session reads back whole and lists oldest first", () => {
+      // Arrange
+      const store = harness.open([]);
+
+      store.recover();
+      const newer = createTestThreadRecord("ses_b", "2026-09-02T00:00:00.000Z");
+      const older = createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z");
+
+      // Act
+      store.upsert(newer);
+      store.upsert(older);
+
+      // Assert
+      expect(store.get("ses_a")).toEqual(older);
+      expect(store.list().map((session) => session.id)).toEqual(["ses_a", "ses_b"]);
+      expect(store.get("ses_zzz")).toBeUndefined();
+    });
+
+    test("upsert replaces by id and delete reports whether anything was removed", () => {
+      // Arrange
+      const store = harness.open([]);
+
+      store.recover();
+      store.upsert(createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z"));
+
+      // Act
+      store.upsert(
+        createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", { status: "resolved" }),
+      );
+
+      // Assert
+      expect(store.list()).toHaveLength(1);
+      expect(store.get("ses_a")?.status).toBe("resolved");
+      expect(store.delete("ses_a")).toBe(true);
+      expect(store.delete("ses_a")).toBe(false);
+      expect(store.list()).toEqual([]);
+    });
+
+    test("what was upserted survives a restart, deletions included", () => {
+      // Arrange
+      const store = harness.open([]);
+
+      store.recover();
+      store.upsert(createTestThreadRecord("ses_keep", "2026-09-01T00:00:00.000Z"));
+      store.upsert(createTestThreadRecord("ses_gone", "2026-09-02T00:00:00.000Z"));
+      store.delete("ses_gone");
+
+      // Act
+      const reopened = harness.restart(store);
+      const report = reopened.recover();
+
+      // Assert
+      expect(report.recovered).toEqual(["ses_keep"]);
+      expect(reopened.get("ses_keep")?.artifact.content).toBe("# ses_keep");
+      expect(reopened.get("ses_gone")).toBeUndefined();
+    });
+
+    test("a record without a history reads as a one-branch tree whose head is the artifact", () => {
+      // Arrange: written before histories existed, with two revisions and a comment
+      const legacy = createTestThreadRecord("ses_old", "2026-09-01T00:00:00.000Z", {
+        artifact: { type: "plan", content: "v2", meta: {} },
+        revisions: [
+          { revision: 1, content: "v1", submittedAt: "2026-09-01T00:00:00.000Z" },
+          { revision: 2, content: "v2", submittedAt: "2026-09-01T02:00:00.000Z" },
+        ],
+        annotations: [
+          {
+            id: "a1",
+            kind: "comment",
+            anchor: { quote: "v1", prefix: "", suffix: "" },
+            body: "why",
+            createdAt: "2026-09-01T01:00:00.000Z",
+          },
+        ],
+      });
+      const store = harness.open([legacy]);
+
+      // Act
+      const report = store.recover();
+      const history = store.get("ses_old")?.history;
+
+      // Assert
+      expect(report.recovered).toEqual(["ses_old"]);
+      expect(history?.branch).toBe("main");
+      expect(history?.entries.map((entry) => entry.type)).toEqual([
+        "revision",
+        "comment",
+        "revision",
+      ]);
+      expect(history?.entries.at(-1)).toMatchObject({ type: "revision", content: "v2" });
+      expect(history?.tips.main).toBe(history?.entries.at(-1)?.id);
+    });
+
+    test("a record with no revision still recovers, without a history", () => {
+      // Arrange
+      const store = harness.open([
+        createTestThreadRecord("ses_empty", "2026-09-01T00:00:00.000Z", { revisions: [] }),
+      ]);
+
+      // Act
+      const report = store.recover();
+
+      // Assert
+      expect(report.recovered).toEqual(["ses_empty"]);
+      expect(store.get("ses_empty")?.history).toBeUndefined();
+    });
+
+    test("a record that already carries a history keeps it untouched", () => {
+      // Arrange
+      const withTree = createTestThreadRecord("ses_tree", "2026-09-01T00:00:00.000Z", {
+        history: {
+          entries: [
+            {
+              id: "root",
+              parentId: null,
+              type: "revision",
+              by: "agent",
+              content: "# ses_tree",
+              createdAt: "2026-09-01T00:00:00.000Z",
+            },
+          ],
+          tips: { main: "root", ramble: "root" },
+          branch: "ramble",
+          labels: { root: "start" },
+        },
+      });
+      const store = harness.open([withTree]);
+
+      // Act
+      store.recover();
+
+      // Assert
+      expect(store.get("ses_tree")?.history).toEqual(withTree.history);
+    });
+
+    test("a record that fails validation is skipped and reported, never dropped from the report", () => {
+      // Arrange
+      const store = harness.open([
+        createTestThreadRecord("ses_good", "2026-09-01T00:00:00.000Z"),
+        { schemaVersion: SCHEMA_VERSION, id: "ses_bad" },
+      ]);
+
+      // Act
+      const report = store.recover();
+
+      // Assert
+      expect(report.recovered).toEqual(["ses_good"]);
+      expect(report.skipped).toHaveLength(1);
+      expect(report.skipped[0]!.error).toContain("invalid record");
+      expect(store.get("ses_bad")).toBeUndefined();
+    });
+  });
+}