@cueloop/daemon 0.1.0-alpha.11 → 0.1.0-alpha.111

package/src/store.jsonl.test.ts

@@ -0,0 +1,173 @@
+/**
+ * The JSONL thread store's own behaviors, beyond the shared repository contract: per-project bucket
+ * layout keyed by root commit, one-time migration of the old whole-record files, compaction of a
+ * long log, and a crash's torn trailing line being skipped on read.
+ */
+
+import { describe, expect, test } from "bun:test";
+import {
+  mkdtempSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  existsSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ThreadStore } from "./store";
+import { migratedSessionsDir, sessionsDir, threadBucket } from "./paths";
+import { createTestThreadRecord } from "./testing/store-conformance";
+
+function tmpHome(): string {
+  return mkdtempSync(join(tmpdir(), "cueloop-jsonl-"));
+}
+
+describe("JSONL thread store", () => {
+  test("a thread is written under a per-project bucket keyed by root commit", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        workspace: { repoRoot: "/repo", branch: "main", rootCommit: "abcdef0123456789" },
+      }),
+    );
+
+    const bucket = threadBucket("abcdef0123456789", home);
+
+    expect(readdirSync(bucket)).toContain("ses_a.jsonl");
+  });
+
+  test("a non-commit rootCommit (path traversal) cannot escape the threads directory", () => {
+    const home = tmpHome();
+    const escape = threadBucket("../../../etc/cueloop", home);
+    const standalone = threadBucket(undefined, home);
+
+    // a value that is not a real commit SHA is refused to the standalone bucket, never a parent path
+    expect(escape).toBe(standalone);
+    expect(escape.startsWith(join(home, "threads"))).toBe(true);
+  });
+
+  test("a repo-less thread lands in the standalone bucket", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_s", "2026-09-01T00:00:00.000Z"));
+
+    expect(readdirSync(threadBucket(undefined, home))).toContain("ses_s.jsonl");
+  });
+
+  test("boot migrates the old whole-record file and parks the original", () => {
+    const home = tmpHome();
+
+    mkdirSync(sessionsDir(home), { recursive: true });
+    const legacy = createTestThreadRecord("ses_old", "2026-09-01T00:00:00.000Z");
+
+    writeFileSync(join(sessionsDir(home), "ses_old.json"), JSON.stringify(legacy));
+
+    const store = new ThreadStore(home);
+    const report = store.recover();
+
+    expect(report.recovered).toEqual(["ses_old"]);
+    // the record migrates intact (a history is derived on read, as the old store did)
+    expect(store.get("ses_old")?.id).toBe("ses_old");
+    expect(store.get("ses_old")?.artifact).toEqual(legacy.artifact);
+    // the original is parked, not left in the live legacy dir
+    expect(existsSync(join(sessionsDir(home), "ses_old.json"))).toBe(false);
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_old.json");
+  });
+
+  test("migration never overwrites a newer JSONL with an older legacy snapshot", () => {
+    const home = tmpHome();
+    // a newer JSONL already holds this thread (e.g. written after a rollback)
+    const store = new ThreadStore(home);
+
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "newer" },
+      }),
+    );
+    // an older legacy JSON for the same thread is restored alongside it
+    mkdirSync(sessionsDir(home), { recursive: true });
+    writeFileSync(
+      join(sessionsDir(home), "ses_x.json"),
+      JSON.stringify(
+        createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+          workspace: { repoRoot: "/repo", branch: "older" },
+        }),
+      ),
+    );
+
+    const reopened = new ThreadStore(home);
+
+    reopened.recover();
+    // the newer JSONL wins; the legacy snapshot is parked, not applied
+    expect(reopened.get("ses_x")?.workspace.branch).toBe("newer");
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_x.json");
+  });
+
+  test("a long log compacts to a single line, and the record survives it", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+
+    store.recover();
+    // more upserts than the compaction threshold
+    for (let iteration = 0; iteration < 60; iteration++) {
+      store.upsert(
+        createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+          status: "pending",
+          workspace: { repoRoot: "/repo", branch: `b${iteration}` },
+        }),
+      );
+    }
+
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+    const lines = readFileSync(file, "utf8")
+      .split("\n")
+      .filter((l) => l.trim());
+
+    expect(lines.length).toBeLessThan(60);
+    // the latest snapshot survives the compaction and a restart
+    const reopened = new ThreadStore(home);
+
+    reopened.recover();
+    expect(reopened.get("ses_a")?.workspace.branch).toBe("b59");
+  });
+
+  test("a crash's torn trailing line is skipped; the last valid snapshot wins", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z"));
+
+    // simulate a crash mid-append: a half-written line at the end
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+
+    appendFileSync(file, '{"id":"ses_a","half');
+
+    const reopened = new ThreadStore(home);
+    const report = reopened.recover();
+
+    expect(report.recovered).toEqual(["ses_a"]);
+    expect(reopened.get("ses_a")?.id).toBe("ses_a");
+
+    // an update after a torn recovery must not fuse onto the torn fragment and vanish on restart
+    reopened.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "after-tear" },
+      }),
+    );
+    const restarted = new ThreadStore(home);
+
+    restarted.recover();
+    expect(restarted.get("ses_a")?.workspace.branch).toBe("after-tear");
+  });
+});

package/src/store.ts

@@ -1,68 +1,312 @@
 /**
- * Session persistence (#14): one JSON document per session, every write
- * through a temp file + atomic rename so a crash mid-write can never leave
- * a corrupt record. Recovery is a read-only scan; records that fail to
- * parse are skipped and reported, never deleted.
+ * Thread persistence: one append-only JSONL log per thread, under a per-project bucket keyed by the
+ * repo's durable root commit (a shared bucket for repo-less threads). Each mutation appends a
+ * snapshot line and the last valid line wins, so a crash's torn tail is simply skipped on read; the
+ * log is compacted to one line on the first write, once it grows long, and when the thread resolves.
+ * Boot migrates the old whole-record JSON files into this layout and parks the originals. Records
+ * that fail to parse are skipped and reported, never deleted; records from before histories existed
+ * are given one on read.
+ *
+ * `ThreadRepository` is the contract every adapter satisfies; the conformance
+ * suite in ./testing/store-conformance.ts pins it for the file store and the
+ * in-memory store alike.
  */
 
-import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import type { ReviewSession } from "@cueloop/schema";
-import { validateSessionRecord } from "./validate";
-import { sessionsDir } from "./paths";
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+import { historyFromLinear, type Thread } from "@cueloop/schema";
+import { migratedSessionsDir, sessionsDir, threadBucket, threadsDir } from "./paths";
+import { validateThreadRecord } from "./validate";
+
+/** A thread's log is rewritten to a single snapshot line once it grows past this many, so an
+ *  active thread's file never bloats unbounded while writes stay cheap appends in between. */
+const COMPACT_THRESHOLD = 40;
 
 export interface RecoveryReport {
   recovered: string[];
   skipped: { file: string; error: string }[];
 }
 
-export class SessionStore {
-  private sessions = new Map<string, ReviewSession>();
-  private readonly dir: string;
+/** What the daemon needs from session storage. */
+export interface ThreadRepository {
+  recover(): RecoveryReport;
+  get(id: string): Thread | undefined;
+  list(): Thread[];
+  upsert(session: Thread): void;
+  delete(id: string): boolean;
+}
+
+/**
+ * A record as it is read: a history is derived for records written without
+ * one. A record with no revision has no head to derive from and keeps
+ * reading without a history - migration never loses a record.
+ */
+export function withHistory(session: Thread): Thread {
+  if (session.history || session.revisions.length === 0) return session;
 
-  constructor(home: string) {
-    this.dir = sessionsDir(home);
-    mkdirSync(this.dir, { recursive: true });
+  return { ...session, history: historyFromLinear(session) };
+}
+
+/** Records in the order `list()` promises: oldest first. */
+function oldestFirst(sessions: Iterable<Thread>): Thread[] {
+  return [...sessions].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+}
+
+export class ThreadStore implements ThreadRepository {
+  private sessions = new Map<string, Thread>();
+  /** id -> its JSONL path (the bucket depends on the record's root commit, so it is tracked). */
+  private files = new Map<string, string>();
+  /** id -> lines in its log, so upsert knows when to compact instead of append. */
+  private lineCounts = new Map<string, number>();
+
+  constructor(private readonly home: string) {
+    mkdirSync(threadsDir(home), { recursive: true });
+    // the legacy dir is where a pre-upgrade daemon wrote; ensure it exists so its scan is uniform
+    mkdirSync(sessionsDir(home), { recursive: true });
   }
 
-  /** Read-only scan of the state directory; called once on boot. */
   recover(): RecoveryReport {
     const report: RecoveryReport = { recovered: [], skipped: [] };
-    for (const file of readdirSync(this.dir)) {
+
+    this.migrateLegacy(report);
+    const root = threadsDir(this.home);
+
+    for (const bucket of readdirSync(root)) {
+      const bucketPath = join(root, bucket);
+
+      if (!statSync(bucketPath).isDirectory()) continue;
+
+      for (const file of readdirSync(bucketPath)) {
+        if (!file.endsWith(".jsonl")) continue;
+
+        const filePath = join(bucketPath, file);
+
+        try {
+          const { session, lines, torn } = readThread(filePath);
+
+          this.sessions.set(session.id, session);
+          this.files.set(session.id, filePath);
+          this.lineCounts.set(session.id, lines);
+          report.recovered.push(session.id);
+
+          // a torn trailing fragment would fuse with the next append and lose that update, so rewrite
+          // the log to a single clean snapshot line now that the last valid record is in hand
+          if (torn) this.writeWhole(session);
+        } catch (error) {
+          report.skipped.push({
+            file,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    }
+
+    return report;
+  }
+
+  /** One-time upgrade: write each old whole-record JSON as a thread log and park the original, so
+   *  the threads scan below recovers it. Invalid records are reported and left in place, never lost. */
+  private migrateLegacy(report: RecoveryReport): void {
+    const legacyDir = sessionsDir(this.home);
+
+    if (!existsSync(legacyDir)) return;
+
+    const parked = migratedSessionsDir(this.home);
+
+    for (const file of readdirSync(legacyDir)) {
       if (!file.endsWith(".json")) continue;
+
+      const legacyPath = join(legacyDir, file);
+      let session: Thread;
+
       try {
-        const raw = readFileSync(join(this.dir, file), "utf8");
-        const parsed = validateSessionRecord(JSON.parse(raw));
+        const parsed = validateThreadRecord(JSON.parse(readFileSync(legacyPath, "utf8")));
+
         if (!parsed.ok) throw new Error(`invalid record - ${parsed.error}`);
-        const session = parsed.value as ReviewSession;
-        this.sessions.set(session.id, session);
-        report.recovered.push(session.id);
-      } catch (err) {
-        report.skipped.push({ file, error: err instanceof Error ? err.message : String(err) });
+
+        session = withHistory(parsed.value);
+      } catch (error) {
+        report.skipped.push({
+          file,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        continue;
       }
+      const targetPath = join(
+        threadBucket(session.workspace.rootCommit, this.home),
+        `${session.id}.jsonl`,
+      );
+
+      // after a rollback a newer JSONL can already hold this thread; never overwrite it with the
+      // older legacy snapshot - the existing log stays authoritative and the threads scan recovers it
+      if (!existsSync(targetPath)) this.writeWhole(session);
+
+      // forget the in-memory tracking migration seeded: the threads scan is the one recovery path
+      this.files.delete(session.id);
+      this.lineCounts.delete(session.id);
+      mkdirSync(parked, { recursive: true });
+      renameSync(legacyPath, join(parked, file));
     }
-    return report;
   }
 
-  get(id: string): ReviewSession | undefined {
+  /** Rewrite the thread to a single snapshot line, atomically - the first write and every compaction. */
+  private writeWhole(session: Thread): void {
+    const filePath =
+      this.files.get(session.id) ??
+      join(threadBucket(session.workspace.rootCommit, this.home), `${session.id}.jsonl`);
+
+    mkdirSync(dirname(filePath), { recursive: true });
+    const tempPath = filePath + ".tmp";
+
+    writeFileSync(tempPath, JSON.stringify(session) + "\n");
+    renameSync(tempPath, filePath);
+    this.files.set(session.id, filePath);
+    this.lineCounts.set(session.id, 1);
+  }
+
+  get(id: string): Thread | undefined {
     return this.sessions.get(id);
   }
 
-  list(): ReviewSession[] {
-    return [...this.sessions.values()].sort((a, b) => a.createdAt.localeCompare(b.createdAt));
+  list(): Thread[] {
+    return oldestFirst(this.sessions.values());
   }
 
-  upsert(session: ReviewSession): void {
+  upsert(session: Thread): void {
+    const filePath = this.files.get(session.id);
+    const lines = this.lineCounts.get(session.id) ?? 0;
+
+    // the first write, an over-long log, and a resolved (now immutable) thread all compact to one
+    // line; every other mutation is a cheap append that a torn tail on crash simply drops on read
+    if (
+      filePath === undefined ||
+      lines >= COMPACT_THRESHOLD ||
+      session.status === "resolved" ||
+      session.messageOperations?.length
+    ) {
+      this.writeWhole(session);
+      this.sessions.set(session.id, session);
+
+      return;
+    }
+
+    appendFileSync(filePath, JSON.stringify(session) + "\n");
+    this.lineCounts.set(session.id, lines + 1);
     this.sessions.set(session.id, session);
-    const path = join(this.dir, `${session.id}.json`);
-    const tmp = path + ".tmp";
-    writeFileSync(tmp, JSON.stringify(session, null, 2));
-    renameSync(tmp, path);
   }
 
   delete(id: string): boolean {
     if (!this.sessions.delete(id)) return false;
-    rmSync(join(this.dir, `${id}.json`), { force: true });
+
+    const filePath = this.files.get(id);
+
+    if (filePath) rmSync(filePath, { force: true });
+
+    this.files.delete(id);
+    this.lineCounts.delete(id);
+
     return true;
   }
 }
+
+/**
+ * Read a thread log: the last valid snapshot line wins, so a crash's torn trailing line is skipped.
+ * `torn` is true when that winning line was not the last physical line - the caller rewrites the log
+ * so the next append starts on a clean boundary instead of fusing onto the torn fragment.
+ */
+function readThread(filePath: string) {
+  const allLines = readFileSync(filePath, "utf8")
+    .split("\n")
+    .filter((line) => line.trim().length > 0);
+
+  for (let index = allLines.length - 1; index >= 0; index--) {
+    let record: unknown;
+
+    try {
+      record = JSON.parse(allLines[index]!);
+    } catch {
+      continue;
+    }
+    const parsed = validateThreadRecord(record);
+
+    if (parsed.ok) {
+      return {
+        session: withHistory(parsed.value),
+        lines: allLines.length,
+        torn: index !== allLines.length - 1,
+      };
+    }
+  }
+  throw new Error("no valid record line");
+}
+
+/**
+ * The in-memory adapter: the same contract with nothing on disk. `seed` stands
+ * in for what a file store finds on recovery, so validation and migration are
+ * exercised the same way.
+ */
+export class MemoryThreadStore implements ThreadRepository {
+  private sessions = new Map<string, Thread>();
+
+  constructor(private readonly seed: unknown[] = []) {}
+
+  recover(): RecoveryReport {
+    const report: RecoveryReport = { recovered: [], skipped: [] };
+
+    this.seed.forEach((record, index) => {
+      const parsed = validateThreadRecord(record);
+
+      if (!parsed.ok) {
+        report.skipped.push({ file: `seed[${index}]`, error: `invalid record - ${parsed.error}` });
+
+        return;
+      }
+
+      const session = withHistory(parsed.value);
+
+      this.sessions.set(session.id, session);
+      report.recovered.push(session.id);
+    });
+
+    return report;
+  }
+
+  get(id: string): Thread | undefined {
+    return this.sessions.get(id);
+  }
+
+  list(): Thread[] {
+    return oldestFirst(this.sessions.values());
+  }
+
+  upsert(session: Thread): void {
+    this.sessions.set(session.id, session);
+  }
+
+  delete(id: string): boolean {
+    return this.sessions.delete(id);
+  }
+}
+
+/** @deprecated use ThreadRepository */
+export type SessionRepository = ThreadRepository;
+/** @deprecated use ThreadStore */
+export const SessionStore = ThreadStore;
+
+/** @deprecated use ThreadStore */
+export type SessionStore = ThreadStore;
+/** @deprecated use MemoryThreadStore */
+export const MemorySessionStore = MemoryThreadStore;
+
+/** @deprecated use MemoryThreadStore */
+export type MemorySessionStore = MemoryThreadStore;