@cueloop/daemon 0.1.0-alpha.68 → 0.1.0-alpha.69

package/src/store.ts

@@ -1,21 +1,37 @@
 /**
- * Session persistence: one JSON record per session, written whole through a
- * temp file and an atomic rename so a crash never leaves a torn file. Recovery
- * is a read-only scan of the state directory; records that fail to parse are
- * skipped and reported, never deleted, and records from before histories
- * existed are given one on read.
+ * Thread persistence: one append-only JSONL log per thread, under a per-project bucket keyed by the
+ * repo's durable root commit (a shared bucket for repo-less threads). Each mutation appends a
+ * snapshot line and the last valid line wins, so a crash's torn tail is simply skipped on read; the
+ * log is compacted to one line on the first write, once it grows long, and when the thread resolves.
+ * Boot migrates the old whole-record JSON files into this layout and parks the originals. Records
+ * that fail to parse are skipped and reported, never deleted; records from before histories existed
+ * are given one on read.
  *
  * `SessionRepository` is the contract every adapter satisfies; the conformance
  * suite in ./testing/store-conformance.ts pins it for the file store and the
  * in-memory store alike.
  */
 
-import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
 import { historyFromLinear, type ReviewSession } from "@cueloop/schema";
-import { sessionsDir } from "./paths";
+import { migratedSessionsDir, sessionsDir, threadBucket, threadsDir } from "./paths";
 import { validateSessionRecord } from "./validate";
 
+/** A thread's log is rewritten to a single snapshot line once it grows past this many, so an
+ *  active thread's file never bloats unbounded while writes stay cheap appends in between. */
+const COMPACT_THRESHOLD = 40;
+
 export interface RecoveryReport {
   recovered: string[];
   skipped: { file: string; error: string }[];
@@ -51,33 +67,106 @@ function oldestFirst(sessions: Iterable<ReviewSession>): ReviewSession[] {
 
 export class SessionStore implements SessionRepository {
   private sessions = new Map<string, ReviewSession>();
-  private readonly dir: string;
+  /** id -> its JSONL path (the bucket depends on the record's root commit, so it is tracked). */
+  private files = new Map<string, string>();
+  /** id -> lines in its log, so upsert knows when to compact instead of append. */
+  private lineCounts = new Map<string, number>();
 
-  constructor(home: string) {
-    this.dir = sessionsDir(home);
-    mkdirSync(this.dir, { recursive: true });
+  constructor(private readonly home: string) {
+    mkdirSync(threadsDir(home), { recursive: true });
+    // the legacy dir is where a pre-upgrade daemon wrote; ensure it exists so its scan is uniform
+    mkdirSync(sessionsDir(home), { recursive: true });
   }
 
   recover(): RecoveryReport {
     const report: RecoveryReport = { recovered: [], skipped: [] };
 
-    for (const file of readdirSync(this.dir)) {
+    this.migrateLegacy(report);
+    const root = threadsDir(this.home);
+
+    for (const bucket of readdirSync(root)) {
+      const bucketPath = join(root, bucket);
+
+      if (!statSync(bucketPath).isDirectory()) continue;
+      for (const file of readdirSync(bucketPath)) {
+        if (!file.endsWith(".jsonl")) continue;
+        const filePath = join(bucketPath, file);
+        try {
+          const { session, lines, torn } = readThread(filePath);
+
+          this.sessions.set(session.id, session);
+          this.files.set(session.id, filePath);
+          this.lineCounts.set(session.id, lines);
+          report.recovered.push(session.id);
+          // a torn trailing fragment would fuse with the next append and lose that update, so rewrite
+          // the log to a single clean snapshot line now that the last valid record is in hand
+          if (torn) this.writeWhole(session);
+        } catch (error) {
+          report.skipped.push({
+            file,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    }
+
+    return report;
+  }
+
+  /** One-time upgrade: write each old whole-record JSON as a thread log and park the original, so
+   *  the threads scan below recovers it. Invalid records are reported and left in place, never lost. */
+  private migrateLegacy(report: RecoveryReport): void {
+    const legacyDir = sessionsDir(this.home);
+
+    if (!existsSync(legacyDir)) return;
+    const parked = migratedSessionsDir(this.home);
+
+    for (const file of readdirSync(legacyDir)) {
       if (!file.endsWith(".json")) continue;
+      const legacyPath = join(legacyDir, file);
+      let session: ReviewSession;
+
       try {
-        const raw = readFileSync(join(this.dir, file), "utf8");
-        const parsed = validateSessionRecord(JSON.parse(raw));
+        const parsed = validateSessionRecord(JSON.parse(readFileSync(legacyPath, "utf8")));
 
         if (!parsed.ok) throw new Error(`invalid record - ${parsed.error}`);
-        const session = withHistory(parsed.value);
-
-        this.sessions.set(session.id, session);
-        report.recovered.push(session.id);
-      } catch (err) {
-        report.skipped.push({ file, error: err instanceof Error ? err.message : String(err) });
+        session = withHistory(parsed.value);
+      } catch (error) {
+        report.skipped.push({
+          file,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        continue;
       }
+      const targetPath = join(
+        threadBucket(session.workspace.rootCommit, this.home),
+        `${session.id}.jsonl`,
+      );
+
+      // after a rollback a newer JSONL can already hold this thread; never overwrite it with the
+      // older legacy snapshot - the existing log stays authoritative and the threads scan recovers it
+      if (!existsSync(targetPath)) this.writeWhole(session);
+      // forget the in-memory tracking migration seeded: the threads scan is the one recovery path
+      this.files.delete(session.id);
+      this.lineCounts.delete(session.id);
+      mkdirSync(parked, { recursive: true });
+      renameSync(legacyPath, join(parked, file));
     }
+  }
 
-    return report;
+  /** Rewrite the thread to a single snapshot line, atomically - the first write and every compaction. */
+  private writeWhole(session: ReviewSession): void {
+    const filePath =
+      this.files.get(session.id) ??
+      join(threadBucket(session.workspace.rootCommit, this.home), `${session.id}.jsonl`);
+
+    mkdirSync(dirname(filePath), { recursive: true });
+    const tempPath = filePath + ".tmp";
+
+    writeFileSync(tempPath, JSON.stringify(session) + "\n");
+    renameSync(tempPath, filePath);
+    this.files.set(session.id, filePath);
+    this.lineCounts.set(session.id, 1);
   }
 
   get(id: string): ReviewSession | undefined {
@@ -90,21 +179,63 @@ export class SessionStore implements SessionRepository {
 
   upsert(session: ReviewSession): void {
     this.sessions.set(session.id, session);
-    const path = join(this.dir, `${session.id}.json`);
-    const tempPath = path + ".tmp";
+    const filePath = this.files.get(session.id);
+    const lines = this.lineCounts.get(session.id) ?? 0;
+
+    // the first write, an over-long log, and a resolved (now immutable) thread all compact to one
+    // line; every other mutation is a cheap append that a torn tail on crash simply drops on read
+    if (filePath === undefined || lines >= COMPACT_THRESHOLD || session.status === "resolved") {
+      this.writeWhole(session);
 
-    writeFileSync(tempPath, JSON.stringify(session, null, 2));
-    renameSync(tempPath, path);
+      return;
+    }
+    appendFileSync(filePath, JSON.stringify(session) + "\n");
+    this.lineCounts.set(session.id, lines + 1);
   }
 
   delete(id: string): boolean {
     if (!this.sessions.delete(id)) return false;
-    rmSync(join(this.dir, `${id}.json`), { force: true });
+    const filePath = this.files.get(id);
+
+    if (filePath) rmSync(filePath, { force: true });
+    this.files.delete(id);
+    this.lineCounts.delete(id);
 
     return true;
   }
 }
 
+/**
+ * Read a thread log: the last valid snapshot line wins, so a crash's torn trailing line is skipped.
+ * `torn` is true when that winning line was not the last physical line - the caller rewrites the log
+ * so the next append starts on a clean boundary instead of fusing onto the torn fragment.
+ */
+function readThread(filePath: string) {
+  const allLines = readFileSync(filePath, "utf8")
+    .split("\n")
+    .filter((line) => line.trim().length > 0);
+
+  for (let index = allLines.length - 1; index >= 0; index--) {
+    let record: unknown;
+
+    try {
+      record = JSON.parse(allLines[index]!);
+    } catch {
+      continue;
+    }
+    const parsed = validateSessionRecord(record);
+
+    if (parsed.ok) {
+      return {
+        session: withHistory(parsed.value),
+        lines: allLines.length,
+        torn: index !== allLines.length - 1,
+      };
+    }
+  }
+  throw new Error("no valid record line");
+}
+
 /**
  * The in-memory adapter: the same contract with nothing on disk. `seed` stands
  * in for what a file store finds on recovery, so validation and migration are

package/src/validate.test.ts

@@ -186,6 +186,7 @@ describe("wire pins", () => {
     pr: "org/repo#1",
     herdrPane: "%7",
     title: "Plan",
+    workbench: false,
   };
   const fullAnchor: Required<Anchor> = {
     quote: "q",
@@ -201,6 +202,7 @@ describe("wire pins", () => {
     id: "a1",
     kind: "comment",
     anchor: fullAnchor,
+    target: { kind: "file", path: "src/x.ts", rev: "worktree" },
     body: "b",
     orphan: false,
     author: "SHA256:abc",

package/src/validate.ts

@@ -15,6 +15,7 @@ import {
   SCHEMA_VERSION,
   type Anchor,
   type Annotation,
+  type AnnotationTarget,
   type Artifact,
   type ArtifactMeta,
   type DiffFileContents,
@@ -57,6 +58,7 @@ export const ArtifactMetaSchema = v.object({
   pr: v.optional(v.string()),
   herdrPane: v.optional(v.string()),
   title: v.optional(v.string()),
+  workbench: v.optional(v.boolean()),
 } satisfies EntriesOf<ArtifactMeta>);
 
 export const DiffFileContentsSchema = v.object({
@@ -86,12 +88,20 @@ export const AnchorSchema = v.object({
   selector: v.optional(v.string()),
 } satisfies EntriesOf<Anchor>);
 
+/** The surface a note was made on; absent means the reviewed artifact. */
+export const AnnotationTargetSchema: v.GenericSchema<AnnotationTarget> = v.variant("kind", [
+  v.object({ kind: v.literal("artifact") }),
+  v.object({ kind: v.literal("file"), path: NonEmpty, rev: v.picklist(["worktree", "head"]) }),
+  v.object({ kind: v.literal("welcome") }),
+]);
+
 /** Wire annotations arrive without createdAt - the daemon stamps it. */
 export const AnnotationSchema = v.object({
   id: NonEmpty,
   /** Open kind set: the built-in is comment. */
   kind: NonEmpty,
   anchor: AnchorSchema,
+  target: v.optional(AnnotationTargetSchema),
   body: v.string(),
   orphan: v.optional(v.boolean()),
   author: v.optional(v.string()),
@@ -172,6 +182,8 @@ export const Params = {
   "repo.files": v.object({ cwd: NonEmpty }),
   "repo.fileContents": v.object({ cwd: NonEmpty, path: NonEmpty }),
   "repo.changes": v.object({ cwd: NonEmpty }),
+  "repo.diff": v.object({ cwd: NonEmpty }),
+  "session.workbench": v.object({ cwd: NonEmpty }),
   "session.navigate": v.object({
     id: SessionId,
     entryId: NonEmpty,
@@ -199,6 +211,7 @@ export const Params = {
     id: SessionId,
     verdictKind: v.picklist(["comment", "approve", "request_changes"]),
     summary: v.optional(v.string(), ""),
+    actionBodies: v.optional(v.record(v.string(), v.string())),
   }),
   "session.submitRevision": v.object({
     id: SessionId,

package/src/version.ts

@@ -0,0 +1,6 @@
+/** The daemon's build version, read from the package manifest - the handshake
+ *  uses it so a client never keeps talking to a daemon from an earlier build. */
+
+import manifest from "../package.json" with { type: "json" };
+
+export const DAEMON_VERSION: string = manifest.version;