@cueloop/daemon 0.1.0-alpha.9 → 0.1.0-alpha.90

package/src/share-blob.test.ts

@@ -0,0 +1,112 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { gzipSync } from "node:zlib";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { viewFollowing, type Artifact, type WorkspaceKey } from "@cueloop/schema";
+import { DaemonCore } from "./api";
+import { MAX_BLOB_BYTES, packSessionBlob, unpackSessionBlob } from "./share-blob";
+
+const WS: WorkspaceKey = { repoRoot: "/repo", branch: "main" };
+const PLAN: Artifact = { type: "plan", content: "# Plan\n\nDo the thing.\n", meta: {} };
+
+let home: string;
+let core: DaemonCore;
+
+beforeEach(() => {
+  home = mkdtempSync(join(tmpdir(), "cueloop-blob-"));
+  core = new DaemonCore(home);
+});
+afterEach(() => rmSync(home, { recursive: true, force: true }));
+
+describe("share blob roundtrip", () => {
+  test("pack then unpack restores the session with its annotations", () => {
+    // Arrange
+    const session = core.sessionCreate({ workspace: WS, artifact: PLAN });
+    const annotated = core.sessionAnnotate(session.id, {
+      id: "a_1",
+      kind: "comment",
+      anchor: { quote: "the thing", prefix: "", suffix: "" },
+      body: "why this?",
+    });
+
+    // Act
+    const restored = unpackSessionBlob(packSessionBlob(annotated));
+
+    // Assert
+    expect(restored).toEqual(annotated);
+  });
+});
+
+describe("the payload cap against a long review", () => {
+  test("a plan the size of the examples with a long discussion and its history stays far under the cap", () => {
+    // Arrange: the largest example plan, 200 rounds of comment + reply, 20 reviewer edits, 10 removals
+    const paragraph =
+      "Review sessions currently live only in daemon memory. If the daemon crashes mid-review, every pending annotation is lost. This plan makes sessions durable without changing the daemon's socket protocol.\n\n";
+    const content = `# Implementation Plan: Session Persistence\n\n## Context\n\n${paragraph.repeat(8)}`;
+    const session = core.sessionCreate({
+      workspace: WS,
+      artifact: { type: "plan", content, meta: {} },
+    });
+
+    for (let round = 0; round < 200; round++) {
+      core.sessionAnnotate(session.id, {
+        id: `a_${round}`,
+        kind: "comment",
+        anchor: { quote: "daemon memory", prefix: "live only in ", suffix: ". If the" },
+        body: `Round ${round}: what happens when the store directory is read-only at startup?`,
+        author: "SHA256:collaborator-fingerprint-0123456789abcdef",
+      });
+      core.sessionAnnotate(session.id, {
+        id: `r_${round}`,
+        kind: "comment",
+        anchor: { quote: "daemon memory", prefix: "live only in ", suffix: ". If the" },
+        body: "Fail fast with a readable error; the review never opens on a store that cannot write.",
+        replyTo: `a_${round}`,
+      });
+      if (round % 10 === 0)
+        core.sessionSetWorkingCopy(session.id, `${content}\n\nEdit ${round}.\n`);
+      if (round % 20 === 0) core.sessionRemoveAnnotation(session.id, `r_${round}`);
+    }
+    const shared = viewFollowing(core.sessionGet(session.id));
+
+    // Act
+    const decompressed = Buffer.byteLength(JSON.stringify(shared), "utf8");
+    const packed = packSessionBlob(shared).byteLength;
+
+    // Assert: a quarter of the cap decompressed, and the packed blob a small
+    // fraction of it; the round-trip holds
+    expect(shared.history!.entries.length).toBeGreaterThan(400);
+    expect(decompressed).toBeLessThan(MAX_BLOB_BYTES / 4);
+    expect(packed).toBeLessThan(decompressed);
+    expect(unpackSessionBlob(packSessionBlob(shared)).history!.entries.length).toBe(
+      shared.history!.entries.length,
+    );
+    console.info(
+      `share payload: ${shared.history!.entries.length} entries, ${decompressed} bytes decompressed, ${packed} bytes packed, cap ${MAX_BLOB_BYTES}`,
+    );
+  });
+});
+
+describe("unpack rejects bad input", () => {
+  test("caps the decompressed size, so a compression bomb throws", () => {
+    // Arrange: tiny gzip, huge inflation - the decompression-bomb shape
+    const bomb = gzipSync(Buffer.alloc(MAX_BLOB_BYTES + 1, 0x61));
+
+    // Act / Assert
+    expect(() => unpackSessionBlob(bomb)).toThrow(/exceeds|maxOutputLength/i);
+  });
+
+  test("throws on bytes that are not gzip", () => {
+    // Act / Assert
+    expect(() => unpackSessionBlob(Buffer.from("not gzip at all"))).toThrow(/gzip/i);
+  });
+
+  test("throws on a shape that is not a session", () => {
+    // Arrange
+    const notASession = gzipSync(Buffer.from(JSON.stringify({ hello: "world" })));
+
+    // Act / Assert
+    expect(() => unpackSessionBlob(notASession)).toThrow(/not a valid session/i);
+  });
+});

package/src/share-blob.ts

@@ -0,0 +1,55 @@
+/**
+ * The share-blob wire format: how a Thread travels from `cueloop share`
+ * to the gateway. The planner's CLI packs (this is the only thing it can do -
+ * it holds no encryption key); the gateway unpacks, then encrypts for R2.
+ *
+ * Format is gzip(JSON). Unpack is the untrusted boundary - the bytes come off
+ * an SSH channel from anyone who can reach the `share` user - so it caps the
+ * decompressed size (a decompression-bomb guard) and runs the same record
+ * validator the daemon store uses, never trusting the shape.
+ */
+
+import { gzipSync, gunzipSync } from "node:zlib";
+import type { Thread } from "@cueloop/schema";
+import { validateThreadRecord } from "./validate";
+
+/** Decompressed ceiling for one shared session (ADR 0004's payload cap). */
+export const MAX_BLOB_BYTES = 1024 * 1024;
+
+/** The SSH username `cueloop share` connects as to upload; the gateway routes on it. */
+export const SHARE_UPLOAD_USER = "share";
+
+/** Where shares live, in one place: the client uploads here, the gateway prints it. */
+export const DEFAULT_SHARE_HOST = "cueloop.dev";
+export const DEFAULT_SHARE_PORT = 22;
+
+/** Serialise + compress a session for upload. */
+export function packSessionBlob(session: Thread): Buffer {
+  return gzipSync(Buffer.from(JSON.stringify(session), "utf8"));
+}
+
+/** Decompress + validate an uploaded blob, or throw a precise reason. */
+export function unpackSessionBlob(bytes: Uint8Array): Thread {
+  let json: string;
+
+  try {
+    json = gunzipSync(bytes, { maxOutputLength: MAX_BLOB_BYTES }).toString("utf8");
+  } catch (err) {
+    throw new Error(
+      `blob is not valid gzip or exceeds ${MAX_BLOB_BYTES} bytes: ${err instanceof Error ? err.message : String(err)}`,
+      { cause: err },
+    );
+  }
+  let raw: unknown;
+
+  try {
+    raw = JSON.parse(json);
+  } catch {
+    throw new Error("blob is not valid JSON");
+  }
+  const parsed = validateThreadRecord(raw);
+
+  if (!parsed.ok) throw new Error(`blob is not a valid session: ${parsed.error}`);
+
+  return parsed.value;
+}

package/src/single-instance.test.ts

@@ -1,16 +1,17 @@
 /**
- * Single-instance guarantee (#14): one daemon per home. Two daemons over one
+ * Single-instance guarantee: one daemon per home. Two daemons over one
  * state directory means divergent in-memory sessions and clients seeing
  * whichever half they connect to.
  */
 
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { createServer } from "node:net";
 import { DaemonServer } from "./server";
 import { DaemonClient } from "./client";
-import { lockPath, socketPath } from "./paths";
+import { lockPath, ownerTokenPath, socketPath } from "./paths";
 import type { Artifact, WorkspaceKey } from "@cueloop/schema";
 
 const WS: WorkspaceKey = { repoRoot: "/repo", branch: "main" };
@@ -23,33 +24,61 @@ beforeEach(() => {
   home = mkdtempSync(join(tmpdir(), "cueloop-lock-"));
 });
 afterEach(() => {
-  for (const s of servers.splice(0)) s.stop();
+  for (const runningServer of servers.splice(0)) runningServer.stop();
   rmSync(home, { recursive: true, force: true });
 });
 
 function server(): DaemonServer {
-  const s = new DaemonServer({ home, idleExitMs: 0 });
-  servers.push(s);
-  return s;
+  const daemonServer = new DaemonServer({ home, idleExitMs: 0 });
+
+  servers.push(daemonServer);
+
+  return daemonServer;
 }
 
 describe("one daemon per home", () => {
   test("the second start is refused instead of stealing the socket", () => {
+    // Arrange
     const first = server();
     const path = first.start();
+
     expect(path).toBe(socketPath(home));
     const second = server();
+
+    // Assert
     expect(second.start()).toBeNull();
     // the winner's socket survives untouched
     expect(existsSync(path!)).toBe(true);
   });
 
+  test("stopping a refused server leaves the lock owner's socket intact", async () => {
+    const first = server();
+    const path = first.start();
+    const refused = server();
+
+    expect(refused.start()).toBeNull();
+    refused.stop();
+    expect(existsSync(path!)).toBe(true);
+    const client = await DaemonClient.connect({ home });
+
+    try {
+      expect((await client.ping()).pid).toBe(process.pid);
+    } finally {
+      client.close();
+    }
+  });
+
   test("clients keep talking to the one live daemon after a refused start", async () => {
+    // Arrange
     const first = server();
+
     first.start();
     const created = first.core.sessionCreate({ workspace: WS, artifact: PLAN });
+
     server().start(); // refused
     const client = await DaemonClient.connect({ home });
+
+    // Assert
     try {
       // no split brain: the session created on the winner is visible
       expect((await client.sessionGet(created.id)).id).toBe(created.id);
@@ -60,42 +89,162 @@ describe("one daemon per home", () => {
   });
 
   test("a stale lock from a crashed daemon is reclaimed", () => {
+    // Arrange
     // a pid that cannot be running (init is never a cueloop daemon, and this
     // simulates the record a crashed process leaves behind)
     writeFileSync(lockPath(home), "999999");
-    const s = server();
-    expect(s.start()).toBe(socketPath(home));
+    const daemonServer = server();
+
+    // Assert
+    expect(daemonServer.start()).toBe(socketPath(home));
+  });
+
+  test("a daemon does not steal a lock while its owner writes the pid", async () => {
+    const path = lockPath(home);
+    const script = `
+      import { openSync, writeFileSync } from "node:fs";
+      const fd = openSync(${JSON.stringify(path)}, "wx");
+      console.log("lock created");
+      setTimeout(() => writeFileSync(fd, String(process.pid)), 100);
+      setTimeout(() => {}, 2_000);
+    `;
+    const owner = Bun.spawn([process.execPath, "-e", script], {
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+
+    try {
+      const firstOutput = await owner.stdout.getReader().read();
+
+      expect(new TextDecoder().decode(firstOutput.value)).toContain("lock created");
+      expect(server().start()).toBeNull();
+      expect(existsSync(socketPath(home))).toBe(false);
+    } finally {
+      owner.kill();
+      await owner.exited;
+    }
   });
 
+  test("an autostarted daemon reclaims a stale socket after taking the lock", async () => {
+    writeFileSync(socketPath(home), "stale socket");
+    const client = await DaemonClient.connect({ home, autostart: true });
+
+    try {
+      expect((await client.ping()).pid).toBeGreaterThan(0);
+    } finally {
+      await client.shutdown();
+      client.close();
+    }
+  });
+
+  test("autostart reports the daemon error when the child cannot start", async () => {
+    mkdirSync(ownerTokenPath(home));
+    const previousTimeout = process.env.CUELOOP_START_TIMEOUT_MS;
+
+    // This checks error reporting, not startup speed; allow the child to start on a loaded runner.
+    process.env.CUELOOP_START_TIMEOUT_MS = "5000";
+    try {
+      await expect(DaemonClient.connect({ home, autostart: true })).rejects.toThrow("EISDIR");
+    } finally {
+      if (previousTimeout === undefined) delete process.env.CUELOOP_START_TIMEOUT_MS;
+      else process.env.CUELOOP_START_TIMEOUT_MS = previousTimeout;
+    }
+  }, 10_000);
+
   test("stopping releases the lock so a restart works", () => {
+    // Arrange
     const first = server();
+
     first.start();
+
+    // Act
     first.stop();
+
+    // Assert
     expect(existsSync(lockPath(home))).toBe(false);
     const second = server();
+
     expect(second.start()).toBe(socketPath(home));
   });
 
   test("concurrent autostarts converge on a single daemon", async () => {
+    // Act
     // several clients race to autostart; all must end up on the same daemon
     const clients = await Promise.all([
       DaemonClient.connect({ home, autostart: true }),
       DaemonClient.connect({ home, autostart: true }),
       DaemonClient.connect({ home, autostart: true }),
     ]);
+
+    // Assert
     try {
-      const pids = await Promise.all(clients.map((c) => c.ping()));
-      const unique = new Set(pids.map((p) => p.pid));
+      const pids = await Promise.all(clients.map((client) => client.ping()));
+      const unique = new Set(pids.map((ping) => ping.pid));
+
       expect(unique.size).toBe(1);
       // and state is shared: one client's session is visible to the others
-      const s = await clients[0]!.sessionCreate(WS, PLAN);
-      expect((await clients[2]!.sessionGet(s.id)).id).toBe(s.id);
+      const session = await clients[0]!.sessionCreate(WS, PLAN);
+
+      expect((await clients[2]!.sessionGet(session.id)).id).toBe(session.id);
     } finally {
       for (const c of clients) c.close();
       // the autostarted daemon is not one of `servers` - shut it down
       const admin = await DaemonClient.connect({ home });
+
       await admin.shutdown();
       admin.close();
     }
   }, 60_000);
+
+  test("autostart joins a daemon that bound its socket after the first dial", async () => {
+    const running = server();
+    const path = running.start();
+    const created = running.core.sessionCreate({ workspace: WS, artifact: PLAN });
+    const client = new DaemonClient();
+    const previousTimeout = process.env.CUELOOP_START_TIMEOUT_MS;
+
+    client["home"] = home;
+    process.env.CUELOOP_START_TIMEOUT_MS = "250";
+
+    try {
+      await client["attachFreshDaemon"](home, path!);
+
+      expect((await client.sessionGet(created.id)).id).toBe(created.id);
+      expect(existsSync(path!)).toBe(true);
+    } finally {
+      client.close();
+      if (previousTimeout === undefined) delete process.env.CUELOOP_START_TIMEOUT_MS;
+      else process.env.CUELOOP_START_TIMEOUT_MS = previousTimeout;
+    }
+  });
+
+  test("autostart retries after a socket closes during the handshake", async () => {
+    const path = socketPath(home);
+    const replacement = server();
+    const transient = createServer((socket) => {
+      socket.destroy();
+      transient.close(() => {
+        rmSync(path, { force: true });
+        rmSync(lockPath(home), { force: true });
+        replacement.start();
+      });
+    });
+
+    writeFileSync(lockPath(home), String(process.pid));
+    await new Promise<void>((resolve) => transient.listen(path, resolve));
+    const client = new DaemonClient();
+    const previousTimeout = process.env.CUELOOP_START_TIMEOUT_MS;
+
+    client["home"] = home;
+    process.env.CUELOOP_START_TIMEOUT_MS = "1000";
+    try {
+      await client["attachFreshDaemon"](home, path);
+      expect((await client.ping()).pid).toBe(process.pid);
+    } finally {
+      client.close();
+      if (transient.listening) transient.close();
+      if (previousTimeout === undefined) delete process.env.CUELOOP_START_TIMEOUT_MS;
+      else process.env.CUELOOP_START_TIMEOUT_MS = previousTimeout;
+    }
+  });
 });

package/src/store.conformance.test.ts

@@ -0,0 +1,41 @@
+import { mkdtempSync, writeFileSync } from "node:fs";
+import * as v from "valibot";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { MemoryThreadStore, ThreadStore, type ThreadRepository } from "./store";
+import { sessionsDir } from "./paths";
+import { runThreadStoreConformance } from "./testing/store-conformance";
+
+const homes = new WeakMap<ThreadRepository, string>();
+
+runThreadStoreConformance("file store", {
+  open: (records) => {
+    const home = mkdtempSync(join(tmpdir(), "cueloop-store-"));
+    const store = new ThreadStore(home);
+
+    // records land on disk the way an earlier daemon would have written them;
+    // the file name comes from the record's id when it has one
+    records.forEach((record, index) => {
+      const named = v.safeParse(v.object({ id: v.string() }), record);
+      const id = named.success ? named.output.id : `bad_${index}`;
+
+      writeFileSync(join(sessionsDir(home), `${id}.json`), JSON.stringify(record));
+    });
+    homes.set(store, home);
+
+    return store;
+  },
+  restart: (store) => {
+    const reopened = new ThreadStore(homes.get(store)!);
+
+    homes.set(reopened, homes.get(store)!);
+
+    return reopened;
+  },
+});
+
+runThreadStoreConformance("memory store", {
+  open: (records) => new MemoryThreadStore(records),
+  // nothing persists past the instance: a restart recovers what it was seeded with
+  restart: (store) => new MemoryThreadStore(store.list()),
+});

package/src/store.jsonl.test.ts

@@ -0,0 +1,159 @@
+/**
+ * The JSONL thread store's own behaviors, beyond the shared repository contract: per-project bucket
+ * layout keyed by root commit, one-time migration of the old whole-record files, compaction of a
+ * long log, and a crash's torn trailing line being skipped on read.
+ */
+
+import { describe, expect, test } from "bun:test";
+import {
+  mkdtempSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  existsSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ThreadStore } from "./store";
+import { migratedSessionsDir, sessionsDir, threadBucket } from "./paths";
+import { createTestThreadRecord } from "./testing/store-conformance";
+
+function tmpHome(): string {
+  return mkdtempSync(join(tmpdir(), "cueloop-jsonl-"));
+}
+
+describe("JSONL thread store", () => {
+  test("a thread is written under a per-project bucket keyed by root commit", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        workspace: { repoRoot: "/repo", branch: "main", rootCommit: "abcdef0123456789" },
+      }),
+    );
+
+    const bucket = threadBucket("abcdef0123456789", home);
+    expect(readdirSync(bucket)).toContain("ses_a.jsonl");
+  });
+
+  test("a non-commit rootCommit (path traversal) cannot escape the threads directory", () => {
+    const home = tmpHome();
+    const escape = threadBucket("../../../etc/cueloop", home);
+    const standalone = threadBucket(undefined, home);
+
+    // a value that is not a real commit SHA is refused to the standalone bucket, never a parent path
+    expect(escape).toBe(standalone);
+    expect(escape.startsWith(join(home, "threads"))).toBe(true);
+  });
+
+  test("a repo-less thread lands in the standalone bucket", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_s", "2026-09-01T00:00:00.000Z"));
+
+    expect(readdirSync(threadBucket(undefined, home))).toContain("ses_s.jsonl");
+  });
+
+  test("boot migrates the old whole-record file and parks the original", () => {
+    const home = tmpHome();
+    mkdirSync(sessionsDir(home), { recursive: true });
+    const legacy = createTestThreadRecord("ses_old", "2026-09-01T00:00:00.000Z");
+    writeFileSync(join(sessionsDir(home), "ses_old.json"), JSON.stringify(legacy));
+
+    const store = new ThreadStore(home);
+    const report = store.recover();
+
+    expect(report.recovered).toEqual(["ses_old"]);
+    // the record migrates intact (a history is derived on read, as the old store did)
+    expect(store.get("ses_old")?.id).toBe("ses_old");
+    expect(store.get("ses_old")?.artifact).toEqual(legacy.artifact);
+    // the original is parked, not left in the live legacy dir
+    expect(existsSync(join(sessionsDir(home), "ses_old.json"))).toBe(false);
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_old.json");
+  });
+
+  test("migration never overwrites a newer JSONL with an older legacy snapshot", () => {
+    const home = tmpHome();
+    // a newer JSONL already holds this thread (e.g. written after a rollback)
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "newer" },
+      }),
+    );
+    // an older legacy JSON for the same thread is restored alongside it
+    mkdirSync(sessionsDir(home), { recursive: true });
+    writeFileSync(
+      join(sessionsDir(home), "ses_x.json"),
+      JSON.stringify(
+        createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+          workspace: { repoRoot: "/repo", branch: "older" },
+        }),
+      ),
+    );
+
+    const reopened = new ThreadStore(home);
+    reopened.recover();
+    // the newer JSONL wins; the legacy snapshot is parked, not applied
+    expect(reopened.get("ses_x")?.workspace.branch).toBe("newer");
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_x.json");
+  });
+
+  test("a long log compacts to a single line, and the record survives it", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    // more upserts than the compaction threshold
+    for (let iteration = 0; iteration < 60; iteration++) {
+      store.upsert(
+        createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+          status: "pending",
+          workspace: { repoRoot: "/repo", branch: `b${iteration}` },
+        }),
+      );
+    }
+
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+    const lines = readFileSync(file, "utf8")
+      .split("\n")
+      .filter((l) => l.trim());
+    expect(lines.length).toBeLessThan(60);
+    // the latest snapshot survives the compaction and a restart
+    const reopened = new ThreadStore(home);
+    reopened.recover();
+    expect(reopened.get("ses_a")?.workspace.branch).toBe("b59");
+  });
+
+  test("a crash's torn trailing line is skipped; the last valid snapshot wins", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z"));
+
+    // simulate a crash mid-append: a half-written line at the end
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+    appendFileSync(file, '{"id":"ses_a","half');
+
+    const reopened = new ThreadStore(home);
+    const report = reopened.recover();
+    expect(report.recovered).toEqual(["ses_a"]);
+    expect(reopened.get("ses_a")?.id).toBe("ses_a");
+
+    // an update after a torn recovery must not fuse onto the torn fragment and vanish on restart
+    reopened.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "after-tear" },
+      }),
+    );
+    const restarted = new ThreadStore(home);
+    restarted.recover();
+    expect(restarted.get("ses_a")?.workspace.branch).toBe("after-tear");
+  });
+});