@cueloop/daemon 0.1.0-alpha.8 → 0.1.0-alpha.80

package/src/share-blob.ts

@@ -0,0 +1,55 @@
+/**
+ * The share-blob wire format: how a Thread travels from `cueloop share`
+ * to the gateway. The planner's CLI packs (this is the only thing it can do -
+ * it holds no encryption key); the gateway unpacks, then encrypts for R2.
+ *
+ * Format is gzip(JSON). Unpack is the untrusted boundary - the bytes come off
+ * an SSH channel from anyone who can reach the `share` user - so it caps the
+ * decompressed size (a decompression-bomb guard) and runs the same record
+ * validator the daemon store uses, never trusting the shape.
+ */
+
+import { gzipSync, gunzipSync } from "node:zlib";
+import type { Thread } from "@cueloop/schema";
+import { validateThreadRecord } from "./validate";
+
+/** Decompressed ceiling for one shared session (ADR 0004's payload cap). */
+export const MAX_BLOB_BYTES = 1024 * 1024;
+
+/** The SSH username `cueloop share` connects as to upload; the gateway routes on it. */
+export const SHARE_UPLOAD_USER = "share";
+
+/** Where shares live, in one place: the client uploads here, the gateway prints it. */
+export const DEFAULT_SHARE_HOST = "cueloop.dev";
+export const DEFAULT_SHARE_PORT = 22;
+
+/** Serialise + compress a session for upload. */
+export function packSessionBlob(session: Thread): Buffer {
+  return gzipSync(Buffer.from(JSON.stringify(session), "utf8"));
+}
+
+/** Decompress + validate an uploaded blob, or throw a precise reason. */
+export function unpackSessionBlob(bytes: Uint8Array): Thread {
+  let json: string;
+
+  try {
+    json = gunzipSync(bytes, { maxOutputLength: MAX_BLOB_BYTES }).toString("utf8");
+  } catch (err) {
+    throw new Error(
+      `blob is not valid gzip or exceeds ${MAX_BLOB_BYTES} bytes: ${err instanceof Error ? err.message : String(err)}`,
+      { cause: err },
+    );
+  }
+  let raw: unknown;
+
+  try {
+    raw = JSON.parse(json);
+  } catch {
+    throw new Error("blob is not valid JSON");
+  }
+  const parsed = validateThreadRecord(raw);
+
+  if (!parsed.ok) throw new Error(`blob is not a valid session: ${parsed.error}`);
+
+  return parsed.value;
+}

package/src/single-instance.test.ts

@@ -1,5 +1,5 @@
 /**
- * Single-instance guarantee (#14): one daemon per home. Two daemons over one
+ * Single-instance guarantee: one daemon per home. Two daemons over one
  * state directory means divergent in-memory sessions and clients seeing
  * whichever half they connect to.
  */
@@ -23,33 +23,44 @@ beforeEach(() => {
   home = mkdtempSync(join(tmpdir(), "cueloop-lock-"));
 });
 afterEach(() => {
-  for (const s of servers.splice(0)) s.stop();
+  for (const runningServer of servers.splice(0)) runningServer.stop();
   rmSync(home, { recursive: true, force: true });
 });
 
 function server(): DaemonServer {
-  const s = new DaemonServer({ home, idleExitMs: 0 });
-  servers.push(s);
-  return s;
+  const daemonServer = new DaemonServer({ home, idleExitMs: 0 });
+
+  servers.push(daemonServer);
+
+  return daemonServer;
 }
 
 describe("one daemon per home", () => {
   test("the second start is refused instead of stealing the socket", () => {
+    // Arrange
     const first = server();
     const path = first.start();
+
     expect(path).toBe(socketPath(home));
     const second = server();
+
+    // Assert
     expect(second.start()).toBeNull();
     // the winner's socket survives untouched
     expect(existsSync(path!)).toBe(true);
   });
 
   test("clients keep talking to the one live daemon after a refused start", async () => {
+    // Arrange
     const first = server();
+
     first.start();
     const created = first.core.sessionCreate({ workspace: WS, artifact: PLAN });
+
     server().start(); // refused
     const client = await DaemonClient.connect({ home });
+
+    // Assert
     try {
       // no split brain: the session created on the winner is visible
       expect((await client.sessionGet(created.id)).id).toBe(created.id);
@@ -60,42 +71,92 @@ describe("one daemon per home", () => {
   });
 
   test("a stale lock from a crashed daemon is reclaimed", () => {
+    // Arrange
     // a pid that cannot be running (init is never a cueloop daemon, and this
     // simulates the record a crashed process leaves behind)
     writeFileSync(lockPath(home), "999999");
-    const s = server();
-    expect(s.start()).toBe(socketPath(home));
+    const daemonServer = server();
+
+    // Assert
+    expect(daemonServer.start()).toBe(socketPath(home));
+  });
+
+  test("an autostarted daemon reclaims a stale socket after taking the lock", async () => {
+    writeFileSync(socketPath(home), "stale socket");
+    const client = await DaemonClient.connect({ home, autostart: true });
+
+    try {
+      expect((await client.ping()).pid).toBeGreaterThan(0);
+    } finally {
+      await client.shutdown();
+      client.close();
+    }
   });
 
   test("stopping releases the lock so a restart works", () => {
+    // Arrange
     const first = server();
+
     first.start();
+
+    // Act
     first.stop();
+
+    // Assert
     expect(existsSync(lockPath(home))).toBe(false);
     const second = server();
+
     expect(second.start()).toBe(socketPath(home));
   });
 
   test("concurrent autostarts converge on a single daemon", async () => {
+    // Act
     // several clients race to autostart; all must end up on the same daemon
     const clients = await Promise.all([
       DaemonClient.connect({ home, autostart: true }),
       DaemonClient.connect({ home, autostart: true }),
       DaemonClient.connect({ home, autostart: true }),
     ]);
+
+    // Assert
     try {
-      const pids = await Promise.all(clients.map((c) => c.ping()));
-      const unique = new Set(pids.map((p) => p.pid));
+      const pids = await Promise.all(clients.map((client) => client.ping()));
+      const unique = new Set(pids.map((ping) => ping.pid));
+
       expect(unique.size).toBe(1);
       // and state is shared: one client's session is visible to the others
-      const s = await clients[0]!.sessionCreate(WS, PLAN);
-      expect((await clients[2]!.sessionGet(s.id)).id).toBe(s.id);
+      const session = await clients[0]!.sessionCreate(WS, PLAN);
+
+      expect((await clients[2]!.sessionGet(session.id)).id).toBe(session.id);
     } finally {
       for (const c of clients) c.close();
       // the autostarted daemon is not one of `servers` - shut it down
       const admin = await DaemonClient.connect({ home });
+
       await admin.shutdown();
       admin.close();
     }
   }, 60_000);
+
+  test("autostart joins a daemon that bound its socket after the first dial", async () => {
+    const running = server();
+    const path = running.start();
+    const created = running.core.sessionCreate({ workspace: WS, artifact: PLAN });
+    const client = new DaemonClient();
+    const previousTimeout = process.env.CUELOOP_START_TIMEOUT_MS;
+
+    client["home"] = home;
+    process.env.CUELOOP_START_TIMEOUT_MS = "250";
+
+    try {
+      await client["attachFreshDaemon"](home, path!);
+
+      expect((await client.sessionGet(created.id)).id).toBe(created.id);
+      expect(existsSync(path!)).toBe(true);
+    } finally {
+      client.close();
+      if (previousTimeout === undefined) delete process.env.CUELOOP_START_TIMEOUT_MS;
+      else process.env.CUELOOP_START_TIMEOUT_MS = previousTimeout;
+    }
+  });
 });

package/src/store.conformance.test.ts

@@ -0,0 +1,41 @@
+import { mkdtempSync, writeFileSync } from "node:fs";
+import * as v from "valibot";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { MemoryThreadStore, ThreadStore, type ThreadRepository } from "./store";
+import { sessionsDir } from "./paths";
+import { runThreadStoreConformance } from "./testing/store-conformance";
+
+const homes = new WeakMap<ThreadRepository, string>();
+
+runThreadStoreConformance("file store", {
+  open: (records) => {
+    const home = mkdtempSync(join(tmpdir(), "cueloop-store-"));
+    const store = new ThreadStore(home);
+
+    // records land on disk the way an earlier daemon would have written them;
+    // the file name comes from the record's id when it has one
+    records.forEach((record, index) => {
+      const named = v.safeParse(v.object({ id: v.string() }), record);
+      const id = named.success ? named.output.id : `bad_${index}`;
+
+      writeFileSync(join(sessionsDir(home), `${id}.json`), JSON.stringify(record));
+    });
+    homes.set(store, home);
+
+    return store;
+  },
+  restart: (store) => {
+    const reopened = new ThreadStore(homes.get(store)!);
+
+    homes.set(reopened, homes.get(store)!);
+
+    return reopened;
+  },
+});
+
+runThreadStoreConformance("memory store", {
+  open: (records) => new MemoryThreadStore(records),
+  // nothing persists past the instance: a restart recovers what it was seeded with
+  restart: (store) => new MemoryThreadStore(store.list()),
+});

package/src/store.jsonl.test.ts

@@ -0,0 +1,159 @@
+/**
+ * The JSONL thread store's own behaviors, beyond the shared repository contract: per-project bucket
+ * layout keyed by root commit, one-time migration of the old whole-record files, compaction of a
+ * long log, and a crash's torn trailing line being skipped on read.
+ */
+
+import { describe, expect, test } from "bun:test";
+import {
+  mkdtempSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  existsSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ThreadStore } from "./store";
+import { migratedSessionsDir, sessionsDir, threadBucket } from "./paths";
+import { createTestThreadRecord } from "./testing/store-conformance";
+
+function tmpHome(): string {
+  return mkdtempSync(join(tmpdir(), "cueloop-jsonl-"));
+}
+
+describe("JSONL thread store", () => {
+  test("a thread is written under a per-project bucket keyed by root commit", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        workspace: { repoRoot: "/repo", branch: "main", rootCommit: "abcdef0123456789" },
+      }),
+    );
+
+    const bucket = threadBucket("abcdef0123456789", home);
+    expect(readdirSync(bucket)).toContain("ses_a.jsonl");
+  });
+
+  test("a non-commit rootCommit (path traversal) cannot escape the threads directory", () => {
+    const home = tmpHome();
+    const escape = threadBucket("../../../etc/cueloop", home);
+    const standalone = threadBucket(undefined, home);
+
+    // a value that is not a real commit SHA is refused to the standalone bucket, never a parent path
+    expect(escape).toBe(standalone);
+    expect(escape.startsWith(join(home, "threads"))).toBe(true);
+  });
+
+  test("a repo-less thread lands in the standalone bucket", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_s", "2026-09-01T00:00:00.000Z"));
+
+    expect(readdirSync(threadBucket(undefined, home))).toContain("ses_s.jsonl");
+  });
+
+  test("boot migrates the old whole-record file and parks the original", () => {
+    const home = tmpHome();
+    mkdirSync(sessionsDir(home), { recursive: true });
+    const legacy = createTestThreadRecord("ses_old", "2026-09-01T00:00:00.000Z");
+    writeFileSync(join(sessionsDir(home), "ses_old.json"), JSON.stringify(legacy));
+
+    const store = new ThreadStore(home);
+    const report = store.recover();
+
+    expect(report.recovered).toEqual(["ses_old"]);
+    // the record migrates intact (a history is derived on read, as the old store did)
+    expect(store.get("ses_old")?.id).toBe("ses_old");
+    expect(store.get("ses_old")?.artifact).toEqual(legacy.artifact);
+    // the original is parked, not left in the live legacy dir
+    expect(existsSync(join(sessionsDir(home), "ses_old.json"))).toBe(false);
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_old.json");
+  });
+
+  test("migration never overwrites a newer JSONL with an older legacy snapshot", () => {
+    const home = tmpHome();
+    // a newer JSONL already holds this thread (e.g. written after a rollback)
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(
+      createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "newer" },
+      }),
+    );
+    // an older legacy JSON for the same thread is restored alongside it
+    mkdirSync(sessionsDir(home), { recursive: true });
+    writeFileSync(
+      join(sessionsDir(home), "ses_x.json"),
+      JSON.stringify(
+        createTestThreadRecord("ses_x", "2026-09-01T00:00:00.000Z", {
+          workspace: { repoRoot: "/repo", branch: "older" },
+        }),
+      ),
+    );
+
+    const reopened = new ThreadStore(home);
+    reopened.recover();
+    // the newer JSONL wins; the legacy snapshot is parked, not applied
+    expect(reopened.get("ses_x")?.workspace.branch).toBe("newer");
+    expect(readdirSync(migratedSessionsDir(home))).toContain("ses_x.json");
+  });
+
+  test("a long log compacts to a single line, and the record survives it", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    // more upserts than the compaction threshold
+    for (let iteration = 0; iteration < 60; iteration++) {
+      store.upsert(
+        createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+          status: "pending",
+          workspace: { repoRoot: "/repo", branch: `b${iteration}` },
+        }),
+      );
+    }
+
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+    const lines = readFileSync(file, "utf8")
+      .split("\n")
+      .filter((l) => l.trim());
+    expect(lines.length).toBeLessThan(60);
+    // the latest snapshot survives the compaction and a restart
+    const reopened = new ThreadStore(home);
+    reopened.recover();
+    expect(reopened.get("ses_a")?.workspace.branch).toBe("b59");
+  });
+
+  test("a crash's torn trailing line is skipped; the last valid snapshot wins", () => {
+    const home = tmpHome();
+    const store = new ThreadStore(home);
+    store.recover();
+    store.upsert(createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z"));
+
+    // simulate a crash mid-append: a half-written line at the end
+    const file = join(threadBucket(undefined, home), "ses_a.jsonl");
+    appendFileSync(file, '{"id":"ses_a","half');
+
+    const reopened = new ThreadStore(home);
+    const report = reopened.recover();
+    expect(report.recovered).toEqual(["ses_a"]);
+    expect(reopened.get("ses_a")?.id).toBe("ses_a");
+
+    // an update after a torn recovery must not fuse onto the torn fragment and vanish on restart
+    reopened.upsert(
+      createTestThreadRecord("ses_a", "2026-09-01T00:00:00.000Z", {
+        status: "pending",
+        workspace: { repoRoot: "/repo", branch: "after-tear" },
+      }),
+    );
+    const restarted = new ThreadStore(home);
+    restarted.recover();
+    expect(restarted.get("ses_a")?.workspace.branch).toBe("after-tear");
+  });
+});