@vellumai/assistant 0.10.7 → 0.10.8-dev.202607102228.5945895

package/src/persistence/__tests__/bulk-write-gate.test.ts

@@ -0,0 +1,159 @@
+import { rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+
+import { assertNotLiveDb } from "../../__tests__/assert-not-live-db.js";
+import { bulkWriteGateHolder, withBulkWriteGate } from "../bulk-write-gate.js";
+import { deleteConversationRowsInBatches } from "../conversation-row-batch-delete.js";
+import { copyForkMessagesViaSubprocess } from "../fork-message-copy.js";
+
+/** A promise whose resolution the test controls. */
+function deferred(): { promise: Promise<void>; release: () => void } {
+  let release!: () => void;
+  const promise = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  return { promise, release };
+}
+
+describe("withBulkWriteGate", () => {
+  test("runs callers FIFO, one at a time", async () => {
+    const events: string[] = [];
+    const first = deferred();
+
+    const run = (id: string, block?: Promise<void>) =>
+      withBulkWriteGate(id, async () => {
+        events.push(`${id}:start`);
+        if (block) {
+          await block;
+        }
+        events.push(`${id}:end`);
+      });
+
+    const p1 = run("a", first.promise);
+    const p2 = run("b");
+    const p3 = run("c");
+
+    await Bun.sleep(20);
+    expect(events).toEqual(["a:start"]);
+
+    first.release();
+    await Promise.all([p1, p2, p3]);
+    expect(events).toEqual([
+      "a:start",
+      "a:end",
+      "b:start",
+      "b:end",
+      "c:start",
+      "c:end",
+    ]);
+  });
+
+  test("releases on throw and propagates the error", async () => {
+    const failing = withBulkWriteGate("boom", async () => {
+      throw new Error("kaput");
+    });
+    await expect(failing).rejects.toThrow("kaput");
+
+    const after = await withBulkWriteGate("after", async () => 42);
+    expect(after).toBe(42);
+  });
+
+  test("returns the wrapped value and reports the holder label", async () => {
+    const holdersWhileRunning: Array<string | null> = [];
+    const value = await withBulkWriteGate("labelled", async () => {
+      holdersWhileRunning.push(bulkWriteGateHolder());
+      return "ok";
+    });
+    expect(value).toBe("ok");
+    expect(holdersWhileRunning).toEqual(["labelled"]);
+    expect(bulkWriteGateHolder()).toBeNull();
+  });
+});
+
+describe("gated bulk writers", () => {
+  test("fork message-copy queues behind a held gate", async () => {
+    const holder = deferred();
+    const holding = withBulkWriteGate("holder", () => holder.promise);
+
+    let copySettled = false;
+    const copy = copyForkMessagesViaSubprocess({
+      forkConversationId: "fork-1",
+      idPairs: [{ oldId: "old-1", newId: "new-1" }],
+      forceInProcess: true,
+    }).then((result) => {
+      copySettled = true;
+      return result;
+    });
+
+    await Bun.sleep(20);
+    expect(copySettled).toBe(false);
+
+    holder.release();
+    await holding;
+    await copy;
+    expect(copySettled).toBe(true);
+  });
+
+  test("empty fork message-copy skips the gate entirely", async () => {
+    const holder = deferred();
+    const holding = withBulkWriteGate("holder", () => holder.promise);
+
+    const result = await copyForkMessagesViaSubprocess({
+      forkConversationId: "fork-1",
+      idPairs: [],
+    });
+    expect(result.ok).toBe(true);
+
+    holder.release();
+    await holding;
+  });
+
+  test("main-DB batched delete queues behind a held gate", async () => {
+    const holder = deferred();
+    const holding = withBulkWriteGate("holder", () => holder.promise);
+
+    let deleteSettled = false;
+    const del = deleteConversationRowsInBatches({
+      conversationId: "conv-1",
+      table: "messages",
+      forceInProcess: true,
+    }).then((result) => {
+      deleteSettled = true;
+      return result;
+    });
+
+    await Bun.sleep(20);
+    expect(deleteSettled).toBe(false);
+
+    holder.release();
+    await holding;
+    await del;
+    expect(deleteSettled).toBe(true);
+  });
+
+  test("dedicated-file batched delete bypasses the gate", async () => {
+    const dbPath = join(tmpdir(), `bulk-write-gate-test-${Date.now()}.db`);
+    const holder = deferred();
+    const holding = withBulkWriteGate("holder", () => holder.promise);
+
+    try {
+      // Must settle while the gate is still held — a dedicated file has its
+      // own write lock, so its drain never queues behind main-DB streams.
+      const result = await deleteConversationRowsInBatches({
+        conversationId: "conv-1",
+        table: "some_table",
+        dbPath,
+        forceInProcess: true,
+      });
+      expect(bulkWriteGateHolder()).toBe("holder");
+      expect(result.backend).toBe("in-process-blocking");
+    } finally {
+      holder.release();
+      await holding;
+      assertNotLiveDb(dbPath);
+      rmSync(dbPath, { force: true });
+    }
+  });
+});

package/src/persistence/__tests__/conversation-queries-search.test.ts

@@ -16,13 +16,6 @@ import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
 
 import type { MessageLexicalSearchResult } from "../embeddings/messages-lexical-index.js";
 
-mock.module("../../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
 // The lexical candidate source is the single seam this cutover swaps in. Mock
 // it directly (returning caller-controlled candidate ids/scores) and, per test,
 // override it to throw to exercise the Qdrant-error degrade path. Spreading the

package/src/persistence/__tests__/immediate-transaction-guard.test.ts

@@ -0,0 +1,70 @@
+import { rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+import { expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { assertNotLiveDb } from "../../__tests__/assert-not-live-db.js";
+
+// Guard for the driver contract `forkConversationForRetrospective`'s phase-3
+// transaction relies on: drizzle's bun-sqlite driver must map
+// `db.transaction(cb, { behavior: "immediate" })` onto BEGIN IMMEDIATE, i.e.
+// acquire the write lock at BEGIN rather than at the first write. If a
+// drizzle upgrade silently dropped the behavior config, the phase-3
+// SQLITE_BUSY_SNAPSHOT fix would regress to a deferred read→write upgrade.
+test("bun-sqlite drizzle transaction honors behavior: 'immediate'", () => {
+  const dbPath = join(tmpdir(), `immediate-txn-guard-${Date.now()}.db`);
+  const writer = new Database(dbPath);
+  const probe = new Database(dbPath);
+  try {
+    writer.exec("PRAGMA journal_mode=WAL");
+    writer.exec("CREATE TABLE t (x INTEGER)");
+    // Fail immediately instead of waiting when the write lock is taken.
+    probe.exec("PRAGMA busy_timeout=0");
+    const probeDb = drizzle(probe);
+
+    writer.exec("BEGIN IMMEDIATE");
+    try {
+      // A DEFERRED empty transaction acquires no locks, so it succeeds even
+      // while the writer holds the write lock — the baseline that proves the
+      // failure below comes from the behavior config, not the environment.
+      const ran: string[] = [];
+      probeDb.transaction(() => {
+        ran.push("deferred");
+      });
+      expect(ran).toEqual(["deferred"]);
+
+      // An IMMEDIATE transaction must take the write lock at BEGIN and fail
+      // fast (busy_timeout=0) while the writer holds it.
+      expect(() =>
+        probeDb.transaction(
+          () => {
+            /* never reached: BEGIN IMMEDIATE itself must throw */
+          },
+          { behavior: "immediate" },
+        ),
+      ).toThrow(/SQLITE_BUSY|database is locked/i);
+    } finally {
+      writer.exec("COMMIT");
+    }
+
+    // With the write lock free, the immediate transaction runs normally.
+    const value = probeDb.transaction(
+      () => {
+        probe.exec("INSERT INTO t (x) VALUES (1)");
+        return "committed";
+      },
+      { behavior: "immediate" },
+    );
+    expect(value).toBe("committed");
+  } finally {
+    writer.close();
+    probe.close();
+    assertNotLiveDb(dbPath);
+    rmSync(dbPath, { force: true });
+    rmSync(`${dbPath}-wal`, { force: true });
+    rmSync(`${dbPath}-shm`, { force: true });
+  }
+});

package/src/persistence/__tests__/message-content-file.test.ts

@@ -0,0 +1,259 @@
+/**
+ * Tests for the file-backed message content module: the JSONL delta fold
+ * semantics, the append/fold roundtrip, the reserved-path ref discriminator,
+ * and the `resolveMessageContentBlocks` resolver across inline content,
+ * ref resolution, legacy plain strings, missing-file and workspace-escape
+ * fallbacks.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+
+import type { ContentBlock } from "../../providers/types.js";
+import { getWorkspaceDir } from "../../util/platform.js";
+import {
+  appendContentDeltas,
+  foldContentDeltas,
+  foldContentFile,
+  messageContentRefSchema,
+  resolveMessageContentBlocks,
+} from "../message-content-file.js";
+
+function textBlock(text: string): ContentBlock {
+  return { type: "text", text };
+}
+
+describe("foldContentDeltas", () => {
+  test("keeps the highest-seq line per index, ordered by index", () => {
+    const lines = [
+      { i: 1, seq: 1, block: textBlock("second, v1") },
+      { i: 0, seq: 2, block: textBlock("first, v1") },
+      { i: 1, seq: 3, block: textBlock("second, v2") },
+      { i: 0, seq: 0, block: textBlock("first, stale") },
+    ]
+      .map((l) => JSON.stringify(l))
+      .join("\n");
+    expect(foldContentDeltas(lines)).toEqual([
+      textBlock("first, v1"),
+      textBlock("second, v2"),
+    ]);
+  });
+
+  test("skips malformed and crash-truncated lines", () => {
+    const text =
+      JSON.stringify({ i: 0, seq: 1, block: textBlock("kept") }) +
+      "\n" +
+      "not json\n" +
+      JSON.stringify({ seq: 2, block: textBlock("missing index") }) +
+      "\n" +
+      JSON.stringify({ i: 2, seq: 4, block: { typo: "no type field" } }) +
+      "\n" +
+      '{"i": 1, "seq": 3, "block": {"type": "text", "te'; // truncated
+    expect(foldContentDeltas(text)).toEqual([textBlock("kept")]);
+  });
+
+  test("preserves internal rider fields on blocks (passthrough, not strip)", () => {
+    // Timing stamps like _startedAt ride inside persisted thinking blocks;
+    // the fold must round-trip them, not strip them as unknown keys.
+    const stamped = {
+      type: "thinking",
+      thinking: "hmm",
+      signature: "sig",
+      _startedAt: 123,
+    };
+    const text = JSON.stringify({ i: 0, seq: 1, block: stamped });
+    expect(foldContentDeltas(text)).toEqual([stamped as ContentBlock]);
+  });
+
+  test("empty input folds to empty content", () => {
+    expect(foldContentDeltas("")).toEqual([]);
+  });
+});
+
+describe("appendContentDeltas + foldContentFile", () => {
+  test("roundtrips across multiple appends, creating parent dirs", () => {
+    const path = join(
+      getWorkspaceDir(),
+      "conversations",
+      "test-roundtrip",
+      "inflight",
+      "msg-1.jsonl",
+    );
+    appendContentDeltas(path, [{ i: 0, seq: 1, block: textBlock("hel") }]);
+    appendContentDeltas(path, [
+      { i: 0, seq: 2, block: textBlock("hello") },
+      { i: 1, seq: 3, block: textBlock("world") },
+    ]);
+    expect(foldContentFile(path)).toEqual([
+      textBlock("hello"),
+      textBlock("world"),
+    ]);
+  });
+
+  test("returns null for a missing file", () => {
+    expect(
+      foldContentFile(join(getWorkspaceDir(), "does-not-exist.jsonl")),
+    ).toBeNull();
+  });
+});
+
+describe("messageContentRefSchema", () => {
+  test("accepts exactly { ref } naming a reserved conversations/ .jsonl path", () => {
+    expect(
+      messageContentRefSchema.safeParse({
+        ref: "conversations/a/inflight/b.jsonl",
+      }).success,
+    ).toBe(true);
+  });
+
+  test("rejects non-reserved paths — the legacy-text discriminator", () => {
+    for (const ref of [
+      "a/b.jsonl",
+      "conversations/a/b.txt",
+      "/etc/passwd",
+      "../../../etc/passwd",
+    ]) {
+      expect(messageContentRefSchema.safeParse({ ref }).success).toBe(false);
+    }
+  });
+
+  test("strips unrecognized extra keys rather than rejecting them", () => {
+    const result = messageContentRefSchema.safeParse({
+      ref: "conversations/a/b.jsonl",
+      other: 1,
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data).toEqual({ ref: "conversations/a/b.jsonl" });
+    }
+  });
+
+  test("rejects arrays, empty and non-string refs", () => {
+    for (const value of [[textBlock("x")], { ref: "" }, { ref: 42 }, null]) {
+      expect(messageContentRefSchema.safeParse(value).success).toBe(false);
+    }
+  });
+});
+
+function writeRefFixture(dir: string, file: string): string {
+  const rel = join("conversations", dir, "inflight", file);
+  mkdirSync(join(getWorkspaceDir(), "conversations", dir, "inflight"), {
+    recursive: true,
+  });
+  writeFileSync(
+    join(getWorkspaceDir(), rel),
+    [
+      JSON.stringify({ i: 0, seq: 1, block: textBlock("partial") }),
+      JSON.stringify({ i: 0, seq: 2, block: textBlock("final") }),
+    ].join("\n") + "\n",
+  );
+  return rel;
+}
+
+describe("resolveMessageContentBlocks", () => {
+  test("parses inline ContentBlock[] JSON to blocks", () => {
+    expect(
+      resolveMessageContentBlocks(JSON.stringify([textBlock("hello")])),
+    ).toEqual([textBlock("hello")]);
+  });
+
+  test("parses a nested tool_result with contentBlocks (recursive schema)", () => {
+    const blocks: ContentBlock[] = [
+      {
+        type: "tool_result",
+        tool_use_id: "tu-1",
+        content: "done",
+        contentBlocks: [textBlock("nested")],
+      },
+    ];
+    expect(resolveMessageContentBlocks(JSON.stringify(blocks))).toEqual(blocks);
+  });
+
+  test("passes typed blocks outside the union through untouched", () => {
+    // Persisted kinds like ui_surface live outside the provider union;
+    // their renderers own their shape, so repair must not rewrite them.
+    const historical = [
+      textBlock("kept as-is"),
+      { type: "ui_surface", surfaceType: "call_summary", data: { x: 1 } },
+      { type: "some_retired_block_kind", payload: 1 },
+    ];
+    expect(resolveMessageContentBlocks(JSON.stringify(historical))).toEqual(
+      historical as ContentBlock[],
+    );
+  });
+
+  test("wraps type-less values in a text block carrying the payload", () => {
+    const historical = [{ payload: 1 }, "bare string"];
+    expect(resolveMessageContentBlocks(JSON.stringify(historical))).toEqual([
+      textBlock('{"payload":1}'),
+      textBlock('"bare string"'),
+    ]);
+  });
+
+  test("repairs a web_search_tool_result with a missing tool_use_id", () => {
+    const historical = [
+      { type: "web_search_tool_result", content: { encrypted: "blob" } },
+    ];
+    expect(resolveMessageContentBlocks(JSON.stringify(historical))).toEqual([
+      {
+        type: "web_search_tool_result",
+        tool_use_id: "",
+        content: { encrypted: "blob" },
+      },
+    ]);
+  });
+
+  test("repairs a text block with missing or non-string text", () => {
+    const historical = [{ type: "text" }, { type: "text", text: 42 }];
+    expect(resolveMessageContentBlocks(JSON.stringify(historical))).toEqual([
+      textBlock(""),
+      textBlock("42"),
+    ]);
+  });
+
+  test("repairs a tool_result with object-valued content in place", () => {
+    const historical = [
+      { type: "tool_result", tool_use_id: "tu-1", content: { some: "obj" } },
+    ];
+    expect(resolveMessageContentBlocks(JSON.stringify(historical))).toEqual([
+      {
+        type: "tool_result",
+        tool_use_id: "tu-1",
+        content: '{"some":"obj"}',
+      },
+    ]);
+  });
+
+  test("folds a reserved-path ref to blocks", () => {
+    const rel = writeRefFixture("test-resolve-blocks", "m.jsonl");
+    expect(resolveMessageContentBlocks(JSON.stringify({ ref: rel }))).toEqual([
+      textBlock("final"),
+    ]);
+  });
+
+  test("wraps legacy plain strings in a single text block", () => {
+    expect(resolveMessageContentBlocks("plain old text")).toEqual([
+      textBlock("plain old text"),
+    ]);
+    const legacyRefLikeText = JSON.stringify({ ref: "missing.jsonl" });
+    expect(resolveMessageContentBlocks(legacyRefLikeText)).toEqual([
+      textBlock(legacyRefLikeText),
+    ]);
+  });
+
+  test("resolves a missing ref file to empty blocks", () => {
+    expect(
+      resolveMessageContentBlocks(
+        JSON.stringify({ ref: "conversations/gone/inflight/m.jsonl" }),
+      ),
+    ).toEqual([]);
+  });
+
+  test("resolves a reserved-prefix ref that escapes the workspace to empty blocks", () => {
+    expect(
+      resolveMessageContentBlocks(
+        JSON.stringify({ ref: "conversations/../../../etc/evil.jsonl" }),
+      ),
+    ).toEqual([]);
+  });
+});

package/src/persistence/__tests__/slow-query-log.test.ts

@@ -1,6 +1,8 @@
 import { Database } from "bun:sqlite";
 import { describe, expect, test } from "bun:test";
 
+import { runWithSqliteQueryLabel } from "../../util/sqlite-query-label.js";
+import { withSqliteRetry } from "../../util/sqlite-retry.js";
 import {
   callerFromStack,
   SLOW_QUERY_THRESHOLD_MS,
@@ -211,6 +213,99 @@ describe("slow-query-log", () => {
     const db = new Database(":memory:");
     expect(wrapSqliteForSlowQueryLogging(db)).toBe(db);
   });
+
+  test("an ambient label tags the event and keeps the derived caller", () => {
+    const events: SlowQueryEvent[] = [];
+    const db = new Database(":memory:");
+    db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)");
+    wrapSqliteForSlowQueryLogging(db, {
+      thresholdMs: 100,
+      now: fakeClock(0, 500),
+      onSlowQuery: (e) => events.push(e),
+    });
+
+    runWithSqliteQueryLabel("memory:upsertRetrospectiveState", () => {
+      db.query("SELECT * FROM t").all();
+    });
+
+    expect(events).toHaveLength(1);
+    expect(events[0].label).toBe("memory:upsertRetrospectiveState");
+    // Unlike an explicit `.label()`, an ambient label marks an outer boundary,
+    // so the statement's own call site is still derived from the stack.
+    expect(events[0].caller).toContain("slow-query-log.test.ts");
+  });
+
+  test("an explicit .label() wins over the ambient label", () => {
+    const events: SlowQueryEvent[] = [];
+    const db = new Database(":memory:");
+    db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)");
+    wrapSqliteForSlowQueryLogging(db, {
+      thresholdMs: 100,
+      now: fakeClock(0, 500),
+      onSlowQuery: (e) => events.push(e),
+    });
+
+    runWithSqliteQueryLabel("ambient:outer", () => {
+      db.label("schedule::claimDue")
+        .query("INSERT INTO t (v) VALUES (?)")
+        .run("a");
+    });
+
+    expect(events).toHaveLength(1);
+    expect(events[0].label).toBe("schedule::claimDue");
+    expect(events[0].caller).toBeUndefined();
+  });
+
+  test("withSqliteRetry's op labels slow queries inside it, retries included", async () => {
+    const events: SlowQueryEvent[] = [];
+    const db = new Database(":memory:");
+    db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)");
+    wrapSqliteForSlowQueryLogging(db, {
+      thresholdMs: 100,
+      now: fakeClock(0, 500),
+      onSlowQuery: (e) => events.push(e),
+    });
+
+    // First attempt loses the write-lock race; the retry (which runs from a
+    // stack truncated by the backoff sleep) issues the slow query.
+    let attempt = 0;
+    await withSqliteRetry(
+      () => {
+        if (attempt++ === 0) {
+          throw Object.assign(new Error("busy"), { code: "SQLITE_BUSY" });
+        }
+        return db.query("SELECT * FROM t").all();
+      },
+      { op: "test:claimRow", baseDelayMs: 1 },
+    );
+
+    expect(events).toHaveLength(1);
+    expect(events[0].label).toBe("test:claimRow");
+  });
+
+  test("withSqliteRetry's op labels a lazy thenable that runs when awaited", async () => {
+    const events: SlowQueryEvent[] = [];
+    const db = new Database(":memory:");
+    db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY)");
+    wrapSqliteForSlowQueryLogging(db, {
+      thresholdMs: 100,
+      now: fakeClock(0, 500),
+      onSlowQuery: (e) => events.push(e),
+    });
+
+    // Mirrors a Drizzle QueryPromise: fn returns immediately and the statement
+    // only executes when the thenable is assimilated by an await. The retry
+    // helper must perform that await inside its label scope.
+    const lazy = {
+      then(resolve: (value: unknown) => void) {
+        resolve(db.query("SELECT * FROM t").all());
+      },
+    } as unknown as Promise<unknown>;
+    await withSqliteRetry(() => lazy, { op: "test:lazyUpdate" });
+
+    expect(events).toHaveLength(1);
+    expect(events[0].label).toBe("test:lazyUpdate");
+  });
 });
 
 describe("callerFromStack", () => {
@@ -254,6 +349,36 @@ describe("callerFromStack", () => {
     );
   });
 
+  test("prefers the nearest named app frame over an anonymous callback", () => {
+    // A statement run inside `db.transaction((tx) => …)` sits in an anonymous
+    // app frame, with the accountable named function above the ORM plumbing.
+    const stack = [
+      "Error",
+      "    at emit (/repo/assistant/src/persistence/slow-query-log.ts:192:5)",
+      "    at <anonymous> (/repo/assistant/src/persistence/conversation-crud.ts:629:44)",
+      "    at <anonymous> (/root/.bun/install/cache/drizzle-orm@0.45.2@@@1/bun-sqlite/session.js:35:16)",
+      "    at transaction (bun:sqlite:416:27)",
+      "    at transaction (/root/.bun/install/cache/drizzle-orm@0.45.2@@@1/bun-sqlite/session.js:37:33)",
+      "    at insertMessageCore (/repo/assistant/src/persistence/conversation-crud.ts:474:10)",
+    ].join("\n");
+    expect(callerFromStack(stack)).toBe(
+      "persistence/conversation-crud.ts:insertMessageCore:474",
+    );
+  });
+
+  test("falls back to the innermost anonymous app frame when nothing is named", () => {
+    const stack = [
+      "Error",
+      "    at emit (/repo/assistant/src/persistence/slow-query-log.ts:192:5)",
+      "    at <anonymous> (/repo/assistant/src/persistence/conversation-crud.ts:629:44)",
+      "    at <anonymous> (/repo/assistant/src/daemon/lifecycle.ts:88:12)",
+      "    at processTicksAndRejections (native:7:39)",
+    ].join("\n");
+    expect(callerFromStack(stack)).toBe(
+      "persistence/conversation-crud.ts:<anonymous>:629",
+    );
+  });
+
   test("returns undefined when no application frame is present", () => {
     const stack = [
       "Error",