@vellumai/assistant 0.12.2-staging.6 → 0.12.2-staging.7

package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts

@@ -0,0 +1,320 @@
+/**
+ * Tests for `buffer-file.ts`: the two writers of `memory/buffer.md`.
+ *
+ * The consume cases pin the invariant the module exists for: an entry saved
+ * at any point while a consolidation run is in flight is either the run's
+ * own material or still in the buffer afterwards, never destroyed. The
+ * cross-process case is the reason to believe it: a child process appends
+ * as fast as it can while this process consumes in a loop, and every tagged
+ * entry must end up consumed or present. The same harness run against a
+ * naive read-modify-write consume loses entries, which is what the
+ * mechanism is measured against.
+ */
+
+import {
+  appendFileSync,
+  chmodSync,
+  closeSync,
+  existsSync,
+  mkdtempSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+} from "bun:test";
+
+import {
+  appendBufferAndArchive,
+  consumeBufferEntries,
+} from "../buffer-file.js";
+import {
+  type BufferEntryLines,
+  bufferEntryText,
+  formatRememberEntry,
+  joinBufferEntries,
+  splitBufferContent,
+} from "../buffer-format.js";
+
+let tmp: string;
+beforeAll(() => {
+  tmp = mkdtempSync(join(tmpdir(), "memory-buffer-file-test-"));
+});
+afterAll(() => {
+  rmSync(tmp, { recursive: true, force: true });
+});
+
+let memoryDir: string;
+let bufferPath: string;
+beforeEach(() => {
+  memoryDir = mkdtempSync(join(tmp, "memory-"));
+  bufferPath = join(memoryDir, "buffer.md");
+});
+
+const A = "- [Apr 27, 9:00 AM] Alice prefers VS Code.";
+const B = "- [Apr 27, 9:01 AM] Bob shared a snippet:\n  line two\n  line three";
+const C = "- [Apr 27, 9:02 AM] Carol loves jazz.";
+const D = "- [Apr 27, 9:03 AM] Dave runs marathons.";
+
+function file(...entries: string[]): string {
+  return entries.map((e) => `${e}\n`).join("");
+}
+function entries(content: string): BufferEntryLines[] {
+  return splitBufferContent(content);
+}
+function texts(): string[] {
+  return entries(readFileSync(bufferPath, "utf-8")).map(bufferEntryText);
+}
+
+describe("appendBufferAndArchive", () => {
+  test("appends to the buffer and the dated archive, seeding the archive header once", () => {
+    const now = new Date(2026, 3, 27, 9, 0);
+    const entry = formatRememberEntry("Alice prefers VS Code.", now);
+    const first = appendBufferAndArchive({ rootDir: memoryDir, entry, now });
+    appendBufferAndArchive({ rootDir: memoryDir, entry, now });
+
+    expect(first.bufferPath).toBe(bufferPath);
+    expect(readFileSync(bufferPath, "utf-8")).toBe(entry + entry);
+    expect(first.archivePath).toBe(join(memoryDir, "archive", "2026-04-27.md"));
+    expect(readFileSync(first.archivePath, "utf-8")).toBe(
+      `# Apr 27, 2026\n\n${entry}${entry}`,
+    );
+  });
+});
+
+describe("consumeBufferEntries", () => {
+  test("removes exactly the consumed entries and keeps the rest verbatim and in order", async () => {
+    writeFileSync(bufferPath, file(A, B, C, D));
+    const pass = entries(file(A, B));
+
+    const result = await consumeBufferEntries(bufferPath, pass, {
+      lateAppendGraceMs: 0,
+    });
+
+    expect(result).toEqual({
+      removed: 2,
+      alreadyAbsent: 0,
+      lateAppendBytesRecovered: 0,
+      unrecoveredLateAppendBytes: 0,
+      lateAppendDrainFailed: false,
+    });
+    expect(readFileSync(bufferPath, "utf-8")).toBe(file(C, D));
+  });
+
+  test("entries appended after the snapshot survive, including one that duplicates a consumed entry", async () => {
+    // The snapshot held one copy of A; the same fact was remembered again
+    // during the run. Multiset semantics: one copy goes, one stays.
+    writeFileSync(bufferPath, file(A, B));
+    const pass = entries(file(A, B));
+    appendFileSync(bufferPath, file(C, A));
+
+    const result = await consumeBufferEntries(bufferPath, pass, {
+      lateAppendGraceMs: 0,
+    });
+
+    expect(result.removed).toBe(2);
+    expect(texts()).toEqual([C, A]);
+  });
+
+  test("a consumed entry missing from the live file is counted, not searched for", async () => {
+    writeFileSync(bufferPath, file(B, C));
+    const pass = entries(file(A, B));
+
+    const result = await consumeBufferEntries(bufferPath, pass, {
+      lateAppendGraceMs: 0,
+    });
+
+    expect(result).toMatchObject({ removed: 1, alreadyAbsent: 1 });
+    expect(texts()).toEqual([C]);
+  });
+
+  test("a missing buffer file consumes nothing and creates nothing", async () => {
+    const result = await consumeBufferEntries(bufferPath, entries(file(A)), {
+      lateAppendGraceMs: 0,
+    });
+
+    expect(result).toEqual({
+      removed: 0,
+      alreadyAbsent: 1,
+      lateAppendBytesRecovered: 0,
+      unrecoveredLateAppendBytes: 0,
+      lateAppendDrainFailed: false,
+    });
+    expect(existsSync(bufferPath)).toBe(false);
+  });
+
+  test("consuming everything leaves an empty file and no temp file behind", async () => {
+    writeFileSync(bufferPath, file(A, B));
+
+    await consumeBufferEntries(bufferPath, entries(file(A, B)), {
+      lateAppendGraceMs: 0,
+    });
+
+    expect(readFileSync(bufferPath, "utf-8")).toBe("");
+    expect(readdirSync(memoryDir)).toEqual(["buffer.md"]);
+  });
+
+  test("an append that landed on the replaced inode is drained back into the buffer", async () => {
+    // An appender that opened the file before the rename writes to the old
+    // inode. The consumer holds that inode open and copies the late bytes
+    // over. The test plays the appender: open before, write after the
+    // synchronous phase, before the grace-window drain.
+    writeFileSync(bufferPath, file(A, B));
+    const lateFd = openSync(bufferPath, "a");
+    const consuming = consumeBufferEntries(bufferPath, entries(file(A)), {
+      lateAppendGraceMs: 50,
+    });
+    // The synchronous phase (read, rewrite, rename) ran before the first
+    // await inside consume, so this write lands on the orphaned inode.
+    appendFileSync(lateFd, file(C));
+    closeSync(lateFd);
+
+    const result = await consuming;
+
+    expect(result.removed).toBe(1);
+    expect(result.lateAppendBytesRecovered).toBe(file(C).length);
+    expect(result.unrecoveredLateAppendBytes).toBe(0);
+    expect(texts()).toEqual([B, C]);
+  });
+
+  test("a late append the drain cannot copy back is reported, not thrown, and the pass stays consumed", async () => {
+    // The rename has committed when the drain runs, so a failing copy must
+    // not surface as "nothing happened". The test makes the rewritten
+    // buffer read-only before the late bytes land, so every copy attempt
+    // fails.
+    writeFileSync(bufferPath, file(A, B));
+    const lateFd = openSync(bufferPath, "a");
+    const consuming = consumeBufferEntries(bufferPath, entries(file(A)), {
+      lateAppendGraceMs: 50,
+    });
+    chmodSync(bufferPath, 0o444);
+    appendFileSync(lateFd, file(C));
+    closeSync(lateFd);
+
+    try {
+      const result = await consuming;
+      expect(result.removed).toBe(1);
+      expect(result.lateAppendBytesRecovered).toBe(0);
+      expect(result.unrecoveredLateAppendBytes).toBe(file(C).length);
+      expect(result.lateAppendDrainFailed).toBe(false);
+      expect(texts()).toEqual([B]);
+    } finally {
+      chmodSync(bufferPath, 0o644);
+    }
+  });
+});
+
+/**
+ * The cross-process invariant. A child appends `count` tagged entries as
+ * fast as it can through `appendFileSync`; this process consumes the leading
+ * entries (up to `passSize` at a time) in a loop through `consume`. Returns
+ * the tags that were neither consumed by a pass nor left in the buffer: the
+ * entries the consume destroyed.
+ */
+async function lostUnder(
+  consume: (path: string, pass: BufferEntryLines[]) => Promise<unknown>,
+  count: number,
+  passSize: number,
+): Promise<{ lost: string[]; duplicated: string[]; passes: number }> {
+  const tag = `t${Math.random().toString(36).slice(2, 8)}`;
+  writeFileSync(bufferPath, "");
+  const child = Bun.spawn(
+    [
+      process.execPath,
+      join(import.meta.dir, "fixtures", "buffer-appender.ts"),
+      bufferPath,
+      String(count),
+      tag,
+    ],
+    { stdout: "ignore", stderr: "inherit" },
+  );
+  const consumed: string[] = [];
+  let passes = 0;
+  let childDone = false;
+  void child.exited.then(() => {
+    childDone = true;
+  });
+  while (!childDone) {
+    const snapshot = entries(readFileSync(bufferPath, "utf-8"));
+    const pass = snapshot.slice(0, passSize);
+    if (pass.length === 0) {
+      await Bun.sleep(1);
+      continue;
+    }
+    await consume(bufferPath, pass);
+    consumed.push(...pass.map(bufferEntryText));
+    passes += 1;
+  }
+  await child.exited;
+  const seen = new Map<string, number>();
+  for (const text of [...consumed, ...texts()]) {
+    seen.set(text, (seen.get(text) ?? 0) + 1);
+  }
+  const lost: string[] = [];
+  const duplicated: string[] = [];
+  for (let i = 0; i < count; i++) {
+    const text = `- [Apr 27, 9:00 AM] ${tag}-${i}`;
+    const n = seen.get(text) ?? 0;
+    if (n === 0) {
+      lost.push(text);
+    } else if (n > 1) {
+      duplicated.push(text);
+    }
+  }
+  return { lost, duplicated, passes };
+}
+
+/** The read-modify-write the production consume replaces. */
+async function naiveConsume(
+  path: string,
+  pass: BufferEntryLines[],
+): Promise<void> {
+  const pending = pass.map(bufferEntryText);
+  const remaining = entries(readFileSync(path, "utf-8")).filter((entry) => {
+    const index = pending.indexOf(bufferEntryText(entry));
+    if (index === -1) {
+      return true;
+    }
+    pending.splice(index, 1);
+    return false;
+  });
+  writeFileSync(path, joinBufferEntries(remaining), "utf-8");
+}
+
+describe("consumeBufferEntries under a concurrent appender in another process", () => {
+  const APPENDS = 4000;
+
+  test("no entry appended during consumption is lost or duplicated", async () => {
+    // Runs the consume exactly as production does, grace window included:
+    // the window is what makes the invariant hold, and a loaded machine
+    // can deschedule the appender between its open and its write for
+    // longer than a token grace would cover. Larger passes keep the run
+    // short with the full grace paid per pass.
+    const outcome = await lostUnder(
+      (path, pass) => consumeBufferEntries(path, pass),
+      APPENDS,
+      500,
+    );
+    expect(outcome.passes).toBeGreaterThan(0);
+    expect(outcome.lost).toEqual([]);
+    expect(outcome.duplicated).toEqual([]);
+  }, 60_000);
+
+  test("the naive read-modify-write consume loses entries under the same load", async () => {
+    // Sensitivity check for the harness: the mechanism above is only
+    // evidence if this shape, run identically, fails.
+    const outcome = await lostUnder(naiveConsume, APPENDS, 500);
+    expect(outcome.passes).toBeGreaterThan(0);
+    expect(outcome.lost.length).toBeGreaterThan(0);
+  }, 60_000);
+});

package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts

@@ -10,10 +10,13 @@
 import { describe, expect, test } from "bun:test";
 
 import {
+  bufferEntryText,
   formatBufferTimestamp,
   formatRememberEntry,
   isBufferEntryStart,
+  joinBufferEntries,
   matchBufferEntryStart,
+  splitBufferContent,
   splitBufferEntries,
 } from "../buffer-format.js";
 
@@ -202,3 +205,43 @@ describe("splitBufferEntries", () => {
     expect(splitBufferEntries([])).toEqual([]);
   });
 });
+
+describe("splitBufferContent / joinBufferEntries", () => {
+  const content =
+    formatRememberEntry("Alice prefers dark mode", new Date(2026, 0, 1, 9, 0)) +
+    formatRememberEntry(
+      "Bob's plan:\n- [ ] step one\n\nend",
+      new Date(2026, 0, 1, 9, 1),
+    );
+
+  test("round-trips newline-terminated content byte for byte", () => {
+    expect(joinBufferEntries(splitBufferContent(content))).toBe(content);
+  });
+
+  test("an entry's text is the same whether or not it is last in the file", () => {
+    const [first] = splitBufferContent(content);
+    const [alone] = splitBufferContent(
+      formatRememberEntry(
+        "Alice prefers dark mode",
+        new Date(2026, 0, 1, 9, 0),
+      ),
+    );
+    expect(bufferEntryText(alone!)).toBe(bufferEntryText(first!));
+    expect(bufferEntryText(alone!)).toBe(
+      "- [Jan 1, 9:00 AM] Alice prefers dark mode",
+    );
+  });
+
+  test("empty content has no entries and joins back to empty", () => {
+    expect(splitBufferContent("")).toEqual([]);
+    expect(joinBufferEntries([])).toBe("");
+  });
+
+  test("content without a terminating newline keeps its last line intact", () => {
+    const entries = splitBufferContent("- [Jan 1, 9:00 AM] partial appe");
+    expect(entries).toHaveLength(1);
+    expect(bufferEntryText(entries[0]!)).toBe(
+      "- [Jan 1, 9:00 AM] partial appe",
+    );
+  });
+});

package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts

@@ -0,0 +1,17 @@
+// Child process for the buffer-file cross-process test: appends `count`
+// uniquely tagged entries to `bufferPath` as fast as it can, through the same
+// `appendFileSync` call the production appender uses. Hand-formats the entry
+// line so the fixture imports nothing from `src/`.
+//
+// usage: bun buffer-appender.ts <bufferPath> <count> <tag>
+
+import { appendFileSync } from "node:fs";
+
+const [bufferPath, countArg, tag] = process.argv.slice(2);
+if (!bufferPath || !countArg || !tag) {
+  throw new Error("usage: buffer-appender <bufferPath> <count> <tag>");
+}
+const count = Number.parseInt(countArg, 10);
+for (let i = 0; i < count; i++) {
+  appendFileSync(bufferPath, `- [Apr 27, 9:00 AM] ${tag}-${i}\n`, "utf-8");
+}

package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts

@@ -117,6 +117,17 @@ let loadedConversations: Record<string, { processing: boolean }> = {};
 // the stale-flag override's age check. Absent ids read as null (stampless).
 let processingStartedAtById: Record<string, number | null> = {};
 
+// The source conversation's recorded wire tool surface (what its last live
+// turn sent to the provider). `null` = no live turn has recorded one yet.
+let mockSourceToolSurface: Array<Record<string, unknown>> | null = null;
+let toolSurfaceReads: string[] = [];
+mock.module("../../../../persistence/conversation-tool-surface.js", () => ({
+  getConversationToolSurface: (conversationId: string) => {
+    toolSurfaceReads.push(conversationId);
+    return mockSourceToolSurface;
+  },
+}));
+
 const watchdogEvents: Array<{
   checkName: string;
   value?: number | null;
@@ -509,6 +520,8 @@ describe("memoryRetrospectiveJob", () => {
     resolveUserSlugCalls = [];
     mockV3TierActive = true;
     mockSkillImprovementActive = false;
+    mockSourceToolSurface = null;
+    toolSurfaceReads = [];
   });
 
   test("first-run happy path: no state row, no prior retrospective, both pointer fields set on success", async () => {
@@ -1842,6 +1855,39 @@ describe("memoryRetrospectiveJob", () => {
     });
   });
 
+  test("replays the source's recorded wire tool surface verbatim on the fork wake", async () => {
+    mockSourceToolSurface = [
+      { name: "bash", description: "Run", input_schema: {} },
+      { name: "remember", description: "Save", input_schema: {} },
+      { name: "bell_jingle", description: "Ring", input_schema: {} },
+      { type: "web_search_20250305", name: "web_search", max_uses: 5 },
+    ];
+
+    await memoryRetrospectiveJob(makeJob(), stubConfig);
+
+    expect(toolSurfaceReads).toEqual(["src-conv-1"]);
+    expect(wakeCalls).toHaveLength(1);
+    // The exact recorded array rides to the wake; the fork sends it in place
+    // of the surface it would resolve for itself.
+    expect(wakeCalls[0]!.opts.wireToolDefinitions).toEqual(
+      mockSourceToolSurface,
+    );
+    // The execution-side pin still rides alongside it.
+    expect(wakeCalls[0]!.opts.toolGateMode).toBe("execution");
+    expect(wakeCalls[0]!.opts.toolContextPin).toBeDefined();
+  });
+
+  test("no recorded source surface → no wireToolDefinitions; the pin alone shapes the wire", async () => {
+    mockSourceToolSurface = null;
+
+    await memoryRetrospectiveJob(makeJob(), stubConfig);
+
+    expect(toolSurfaceReads).toEqual(["src-conv-1"]);
+    expect(wakeCalls).toHaveLength(1);
+    expect("wireToolDefinitions" in wakeCalls[0]!.opts).toBe(false);
+    expect(wakeCalls[0]!.opts.toolContextPin).toBeDefined();
+  });
+
   test("execution mode is unconditional → toolContextPin rides even without a resolved profile", async () => {
     await memoryRetrospectiveJob(
       makeJob(),

package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Tests for `memory-run-evidence.ts`: the readers of what a background
+ * memory run durably produced, shared by the retrospective's cursor advance
+ * and consolidation's buffer consumption.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import {
+  collectSuccessfulToolResultIds,
+  countDurableToolUses,
+  endsWithTextReply,
+  extractRememberContents,
+  hasCommittedTextReply,
+  type MessageLike,
+} from "../memory-run-evidence.js";
+
+const DURABLE = new Set(["file_write", "remember"]);
+
+/**
+ * Persisted message rows carry their blocks as a JSON string; the readers
+ * parse it. Rows are built that way here so the test exercises the same
+ * shape the jobs read.
+ */
+function row(role: string, blocks: unknown[]): MessageLike {
+  return { role, content: JSON.stringify(blocks) };
+}
+const use = (id: string, name: string, input: unknown = {}) => ({
+  type: "tool_use",
+  id,
+  name,
+  input,
+});
+const result = (id: string, isError = false) => ({
+  type: "tool_result",
+  tool_use_id: id,
+  is_error: isError,
+  content: "…",
+});
+const text = (value: string) => ({ type: "text", text: value });
+
+describe("collectSuccessfulToolResultIds", () => {
+  test("collects non-error tool_result ids from user rows only, tolerating malformed rows", () => {
+    const ids = collectSuccessfulToolResultIds([
+      row("user", [result("ok"), result("bad", true)]),
+      row("assistant", [result("not-a-user-row")]),
+      { role: "user", content: "{not json" },
+      row("user", [result("second")]),
+    ]);
+    expect([...ids].sort()).toEqual(["ok", "second"]);
+  });
+});
+
+describe("countDurableToolUses", () => {
+  const messages = [
+    row("assistant", [
+      use("a", "file_write"),
+      use("b", "file_write"),
+      use("c", "file_read"),
+    ]),
+    row("user", [result("a"), result("b", true), result("c")]),
+  ];
+
+  test("counts only the named tools, and with a success set only verified executions", () => {
+    expect(countDurableToolUses(messages, DURABLE, null)).toBe(2);
+    expect(
+      countDurableToolUses(
+        messages,
+        DURABLE,
+        collectSuccessfulToolResultIds(messages),
+      ),
+    ).toBe(1);
+    expect(countDurableToolUses(messages, new Set(["file_read"]), null)).toBe(
+      1,
+    );
+  });
+});
+
+describe("hasCommittedTextReply", () => {
+  test("true only when the LAST assistant row carries non-blank text", () => {
+    expect(
+      hasCommittedTextReply([
+        row("assistant", [text("narrating")]),
+        row("assistant", [text("   ")]),
+      ]),
+    ).toBe(false);
+    expect(
+      hasCommittedTextReply([
+        row("assistant", [use("a", "file_write")]),
+        row("user", [result("a")]),
+        row("assistant", [text("Done.")]),
+      ]),
+    ).toBe(true);
+    expect(hasCommittedTextReply([{ role: "assistant", content: "{" }])).toBe(
+      false,
+    );
+  });
+});
+
+describe("endsWithTextReply", () => {
+  test("true only when the final row is an assistant reply with text and no tool call", () => {
+    expect(
+      endsWithTextReply([
+        row("assistant", [use("a", "file_write")]),
+        row("user", [result("a")]),
+        row("assistant", [text("Filed it.")]),
+      ]),
+    ).toBe(true);
+    // Narration on the row that also calls a tool, then a tool result: the
+    // run stopped mid-loop, whatever the narration said.
+    expect(
+      endsWithTextReply([
+        row("assistant", [
+          text("Fixing that page now."),
+          use("a", "file_write"),
+        ]),
+        row("user", [result("a")]),
+      ]),
+    ).toBe(false);
+    expect(
+      endsWithTextReply([
+        row("assistant", [text("Done."), use("b", "file_read")]),
+      ]),
+    ).toBe(false);
+    expect(endsWithTextReply([row("assistant", [text("   ")])])).toBe(false);
+    expect(endsWithTextReply([{ role: "assistant", content: "{" }])).toBe(
+      false,
+    );
+    expect(endsWithTextReply([])).toBe(false);
+  });
+});
+
+describe("extractRememberContents", () => {
+  const messages = [
+    row("assistant", [
+      use("a", "remember", { content: " one " }),
+      use("b", "remember", { content: ["two", "", "three"] }),
+      use("c", "remember", { content: "failed" }),
+      use("d", "file_write", { content: "not a remember" }),
+    ]),
+    row("user", [result("a"), result("b"), result("c", true)]),
+  ];
+
+  test("flattens batched facts, trims, and drops blanks", () => {
+    expect(extractRememberContents(messages)).toEqual([
+      "one",
+      "two",
+      "three",
+      "failed",
+    ]);
+  });
+
+  test("with a success set, a failed remember's facts are excluded", () => {
+    expect(
+      extractRememberContents(
+        messages,
+        collectSuccessfulToolResultIds(messages),
+      ),
+    ).toEqual(["one", "two", "three"]);
+  });
+});