@vellumai/assistant 0.8.0 → 0.8.2

package/src/memory/v2/__tests__/prompts-router.test.ts

@@ -0,0 +1,309 @@
+/**
+ * Tests for `assistant/src/memory/v2/prompts/router.ts` —
+ * `renderRouterPrompt` (placeholder substitution in the bundled body) and
+ * `resolveRouterPrompt` (file-override path with fallback).
+ */
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { renderRouterPrompt, resolveRouterPrompt } from "../prompts/router.js";
+
+const SAMPLE_INDEX = `[1] morning-routine — coffee, walk, journal (edges: 2)
+[2] journal-style — terse, dated, no fluff (edges: 1)
+[3] taxes-2025 — Q1 estimate due April 15 (edges: )`;
+
+describe("renderRouterPrompt — substitution", () => {
+  test("replaces all three placeholders with the supplied values", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).not.toContain("{{ASSISTANT_NAME}}");
+    expect(out).not.toContain("{{USER_NAME}}");
+    expect(out).not.toContain("{{PAGE_INDEX}}");
+    expect(out).toContain("Aria");
+    expect(out).toContain("Alice");
+    expect(out).toContain(SAMPLE_INDEX);
+  });
+
+  test("substitutes every occurrence of the assistant name placeholder", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    // Body references the assistant name in multiple sentences; ensure none
+    // of them leak the raw placeholder.
+    const matches = out.match(/Aria/g) ?? [];
+    expect(matches.length).toBeGreaterThanOrEqual(2);
+  });
+});
+
+describe("renderRouterPrompt — neutral fallbacks", () => {
+  test("falls back to 'the assistant' when assistantName is null", () => {
+    const out = renderRouterPrompt({
+      assistantName: null,
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("the assistant");
+    expect(out).not.toContain("{{ASSISTANT_NAME}}");
+  });
+
+  test("falls back to 'the user' when userName is null", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: null,
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("the user");
+    expect(out).not.toContain("{{USER_NAME}}");
+  });
+
+  test("uses both fallbacks when both names are null", () => {
+    const out = renderRouterPrompt({
+      assistantName: null,
+      userName: null,
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("the assistant");
+    expect(out).toContain("the user");
+  });
+
+  test("falls back when names are whitespace-only strings", () => {
+    const out = renderRouterPrompt({
+      assistantName: "   ",
+      userName: "\t\n",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("the assistant");
+    expect(out).toContain("the user");
+  });
+});
+
+describe("renderRouterPrompt — page index handling", () => {
+  test("substitutes an empty pageIndexBlock cleanly without double-newline artifacts", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: "",
+    });
+
+    expect(out).not.toContain("{{PAGE_INDEX}}");
+    // The header should still be present and not followed by a stray
+    // triple-newline run from collapsing the empty block.
+    expect(out).toContain("# Concept Page Index");
+    expect(out).not.toMatch(/\n\n\n/);
+    // Output should end at the header section without trailing whitespace.
+    expect(out.endsWith("# Concept Page Index\n\n")).toBe(true);
+  });
+
+  test("preserves the page index body verbatim, including edges syntax", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain(
+      "[1] morning-routine — coffee, walk, journal (edges: 2)",
+    );
+    expect(out).toContain(
+      "[3] taxes-2025 — Q1 estimate due April 15 (edges: )",
+    );
+  });
+});
+
+describe("renderRouterPrompt — replacement-pattern specials", () => {
+  // String.prototype.replaceAll interprets `$&`, `$'`, `` $` ``, `$$`, and
+  // `$n` in the replacement string as backreferences. LLM-generated page
+  // index content can contain literal `$` runs, so the substituter must
+  // pass values through unchanged.
+  const SPECIALS = "$& and $' and $` and $$ and $1";
+
+  test.each([
+    [
+      "pageIndexBlock",
+      { assistantName: "Aria", userName: "Alice", pageIndexBlock: SPECIALS },
+    ],
+    [
+      "assistantName",
+      {
+        assistantName: SPECIALS,
+        userName: "Alice",
+        pageIndexBlock: SAMPLE_INDEX,
+      },
+    ],
+    [
+      "userName",
+      {
+        assistantName: "Aria",
+        userName: SPECIALS,
+        pageIndexBlock: SAMPLE_INDEX,
+      },
+    ],
+  ])("renders %s with $ specials verbatim", (_, opts) => {
+    expect(renderRouterPrompt(opts)).toContain(SPECIALS);
+  });
+});
+
+describe("renderRouterPrompt — determinism & snapshot stability", () => {
+  test("returns the same string for the same inputs", () => {
+    const opts = {
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    };
+    expect(renderRouterPrompt(opts)).toBe(renderRouterPrompt(opts));
+  });
+
+  test("snapshot of fixed inputs", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toMatchSnapshot();
+  });
+});
+
+describe("renderRouterPrompt — content expectations", () => {
+  test("references the select_pages_to_inject tool name", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("select_pages_to_inject");
+  });
+
+  test("describes the already_injected_ids and now markers", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out).toContain("<already_injected_ids>");
+    expect(out).toContain("<now>");
+  });
+
+  test("biases toward inclusion when in doubt", () => {
+    const out = renderRouterPrompt({
+      assistantName: "Aria",
+      userName: "Alice",
+      pageIndexBlock: SAMPLE_INDEX,
+    });
+
+    expect(out.toLowerCase()).toContain("lean toward inclusion");
+    expect(out.toLowerCase()).toContain("missing a relevant page");
+  });
+});
+
+describe("resolveRouterPrompt — override path", () => {
+  let tmpDir: string;
+
+  beforeEach(() => {
+    tmpDir = mkdtempSync(join(tmpdir(), "vellum-router-prompt-"));
+  });
+
+  afterEach(() => {
+    rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  const STD_OPTS = {
+    assistantName: "Aria",
+    userName: "Alice",
+    pageIndexBlock: SAMPLE_INDEX,
+  };
+
+  test("null overridePath returns the bundled prompt verbatim", () => {
+    expect(resolveRouterPrompt(null, tmpDir, STD_OPTS)).toEqual(
+      renderRouterPrompt(STD_OPTS),
+    );
+  });
+
+  test("loads override and substitutes placeholders", () => {
+    const overridePath = join(tmpDir, "router.md");
+    writeFileSync(
+      overridePath,
+      "Hi {{ASSISTANT_NAME}}, you are routing for {{USER_NAME}}.\n\n{{PAGE_INDEX}}",
+      "utf-8",
+    );
+
+    const out = resolveRouterPrompt(overridePath, tmpDir, STD_OPTS);
+    expect(out).toContain("Hi Aria, you are routing for Alice.");
+    expect(out).toContain(SAMPLE_INDEX);
+    expect(out).not.toContain("{{ASSISTANT_NAME}}");
+    expect(out).not.toContain("{{PAGE_INDEX}}");
+  });
+
+  test("relative override path is resolved under the passed workspaceDir, not the default workspace", () => {
+    // Write the override into the per-test temp dir, which acts as a
+    // non-default workspace. The configured path is purely relative so the
+    // loader must resolve it against the supplied workspaceDir — if it
+    // resolved against the process-wide default workspace instead, the file
+    // wouldn't be found and the bundled prompt would be returned.
+    const relativeName = "router-override.md";
+    writeFileSync(
+      join(tmpDir, relativeName),
+      "Routed via {{ASSISTANT_NAME}} for {{USER_NAME}} :: {{PAGE_INDEX}}",
+      "utf-8",
+    );
+
+    const out = resolveRouterPrompt(relativeName, tmpDir, STD_OPTS);
+    expect(out).toContain("Routed via Aria for Alice");
+    expect(out).toContain(SAMPLE_INDEX);
+    expect(out).not.toEqual(renderRouterPrompt(STD_OPTS));
+  });
+
+  test("missing override file falls back to bundled prompt", () => {
+    const overridePath = join(tmpDir, "does-not-exist.md");
+    expect(resolveRouterPrompt(overridePath, tmpDir, STD_OPTS)).toEqual(
+      renderRouterPrompt(STD_OPTS),
+    );
+  });
+
+  test("empty override file falls back to bundled prompt", () => {
+    const overridePath = join(tmpDir, "empty.md");
+    writeFileSync(overridePath, "   \n\t\n", "utf-8");
+    expect(resolveRouterPrompt(overridePath, tmpDir, STD_OPTS)).toEqual(
+      renderRouterPrompt(STD_OPTS),
+    );
+  });
+
+  test("override that is a directory falls back to bundled prompt", () => {
+    // Pass the temp directory itself as the override path — lstat sees a
+    // directory, not a regular file, so the loader bails to bundled.
+    expect(resolveRouterPrompt(tmpDir, tmpDir, STD_OPTS)).toEqual(
+      renderRouterPrompt(STD_OPTS),
+    );
+  });
+
+  test("override applies neutral fallbacks for missing names", () => {
+    const overridePath = join(tmpDir, "neutral.md");
+    writeFileSync(
+      overridePath,
+      "Hi {{ASSISTANT_NAME}}, routing for {{USER_NAME}}.",
+      "utf-8",
+    );
+
+    const out = resolveRouterPrompt(overridePath, tmpDir, {
+      assistantName: null,
+      userName: null,
+      pageIndexBlock: "",
+    });
+    expect(out).toBe("Hi the assistant, routing for the user.");
+  });
+});

package/src/memory/v2/__tests__/qdrant.test.ts

@@ -1,11 +1,30 @@
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
 import { makeMockLogger } from "../../../__tests__/helpers/mock-logger.js";
 
+// Per-suite tmp data dir so the reembed sentinel never lands in the
+// developer's real ~/.vellum workspace.
+const TEST_DATA_DIR = mkdtempSync(join(tmpdir(), "memory-v2-qdrant-test-"));
+const REEMBED_SENTINEL_PATH = join(
+  TEST_DATA_DIR,
+  ".memory-v2-reembed-required",
+);
+
 mock.module("../../../util/logger.js", () => ({
   getLogger: () => makeMockLogger(),
 }));
 
+mock.module("../../../util/platform.js", () => ({
+  getDataDir: () => TEST_DATA_DIR,
+  // Bun shares mocked modules across test files; some peer tests import
+  // `getWorkspaceDir` from this same module, so re-export it here to avoid
+  // an `undefined` if this mock is the one that wins evaluation order.
+  getWorkspaceDir: () => TEST_DATA_DIR,
+}));
+
 // Stub getConfig — only the qdrant.url / vectorSize / onDisk fields matter.
 mock.module("../../../config/loader.js", () => ({
   getConfig: () => ({
@@ -102,6 +121,9 @@ const state = {
   // Throw queue for upsert: first call shifts and throws if non-null;
   // subsequent calls succeed once the queue is exhausted.
   upsertThrowQueue: [] as Array<Error | null>,
+  // Throw queue for createPayloadIndex: each entry maps to the next call,
+  // so tests can simulate index-creation failures (strict-mode, network).
+  createIndexThrowQueue: [] as Array<Error | null>,
 };
 
 class MockQdrantClient {
@@ -138,6 +160,10 @@ class MockQdrantClient {
     params: { field_name: string; field_schema: string },
   ) {
     state.createIndexCalls.push(params);
+    if (state.createIndexThrowQueue.length > 0) {
+      const next = state.createIndexThrowQueue.shift();
+      if (next) throw next;
+    }
     return {};
   }
   async upsert(_name: string, params: { wait: boolean; points: MockPoint[] }) {
@@ -184,6 +210,7 @@ const {
   deleteConceptPageEmbedding,
   hybridQueryConceptPages,
   countConceptPagePoints,
+  clearReembedSentinel,
   MEMORY_V2_COLLECTION,
   _resetMemoryV2QdrantForTests,
 } = await import("../qdrant.js");
@@ -208,7 +235,13 @@ function resetState(): void {
   state.countThrows = null;
   state.countCalls = 0;
   state.upsertThrowQueue.length = 0;
+  state.createIndexThrowQueue.length = 0;
   _resetMemoryV2QdrantForTests();
+  // Drop any sentinel a prior test left behind so the no-drift default path
+  // doesn't accidentally report `migrated: true`.
+  if (existsSync(REEMBED_SENTINEL_PATH)) {
+    rmSync(REEMBED_SENTINEL_PATH);
+  }
 }
 
 describe("memory v2 qdrant — collection lifecycle", () => {
@@ -253,9 +286,10 @@ describe("memory v2 qdrant — collection lifecycle", () => {
     });
     expect(params.on_disk_payload).toBe(true);
 
-    // Slug payload index is created up front.
+    // Slug + kind payload indexes are created up front.
     expect(state.createIndexCalls).toEqual([
       { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
     ]);
   });
 
@@ -270,9 +304,14 @@ describe("memory v2 qdrant — collection lifecycle", () => {
     await ensureConceptPageCollection();
 
     // Existence check fired exactly once thanks to the in-memory readiness
-    // cache; createCollection / createPayloadIndex never ran.
+    // cache; createCollection never ran. Payload indexes are (idempotently)
+    // ensured on the existing-collection path to backfill long-lived installs
+    // that predate the `kind` index.
     expect(state.createCollectionCalls).toBe(0);
-    expect(state.createIndexCalls).toEqual([]);
+    expect(state.createIndexCalls).toEqual([
+      { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
+    ]);
     expect(state.collectionExistsCalls).toBe(1);
   });
 
@@ -289,6 +328,7 @@ describe("memory v2 qdrant — collection lifecycle", () => {
     expect(state.createCollectionCalls).toBe(1);
     expect(state.createIndexCalls).toEqual([
       { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
     ]);
   });
 
@@ -361,6 +401,101 @@ describe("memory v2 qdrant — collection lifecycle", () => {
     expect(result).toEqual({ migrated: false });
   });
 
+  test("preserves the reembed signal across calls when createCollection fails after delete", async () => {
+    // Pre-#29823 schema triggers the destructive recreate path.
+    state.collectionExistsBeforeCreate = true;
+    state.getCollectionInfo = {
+      config: {
+        params: {
+          vectors: { dense: { size: 384 } },
+          sparse_vectors: { sparse: {} },
+        },
+      },
+    };
+    state.createCollectionThrows = new Error("Qdrant transient failure");
+
+    let firstError: unknown = null;
+    try {
+      await ensureConceptPageCollection();
+    } catch (err) {
+      firstError = err;
+    }
+    expect(firstError).not.toBeNull();
+    // The sentinel must outlive the failed call so the retry knows data was lost.
+    expect(existsSync(REEMBED_SENTINEL_PATH)).toBe(true);
+
+    // Simulate a follow-up call after the transient failure clears. The
+    // collection no longer exists (delete succeeded earlier) so the ensure
+    // path falls through to createCollection without re-entering the drift
+    // branch — but the sentinel must still surface as `migrated: true` so
+    // the lifecycle hook enqueues reembed.
+    state.createCollectionThrows = null;
+    _resetMemoryV2QdrantForTests();
+    const result = await ensureConceptPageCollection();
+
+    expect(result).toEqual({ migrated: true });
+
+    // Lifecycle hook clears the sentinel after enqueueing the reembed job.
+    await clearReembedSentinel();
+    expect(existsSync(REEMBED_SENTINEL_PATH)).toBe(false);
+  });
+
+  test("clearReembedSentinel is a no-op when no sentinel exists", async () => {
+    // Idempotent: missing-file does not throw, so the lifecycle hook can
+    // call it unconditionally without guarding.
+    expect(existsSync(REEMBED_SENTINEL_PATH)).toBe(false);
+    await clearReembedSentinel();
+    expect(existsSync(REEMBED_SENTINEL_PATH)).toBe(false);
+  });
+
+  test("swallows 'already exists' on createPayloadIndex but propagates other failures without latching readiness", async () => {
+    // Existing collection that already has the full schema — the ensure
+    // path goes through `ensurePayloadIndexes` to backfill long-lived
+    // installs. The first index call hits an "already exists" race
+    // (benign; swallow); the second hits a strict-mode rejection (must
+    // propagate so readiness is not latched).
+    state.collectionExistsBeforeCreate = true;
+    state.createIndexThrowQueue.push(
+      Object.assign(
+        new Error("Wrong input: Payload field 'slug' already exists"),
+        { status: 400 },
+      ),
+      Object.assign(
+        new Error(
+          "Strict mode prohibits creating payload indexes on this deployment",
+        ),
+        { status: 400 },
+      ),
+    );
+
+    let caught: unknown = null;
+    try {
+      await ensureConceptPageCollection();
+    } catch (err) {
+      caught = err;
+    }
+    expect((caught as Error | null)?.message).toMatch(/strict mode/i);
+
+    // Both attempts ran; the strict-mode failure was not swallowed.
+    expect(state.createIndexCalls).toEqual([
+      { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
+    ]);
+
+    // Readiness must NOT be latched after a non-benign failure — otherwise
+    // later slug/kind-filtered queries (e.g. skill backfill) would keep
+    // failing until a daemon restart. A follow-up ensure must retry.
+    const result = await ensureConceptPageCollection();
+    expect(result).toEqual({ migrated: false });
+    // Indexes attempted again on the retry (no throws queued this time).
+    expect(state.createIndexCalls).toEqual([
+      { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
+      { field_name: "slug", field_schema: "keyword" },
+      { field_name: "kind", field_schema: "keyword" },
+    ]);
+  });
+
   test("concurrent ensure during a schema rebuild only deletes/creates once", async () => {
     state.collectionExistsBeforeCreate = true;
     state.getCollectionInfo = {

package/src/memory/v2/__tests__/reranker.test.ts

@@ -215,8 +215,8 @@ describe("rerankCandidates", () => {
     expect(backendState.calls).toHaveLength(1);
   });
 
-  test("passage construction caps at 240 chars after slug newline", async () => {
-    const longBody = "a".repeat(500);
+  test("passage construction caps at 1500 chars after slug newline", async () => {
+    const longBody = "a".repeat(2000);
     pageState.pages.set("slug", { body: longBody });
     backendState.scores = [0.5];
 
@@ -224,9 +224,9 @@ describe("rerankCandidates", () => {
 
     expect(backendState.calls).toHaveLength(1);
     const passage = backendState.calls[0].passages[0];
-    // "slug\n" prefix + 240 chars of body
+    // "slug\n" prefix + 1500 chars of body
     expect(passage.startsWith("slug\n")).toBe(true);
-    expect(passage.length).toBeLessThanOrEqual(5 + 240);
+    expect(passage.length).toBeLessThanOrEqual(5 + 1500);
   });
 
   test("first paragraph is taken (body truncated at blank line)", async () => {