@vellumai/assistant 0.6.5 → 0.6.6

package/src/__tests__/suggestion-routes.test.ts

@@ -410,7 +410,7 @@ describe("GET /v1/suggestion", () => {
     expect(body.suggestion).toBe("sounds good");
   });
 
-  test("passes prefill message, stop_sequences, and max_tokens", async () => {
+  test("passes stop_sequences, max_tokens, and XML-framed user prompt", async () => {
     const provider = makeMockProvider("on my way");
     mockGetConfiguredProvider.mockImplementation(async () => provider);
     mockGetConversationByKey.mockImplementation(() => ({
@@ -454,9 +454,8 @@ describe("GET /v1/suggestion", () => {
       };
     };
 
-    expect(messages.length).toBe(2);
-    expect(messages[1].role).toBe("assistant");
-    expect(messages[1].content[0].text).toBe("<reply>");
+    expect(messages.length).toBe(1);
+    expect(messages[0].role).toBe("user");
     expect(options.config?.stop_sequences).toEqual(["</reply>"]);
     expect(options.config?.max_tokens).toBe(60);
     expect(messages[0].content[0].text).toContain("<assistant_message>");
@@ -568,4 +567,104 @@ describe("GET /v1/suggestion", () => {
       | undefined;
     expect(options?.config?.callSite).toBe("conversationStarters");
   });
+
+  test("does not send an assistant-role prefill message", async () => {
+    // Regression guard: Anthropic rejects assistant-message prefill
+    // whenever the request triggers extended thinking (e.g. Opus 4.x at
+    // `effort: "xhigh"`). The suggestion generator must only send a
+    // single user-role message so it stays compatible with every
+    // possible `conversationStarters` call-site config.
+    const provider = makeMockProvider("<reply>Sure, works for me</reply>");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-asst-1",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([
+          { type: "text", text: "Want to meet tomorrow?" },
+        ]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    const response = await handleGetSuggestion(url, deps);
+
+    expect(provider.sendMessage).toHaveBeenCalledTimes(1);
+    const callArgs = provider.sendMessage.mock.calls[0] as unknown[];
+    const messages = callArgs[0] as Array<{ role: string }>;
+    expect(messages).toHaveLength(1);
+    expect(messages[0].role).toBe("user");
+    expect(messages.every((m) => m.role === "user")).toBe(true);
+
+    // Tag-wrapped response is extracted back to just the reply text.
+    const body = (await response.json()) as { suggestion: string | null };
+    expect(body.suggestion).toBe("Sure, works for me");
+  });
+
+  test("handles untagged model response by falling back to raw text", async () => {
+    // Some non-Anthropic models and some Anthropic configs drop the
+    // `<reply>…</reply>` wrapper entirely. The parser must still
+    // produce a usable suggestion from the raw text.
+    const provider = makeMockProvider("Sounds good to me");
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-asst-untagged",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([
+          { type: "text", text: "Want to meet tomorrow?" },
+        ]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    const response = await handleGetSuggestion(url, deps);
+
+    const body = (await response.json()) as { suggestion: string | null };
+    expect(body.suggestion).toBe("Sounds good to me");
+  });
+
+  test("extracts reply content when model adds preamble before tag", async () => {
+    // Without assistant-role prefill, nothing stops a chatty model from
+    // adding a preamble. The parser's tag-extraction step must take the
+    // `<reply>…</reply>` span and ignore surrounding commentary.
+    const provider = makeMockProvider(
+      "Here's a good option:\n\n<reply>Let's do it</reply>",
+    );
+    mockGetConfiguredProvider.mockImplementation(async () => provider);
+    mockGetConversationByKey.mockImplementation(() => ({
+      conversationId: "conv-test",
+    }));
+    mockGetMessages.mockImplementation(() => [
+      {
+        id: "msg-asst-preamble",
+        conversationId: "conv-test",
+        role: "assistant",
+        content: JSON.stringify([{ type: "text", text: "Ready to ship it?" }]),
+        createdAt: Date.now(),
+        metadata: null,
+      },
+    ]);
+
+    const url = makeUrl({ conversationKey: "test-key" });
+    const deps = makeDeps();
+    const response = await handleGetSuggestion(url, deps);
+
+    const body = (await response.json()) as { suggestion: string | null };
+    expect(body.suggestion).toBe("Let's do it");
+  });
 });

package/src/__tests__/task-memory-cleanup.test.ts

@@ -18,6 +18,7 @@ mock.module("../memory/qdrant-client.js", () => ({
     deletePoints: async () => {},
   }),
   initQdrantClient: () => {},
+  resolveQdrantUrl: () => "http://127.0.0.1:6333",
 }));
 
 const TEST_CONFIG = {

package/src/__tests__/task-scheduler.test.ts

@@ -141,11 +141,7 @@ describe("scheduler run_task detection", () => {
       directCalls.push({ conversationId, message, options });
     };
 
-    const scheduler = startScheduler(
-      processMessage,
-      () => {},
-      () => {},
-    );
+    const scheduler = startScheduler(processMessage, () => {});
 
     // Wait for the initial tick to complete
     await new Promise((resolve) => setTimeout(resolve, 500));
@@ -190,11 +186,7 @@ describe("scheduler run_task detection", () => {
       processedMessages.push({ conversationId, message, options });
     };
 
-    const scheduler = startScheduler(
-      processMessage,
-      () => {},
-      () => {},
-    );
+    const scheduler = startScheduler(processMessage, () => {});
 
     await new Promise((resolve) => setTimeout(resolve, 500));
     scheduler.stop();
@@ -229,11 +221,7 @@ describe("scheduler run_task detection", () => {
       _message: string,
     ) => {};
 
-    const scheduler = startScheduler(
-      processMessage,
-      () => {},
-      () => {},
-    );
+    const scheduler = startScheduler(processMessage, () => {});
 
     await new Promise((resolve) => setTimeout(resolve, 500));
     scheduler.stop();

package/src/__tests__/test-preload.ts

@@ -20,10 +20,16 @@ import { installGatewayIpcMock } from "../__tests__/mock-gateway-ipc.js";
 import { resetDb } from "../memory/db-connection.js";
 import { _setStorePath } from "../security/encrypted-store.js";
 
+const savedBaseDataDir = process.env.BASE_DATA_DIR;
+
 const testDir = realpathSync(
   mkdtempSync(join(tmpdir(), "vellum-test-workspace-")),
 );
 process.env.VELLUM_WORKSPACE_DIR = testDir;
+// Also set BASE_DATA_DIR so that vellumRoot() (and all derived paths like
+// getProtectedDir()) resolve under the temp dir. This gives each test file
+// its own IPC socket path, preventing socket races between parallel files.
+process.env.BASE_DATA_DIR = testDir;
 process.env.VELLUM_PLATFORM_URL = "https://test-platform.vellum.ai";
 process.exitCode = 0;
 
@@ -56,6 +62,11 @@ afterAll(() => {
   process.exitCode = 0;
   delete process.env.VELLUM_WORKSPACE_DIR;
   delete process.env.VELLUM_PLATFORM_URL;
+  if (savedBaseDataDir !== undefined) {
+    process.env.BASE_DATA_DIR = savedBaseDataDir;
+  } else {
+    delete process.env.BASE_DATA_DIR;
+  }
   if (savedIsContainerized !== undefined) {
     process.env.IS_CONTAINERIZED = savedIsContainerized;
   }

package/src/__tests__/title-generate-pipeline.test.ts

@@ -0,0 +1,224 @@
+/**
+ * Tests for the `titleGenerate` pipeline (PR 28).
+ *
+ * The title-generation side effect used to be a direct call to
+ * `queueGenerateConversationTitle` inside `conversation-agent-loop.ts`.
+ * After PR 28 the assistant routes that call through the plugin pipeline
+ * runner, giving plugins an opportunity to observe/replace the default
+ * implementation.
+ *
+ * Covers:
+ * - The default plugin's terminal delegates to
+ *   `queueGenerateConversationTitle` with the same arguments the agent
+ *   loop constructs.
+ * - A custom plugin can install a short-circuit middleware that replaces
+ *   the terminal with a deterministic generator. The default terminal is
+ *   NOT invoked in that case.
+ *
+ * Mocks `memory/conversation-title-service.js` so the tests don't touch
+ * the real provider stack, and resets the plugin registry between cases.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Stub the title-generation service before importing anything that binds
+// to it, so both the default plugin and the agent loop capture the
+// stubbed binding.
+const queueGenerateConversationTitleMock = mock(
+  (_params: {
+    conversationId: string;
+    provider?: unknown;
+    userMessage?: string;
+    onTitleUpdated?: (title: string) => void;
+  }): void => undefined,
+);
+mock.module("../memory/conversation-title-service.js", () => ({
+  queueGenerateConversationTitle: queueGenerateConversationTitleMock,
+}));
+
+import { defaultTitleGenerateTerminal } from "../plugins/defaults/title-generate.js";
+import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
+import {
+  getMiddlewaresFor,
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import type {
+  Middleware,
+  Plugin,
+  TitleArgs,
+  TitleResult,
+  TurnContext,
+} from "../plugins/types.js";
+
+function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
+  return {
+    requestId: "req-1",
+    conversationId: "conv-1",
+    turnIndex: 0,
+    trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    ...overrides,
+  };
+}
+
+function makeArgs(overrides: Partial<TitleArgs> = {}): TitleArgs {
+  return {
+    conversationId: "conv-1",
+    userMessage: "hello world",
+    ...overrides,
+  };
+}
+
+describe("titleGenerate pipeline", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    queueGenerateConversationTitleMock.mockReset();
+    queueGenerateConversationTitleMock.mockImplementation(() => undefined);
+    // Re-register the default plugin after the registry reset so tests see
+    // the same shape the daemon sees at runtime.
+    registerPlugin({
+      manifest: {
+        name: "default-title-generate",
+        version: "1.0.0",
+        provides: { titleGenerate: "v1" },
+        requires: {
+          pluginRuntime: "v1",
+          titleGenerateApi: "v1",
+        },
+      },
+    });
+  });
+
+  test("default: pipeline terminal queues a title-generation job", async () => {
+    const ctx = makeCtx();
+    const onTitleUpdated = mock((_title: string) => undefined);
+    const args = makeArgs({
+      conversationId: "conv-1",
+      userMessage: "first message",
+      onTitleUpdated,
+    });
+
+    await runPipeline(
+      "titleGenerate",
+      getMiddlewaresFor("titleGenerate"),
+      defaultTitleGenerateTerminal,
+      args,
+      ctx,
+      DEFAULT_TIMEOUTS.titleGenerate,
+    );
+
+    // The default terminal must have delegated to queueGenerateConversationTitle
+    // with every argument the caller supplied, including the callback.
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueGenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call?.userMessage).toBe("first message");
+    expect(call?.onTitleUpdated).toBe(onTitleUpdated);
+  });
+
+  test("default: pipeline result is the empty object from the terminal", async () => {
+    const ctx = makeCtx();
+    const result = await runPipeline(
+      "titleGenerate",
+      getMiddlewaresFor("titleGenerate"),
+      defaultTitleGenerateTerminal,
+      makeArgs(),
+      ctx,
+      DEFAULT_TIMEOUTS.titleGenerate,
+    );
+    expect(result).toEqual({});
+  });
+
+  test("custom plugin: short-circuit middleware replaces the default with a deterministic generator", async () => {
+    // A custom plugin installs middleware that fabricates a title
+    // deterministically and never calls `next`, so the default terminal
+    // (which would delegate to queueGenerateConversationTitle) is
+    // skipped entirely.
+    const observedTitles: string[] = [];
+
+    const deterministicMw: Middleware<TitleArgs, TitleResult> = async (
+      args,
+    ) => {
+      const fabricated = `[deterministic] ${args.userMessage}`;
+      args.onTitleUpdated?.(fabricated);
+      observedTitles.push(fabricated);
+      return {};
+    };
+
+    const customPlugin: Plugin = {
+      manifest: {
+        name: "custom-deterministic-title",
+        version: "0.0.1",
+        requires: {
+          pluginRuntime: "v1",
+          titleGenerateApi: "v1",
+        },
+      },
+      middleware: { titleGenerate: deterministicMw },
+    };
+    registerPlugin(customPlugin);
+
+    const receivedTitle: string[] = [];
+    const args = makeArgs({
+      userMessage: "what is the weather",
+      onTitleUpdated: (title) => {
+        receivedTitle.push(title);
+      },
+    });
+
+    await runPipeline(
+      "titleGenerate",
+      getMiddlewaresFor("titleGenerate"),
+      defaultTitleGenerateTerminal,
+      args,
+      makeCtx(),
+      DEFAULT_TIMEOUTS.titleGenerate,
+    );
+
+    // Deterministic middleware produced the expected title and invoked
+    // the caller's callback.
+    expect(observedTitles).toEqual(["[deterministic] what is the weather"]);
+    expect(receivedTitle).toEqual(["[deterministic] what is the weather"]);
+    // The default terminal must NOT have been reached — it would have
+    // called the real title-service stub.
+    expect(queueGenerateConversationTitleMock).not.toHaveBeenCalled();
+  });
+
+  test("custom plugin: passthrough middleware leaves the default in charge", async () => {
+    // A plugin that always calls `next` just observes — the default
+    // terminal still runs and queues the title-generation job.
+    let middlewareSawArgs = false;
+
+    const passthroughMw: Middleware<TitleArgs, TitleResult> = async (
+      args,
+      next,
+    ) => {
+      middlewareSawArgs = true;
+      return next(args);
+    };
+
+    registerPlugin({
+      manifest: {
+        name: "observer",
+        version: "0.0.1",
+        requires: {
+          pluginRuntime: "v1",
+          titleGenerateApi: "v1",
+        },
+      },
+      middleware: { titleGenerate: passthroughMw },
+    });
+
+    await runPipeline(
+      "titleGenerate",
+      getMiddlewaresFor("titleGenerate"),
+      defaultTitleGenerateTerminal,
+      makeArgs(),
+      makeCtx(),
+      DEFAULT_TIMEOUTS.titleGenerate,
+    );
+
+    expect(middlewareSawArgs).toBe(true);
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+});