@vellumai/assistant 0.8.6 → 0.8.7-dev.202606052118.34cd356

package/src/__tests__/title-generate-hook.test.ts

@@ -0,0 +1,319 @@
+/**
+ * Tests for the default `title-generate` plugin's hooks.
+ *
+ * The plugin contributes two pure-trigger hooks that delegate the title work
+ * to the service:
+ *
+ * - `user-prompt-submit` — first-pass generation from the submitted prompt,
+ *   scheduled on a later macrotask so the main agent-loop LLM request is
+ *   issued first.
+ * - `stop` — second-pass regeneration once the conversation reaches its third
+ *   user turn (turn count derived from the user prompts in history).
+ *
+ * Both let the title service resolve the provider, persist the title, and
+ * broadcast the resulting `conversation_title_updated` / `sync_changed`
+ * events.
+ *
+ * Mocks `memory/conversation-title-service.js` and `config/loader.js` so the
+ * tests don't touch the real provider stack or config, and resets the plugin
+ * registry between cases.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// Stub the title-generation service before importing anything that binds
+// to it, so both the default plugin and the hooks capture the stubbed binding.
+const queueGenerateConversationTitleMock = mock(
+  (_params: {
+    conversationId: string;
+    provider?: unknown;
+    userMessage?: string;
+  }): void => undefined,
+);
+const queueRegenerateConversationTitleMock = mock(
+  (_params: { conversationId: string; provider?: unknown }): void => undefined,
+);
+mock.module("../memory/conversation-title-service.js", () => ({
+  queueGenerateConversationTitle: queueGenerateConversationTitleMock,
+  queueRegenerateConversationTitle: queueRegenerateConversationTitleMock,
+}));
+
+// The `stop` hook reads `conversations.skipAutoRetitling`; stub the loader so
+// the opt-out is controllable per test.
+let skipAutoRetitling = false;
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ conversations: { skipAutoRetitling } }),
+}));
+
+import { HOOKS } from "../plugin-api/constants.js";
+import type {
+  PluginLogger,
+  StopContext,
+  UserPromptSubmitContext,
+} from "../plugin-api/types.js";
+import stop from "../plugins/defaults/title-generate/hooks/stop.js";
+import userPromptSubmit from "../plugins/defaults/title-generate/hooks/user-prompt-submit.js";
+import { defaultTitleGeneratePlugin } from "../plugins/defaults/title-generate/register.js";
+import { runHook } from "../plugins/pipeline.js";
+import {
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import type { Message } from "../providers/types.js";
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+function makeCtx(
+  overrides: Partial<UserPromptSubmitContext> = {},
+): UserPromptSubmitContext {
+  const messages: Message[] = [
+    { role: "user", content: [{ type: "text", text: "first message" }] },
+  ];
+  return {
+    conversationId: "conv-1",
+    prompt: "first message",
+    originalMessages: messages,
+    latestMessages: messages,
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+/** Flush pending `setTimeout(0)` callbacks so the fire-and-forget trigger runs. */
+function flushMacrotasks(): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, 0));
+}
+
+function userTurn(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+function assistantTurn(text: string): Message {
+  return { role: "assistant", content: [{ type: "text", text }] };
+}
+
+/** A user-role message carrying only tool results, not a fresh prompt. */
+function toolResultTurn(): Message {
+  return {
+    role: "user",
+    content: [{ type: "tool_result", tool_use_id: "tool-1", content: "ok" }],
+  };
+}
+
+/** History with `count` genuine user turns interleaved with assistant replies. */
+function historyWithUserTurns(count: number): Message[] {
+  const messages: Message[] = [];
+  for (let i = 1; i <= count; i++) {
+    messages.push(userTurn(`message ${i}`));
+    messages.push(assistantTurn(`reply ${i}`));
+  }
+  return messages;
+}
+
+function makeStopCtx(overrides: Partial<StopContext> = {}): StopContext {
+  return {
+    conversationId: "conv-1",
+    messages: historyWithUserTurns(3),
+    responseContent: [{ type: "text", text: "reply 3" }],
+    stopReason: "end_turn",
+    decision: "stop",
+    logger: noopLogger,
+    ...overrides,
+  };
+}
+
+describe("title-generate user-prompt-submit hook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    queueGenerateConversationTitleMock.mockReset();
+    queueGenerateConversationTitleMock.mockImplementation(() => undefined);
+  });
+
+  test("queues a title-generation job from the submitted prompt", async () => {
+    // GIVEN a fresh user prompt submission
+    const ctx = makeCtx({ conversationId: "conv-1", prompt: "first message" });
+
+    // WHEN the default hook runs and its deferred work flushes
+    await userPromptSubmit(ctx);
+    await flushMacrotasks();
+
+    // THEN the title service is invoked with just the conversation id and the
+    // submitted prompt — provider resolution and emit are owned by the service.
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueGenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call?.userMessage).toBe("first message");
+    expect(call).not.toHaveProperty("provider");
+    expect(call).not.toHaveProperty("onTitleUpdated");
+  });
+
+  test("does not block: returns before the title job is scheduled", async () => {
+    // GIVEN a fresh user prompt submission
+    const ctx = makeCtx();
+
+    // WHEN the hook resolves
+    await userPromptSubmit(ctx);
+
+    // THEN the title job has not run yet (it is deferred to a later macrotask),
+    // AND it runs once the macrotask queue is flushed.
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+    await flushMacrotasks();
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("fires through runHook once the default plugin is registered", async () => {
+    // GIVEN the default title-generate plugin registered in the registry
+    registerPlugin(defaultTitleGeneratePlugin);
+
+    // WHEN a prompt is submitted through the hook chain
+    await runHook(
+      HOOKS.USER_PROMPT_SUBMIT,
+      makeCtx({ prompt: "draft a plan" }),
+    );
+    await flushMacrotasks();
+
+    // THEN the title service is triggered with the submitted prompt text
+    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(
+      queueGenerateConversationTitleMock.mock.calls[0]?.[0]?.userMessage,
+    ).toBe("draft a plan");
+  });
+});
+
+describe("title-generate stop hook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+    queueRegenerateConversationTitleMock.mockReset();
+    queueRegenerateConversationTitleMock.mockImplementation(() => undefined);
+    skipAutoRetitling = false;
+  });
+
+  test("regenerates the title on the third user turn", async () => {
+    // GIVEN a turn ending with three genuine user prompts in history
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the stop hook runs and its deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN the second-pass regeneration is triggered with just the
+    // conversation id — provider resolution and emit are owned by the service.
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    const call = queueRegenerateConversationTitleMock.mock.calls[0]?.[0];
+    expect(call?.conversationId).toBe("conv-1");
+    expect(call).not.toHaveProperty("provider");
+    expect(call).not.toHaveProperty("signal");
+  });
+
+  test("defers the regeneration so the completed turn is persisted first", async () => {
+    // GIVEN a turn ending on the third user turn
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the hook resolves
+    await stop(ctx);
+
+    // THEN the regeneration has not fired yet — it is deferred to a later
+    // macrotask so the turn's assistant reply lands first, AND it fires once
+    // the macrotask queue is flushed.
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+    await flushMacrotasks();
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("does not regenerate before the third user turn", async () => {
+    // GIVEN a turn ending with only two genuine user prompts
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(2) });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires — the conversation lacks enough context yet
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("does not regenerate after the third user turn", async () => {
+    // GIVEN a turn ending with four genuine user prompts
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(4) });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires — the single second pass already passed
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("ignores tool-result user messages when counting turns", async () => {
+    // GIVEN three genuine user prompts plus a tool-result user message
+    const messages: Message[] = [
+      userTurn("message 1"),
+      assistantTurn("reply 1"),
+      userTurn("message 2"),
+      assistantTurn("calling a tool"),
+      toolResultTurn(),
+      assistantTurn("reply 2"),
+      userTurn("message 3"),
+    ];
+    const ctx = makeStopCtx({ messages });
+
+    // WHEN the stop hook runs and its deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN the tool-result message is not counted as a turn, so the third
+    // genuine prompt still triggers the regeneration
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+  });
+
+  test("defers when an earlier hook chose to continue the run", async () => {
+    // GIVEN a third-user-turn stop that an earlier hook flipped to "continue"
+    const ctx = makeStopCtx({
+      messages: historyWithUserTurns(3),
+      decision: "continue",
+    });
+
+    // WHEN the stop hook runs and any deferred work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN it defers to the eventual terminal stop rather than re-titling now
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("respects the skipAutoRetitling opt-out", async () => {
+    // GIVEN the user opted out of second-pass retitling
+    skipAutoRetitling = true;
+    const ctx = makeStopCtx({ messages: historyWithUserTurns(3) });
+
+    // WHEN the stop hook runs on the third user turn and any work flushes
+    await stop(ctx);
+    await flushMacrotasks();
+
+    // THEN no regeneration fires
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(0);
+  });
+
+  test("fires through runHook once the default plugin is registered", async () => {
+    // GIVEN the default title-generate plugin registered in the registry
+    registerPlugin(defaultTitleGeneratePlugin);
+
+    // WHEN a third-user-turn stop is dispatched through the hook chain
+    await runHook(
+      HOOKS.STOP,
+      makeStopCtx({ messages: historyWithUserTurns(3) }),
+    );
+    await flushMacrotasks();
+
+    // THEN the second-pass regeneration is triggered
+    expect(queueRegenerateConversationTitleMock).toHaveBeenCalledTimes(1);
+    expect(
+      queueRegenerateConversationTitleMock.mock.calls[0]?.[0]?.conversationId,
+    ).toBe("conv-1");
+  });
+});

package/src/__tests__/tool-error-hook.test.ts

@@ -0,0 +1,278 @@
+/**
+ * Tests for the default `tool-error` plugin's `post-tool-use` hook.
+ *
+ * Covers:
+ * - The hook surfaces the canonical coaching notice via `additionalContext`
+ *   (leaving the tool result's `content` untouched) when the result carries
+ *   `is_error`, and is a no-op for a successful result.
+ * - The consecutive-failure guard is derived from the conversation history per
+ *   tool name: coaching fires on the 1st through Nth back-to-back failure of a
+ *   tool and is dropped once that tool exceeds the cap. A success in between
+ *   resets the streak, and failures of a different tool don't count.
+ * - End-to-end through `runHook` + the registry: registering the default
+ *   plugin makes the hook fire, and the default runs ahead of a later
+ *   user hook so the user observes the already-set `additionalContext`.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { HOOKS } from "../plugin-api/constants.js";
+import type { PluginLogger, PostToolUseContext } from "../plugin-api/types.js";
+import postToolUse, {
+  TOOL_ERROR_NUDGE_TEXT,
+} from "../plugins/defaults/tool-error/hooks/post-tool-use.js";
+import { defaultToolErrorPlugin } from "../plugins/defaults/tool-error/register.js";
+import { runHook } from "../plugins/pipeline.js";
+import {
+  registerPlugin,
+  resetPluginRegistryForTests,
+} from "../plugins/registry.js";
+import type { Message, ToolResultContent } from "../providers/types.js";
+
+const noopLogger: PluginLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+
+const BASE_CONTENT = "tool blew up";
+
+/** Assistant turn issuing a single `tool_use` block. */
+function toolUseTurn(id: string, name: string): Message {
+  return {
+    role: "assistant",
+    content: [{ type: "tool_use", id, name, input: {} }],
+  };
+}
+
+/** User turn carrying a single `tool_result` for a prior `tool_use`. */
+function toolResultTurn(toolUseId: string, isError: boolean): Message {
+  return {
+    role: "user",
+    content: [
+      {
+        type: "tool_result",
+        tool_use_id: toolUseId,
+        content: "result",
+        is_error: isError,
+      },
+    ],
+  };
+}
+
+function errorResponse(toolUseId: string): ToolResultContent {
+  return {
+    type: "tool_result",
+    tool_use_id: toolUseId,
+    content: BASE_CONTENT,
+    is_error: true,
+  };
+}
+
+function makeCtx(
+  toolResponse: ToolResultContent,
+  messages: Message[],
+): PostToolUseContext {
+  return {
+    conversationId: "conv-tool-error-test",
+    toolResponse,
+    messages,
+    maxInputTokens: 10_000,
+    logger: noopLogger,
+  };
+}
+
+/**
+ * Build a history of `priorErrors` back-to-back failures of `toolName`,
+ * followed by the current turn's `tool_use` (whose result is delivered via
+ * `ctx.toolResponse`, not history). Returns the history and the current
+ * tool_use id.
+ */
+function historyWithConsecutiveErrors(
+  toolName: string,
+  priorErrors: number,
+): { messages: Message[]; currentToolUseId: string } {
+  const messages: Message[] = [];
+  for (let i = 0; i < priorErrors; i++) {
+    const id = `${toolName}-${i}`;
+    messages.push(toolUseTurn(id, toolName));
+    messages.push(toolResultTurn(id, true));
+  }
+  const currentToolUseId = `${toolName}-current`;
+  messages.push(toolUseTurn(currentToolUseId, toolName));
+  return { messages, currentToolUseId };
+}
+
+describe("tool-error post-tool-use hook — direct", () => {
+  test("surfaces the canonical coaching notice via additionalContext on a first error", async () => {
+    // GIVEN an error result whose tool has not failed before.
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+    const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN the coaching notice is surfaced via additionalContext, and the tool
+    // result's own content is left untouched.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("is a no-op for a successful result", async () => {
+    // GIVEN a successful result for a tool.
+    const messages = [toolUseTurn("search-current", "search")];
+    const ctx = makeCtx(
+      {
+        type: "tool_result",
+        tool_use_id: "search-current",
+        content: BASE_CONTENT,
+        is_error: false,
+      },
+      messages,
+    );
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN no coaching is surfaced and the content is left untouched.
+    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("keeps coaching up to and including the consecutive-failure cap", async () => {
+    // GIVEN a tool that has already failed 0, 1, then 2 times in a row — so the
+    // current failure is the 1st, 2nd, then 3rd (the cap) consecutive failure.
+    for (let priorErrors = 0; priorErrors <= 2; priorErrors++) {
+      const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+        "search",
+        priorErrors,
+      );
+      const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+      // WHEN the hook runs.
+      await postToolUse(ctx);
+
+      // THEN it still coaches the retry.
+      expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+    }
+  });
+
+  test("drops the coaching once the tool exceeds the cap", async () => {
+    // GIVEN a tool that has already failed 3 times in a row, so the current
+    // failure is the 4th — past the cap of 3.
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      3,
+    );
+    const ctx = makeCtx(errorResponse(currentToolUseId), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN no coaching is surfaced (the error is likely unrecoverable) and the
+    // result is left untouched.
+    expect(ctx.additionalContext).toBeUndefined();
+    expect(ctx.toolResponse.content).toBe(BASE_CONTENT);
+  });
+
+  test("a successful result between failures resets the streak", async () => {
+    // GIVEN four failures of "search" but with a success in the middle, so the
+    // trailing run of failures is only two — the current failure is the 3rd.
+    const messages: Message[] = [
+      toolUseTurn("search-0", "search"),
+      toolResultTurn("search-0", true),
+      toolUseTurn("search-1", "search"),
+      toolResultTurn("search-1", true),
+      toolUseTurn("search-2", "search"),
+      toolResultTurn("search-2", false),
+      toolUseTurn("search-3", "search"),
+      toolResultTurn("search-3", true),
+      toolUseTurn("search-current", "search"),
+    ];
+    const ctx = makeCtx(errorResponse("search-current"), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN coaching still fires — the success reset the streak below the cap.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+
+  test("counts failures per tool name, not globally", async () => {
+    // GIVEN three failures of "other" and zero of "search" before the current
+    // "search" failure — a global counter would be over the cap, but the
+    // per-tool streak for "search" is just this one failure.
+    const messages: Message[] = [
+      toolUseTurn("other-0", "other"),
+      toolResultTurn("other-0", true),
+      toolUseTurn("other-1", "other"),
+      toolResultTurn("other-1", true),
+      toolUseTurn("other-2", "other"),
+      toolResultTurn("other-2", true),
+      toolUseTurn("search-current", "search"),
+    ];
+    const ctx = makeCtx(errorResponse("search-current"), messages);
+
+    // WHEN the hook runs.
+    await postToolUse(ctx);
+
+    // THEN "search" is coached on its first failure.
+    expect(ctx.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+});
+
+describe("tool-error post-tool-use hook — via runHook", () => {
+  beforeEach(() => {
+    resetPluginRegistryForTests();
+  });
+
+  test("registering the default plugin coaches an error result", async () => {
+    // GIVEN the default tool-error plugin is registered.
+    registerPlugin(defaultToolErrorPlugin);
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+
+    // WHEN the post-tool-use chain runs over an error result.
+    const result = await runHook<PostToolUseContext>(
+      HOOKS.POST_TOOL_USE,
+      makeCtx(errorResponse(currentToolUseId), messages),
+    );
+
+    // THEN the coaching notice is surfaced via additionalContext.
+    expect(result.additionalContext).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+
+  test("default hook runs before a later-registered user hook", async () => {
+    // GIVEN the default plugin is registered first, then a user plugin whose
+    // hook records the additionalContext it observes.
+    let observed: string | undefined;
+    registerPlugin(defaultToolErrorPlugin);
+    registerPlugin({
+      manifest: { name: "observer-plugin", version: "0.0.1" },
+      hooks: {
+        "post-tool-use": async (ctx: PostToolUseContext) => {
+          observed = ctx.additionalContext;
+        },
+      },
+    });
+    const { messages, currentToolUseId } = historyWithConsecutiveErrors(
+      "search",
+      0,
+    );
+
+    // WHEN the chain runs.
+    await runHook<PostToolUseContext>(
+      HOOKS.POST_TOOL_USE,
+      makeCtx(errorResponse(currentToolUseId), messages),
+    );
+
+    // THEN the user hook saw the already-set additionalContext.
+    expect(observed).toBe(TOOL_ERROR_NUDGE_TEXT);
+  });
+});