@vellumai/assistant 0.8.7 → 0.8.8-dev.202606052332.17fc8ea

package/src/__tests__/empty-response-pipeline.test.ts

@@ -1,423 +0,0 @@
-/**
- * Tests for the `emptyResponse` plugin pipeline (PR 18).
- *
- * Covers:
- * - Default plugin decision matches the original inline loop logic for the
- *   canonical cases (empty-after-tools → nudge, visible-text → accept,
- *   tool-use-blocks-present → accept, retries-exhausted → accept,
- *   prior-visible-text-in-run → accept).
- * - Swapping in a custom middleware that returns `action: "accept"` prevents
- *   the nudge and lets the loop fall through to history append.
- * - Swapping in a custom middleware that returns `action: "error"` is
- *   propagated by the pipeline so the loop can surface a clear error.
- *
- * The loop's actual side-effects (history append, retry counter bump, log
- * emission) live in `agent/loop.ts` and are covered by integration tests in
- * `conversation-agent-loop.test.ts`. This file isolates the pipeline.
- */
-
-import { beforeEach, describe, expect, test } from "bun:test";
-
-import type { TrustContext } from "../daemon/trust-context.js";
-import { defaultEmptyResponsePlugin } from "../plugins/defaults/empty-response/register.js";
-import { defaultEmptyResponseTerminal } from "../plugins/defaults/empty-response/terminal.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import type {
-  EmptyResponseArgs,
-  EmptyResponseDecision,
-  Middleware,
-  Plugin,
-  TurnContext,
-} from "../plugins/types.js";
-import type { ContentBlock } from "../providers/types.js";
-
-// ─── Fixtures ────────────────────────────────────────────────────────────────
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeCtx(): TurnContext {
-  return {
-    requestId: "req-empty-response",
-    conversationId: "conv-empty-response",
-    turnIndex: 2,
-    trust,
-  };
-}
-
-/**
- * The nudge text has to match the loop's original inline string verbatim —
- * clients (and the model) may match on this exact text.
- */
-const CANONICAL_NUDGE_TEXT =
-  "<system_notice>Your previous response was empty. You must respond to the user with a summary of what you found or did. Do not use any tools — just respond with text.</system_notice>";
-
-const emptyTextBlock: ContentBlock = { type: "text", text: "   " };
-
-function makeArgs(
-  overrides: Partial<EmptyResponseArgs> = {},
-): EmptyResponseArgs {
-  return {
-    responseContent: [],
-    toolUseBlocksLength: 0,
-    toolUseTurns: 1,
-    emptyResponseRetries: 0,
-    maxEmptyResponseRetries: 1,
-    priorAssistantHadVisibleText: false,
-    // Default to `null` (no stop reason reported) so existing fixtures
-    // exercise the "organic empty turn" path. The refusal branch
-    // dedicated tests below set this to `"refusal"` explicitly.
-    stopReason: null,
-    ...overrides,
-  };
-}
-
-/**
- * Refusal-specific nudge text — keep in sync with `register.ts`. Clients
- * (and the model) may match on this exact text.
- */
-const CANONICAL_REFUSAL_NUDGE_TEXT =
-  '<system_notice>Your previous response was empty because the upstream provider returned stop_reason="refusal". Please answer the user\'s last message directly with a plain-text response. Do not use any tools — just respond with text.</system_notice>';
-
-async function runEmpty(
-  args: EmptyResponseArgs,
-): Promise<EmptyResponseDecision> {
-  return runPipeline(
-    "emptyResponse",
-    getMiddlewaresFor("emptyResponse"),
-    async (a) => defaultEmptyResponseTerminal(a),
-    args,
-    makeCtx(),
-    DEFAULT_TIMEOUTS.emptyResponse,
-  );
-}
-
-// ─── Tests ───────────────────────────────────────────────────────────────────
-
-describe("emptyResponse pipeline — default decisions", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-    registerPlugin(defaultEmptyResponsePlugin);
-  });
-
-  test("empty turn after tool results → nudge with canonical text", async () => {
-    // Whitespace-only text counts as empty (matches inline `trim().length > 0`).
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [emptyTextBlock],
-        toolUseBlocksLength: 0,
-        toolUseTurns: 2,
-        emptyResponseRetries: 0,
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-    expect(decision.action).toBe("nudge");
-    expect(decision.nudgeText).toBe(CANONICAL_NUDGE_TEXT);
-  });
-
-  test("turn contains visible text → accept", async () => {
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [{ type: "text", text: "here is a summary" }],
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("turn contains tool_use blocks → accept (not empty)", async () => {
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [
-          {
-            type: "tool_use",
-            id: "tu-1",
-            name: "read",
-            input: { path: "/tmp/x" },
-          } as ContentBlock,
-        ],
-        toolUseBlocksLength: 1,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("retries already exhausted → accept", async () => {
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [],
-        toolUseTurns: 3,
-        emptyResponseRetries: 1,
-        maxEmptyResponseRetries: 1,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("prior assistant turn already delivered visible text → accept", async () => {
-    // Model said its piece earlier, ended with a side-effect tool, returned
-    // empty. Nudging would force a verbatim re-send of text the user already
-    // saw. Default must accept.
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [],
-        toolUseTurns: 2,
-        priorAssistantHadVisibleText: true,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("no prior tool-use turn (toolUseTurns === 0) → accept", async () => {
-    // Empty first assistant response with no tools is not the pattern the
-    // organic-empty-turn nudge guards against. Default accepts (unless the
-    // stop reason is `"refusal"` — see the refusal-specific tests below).
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [],
-        toolUseTurns: 0,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  // ─── Refusal stop ────────────────────────────────────────────────────────
-
-  test("stopReason='refusal' on turn 0 with no content → nudge with refusal text", async () => {
-    // The canonical failure mode this branch exists to catch: Anthropic's
-    // safety classifier zeros the response on the very first model call,
-    // returning a single thinking block and `stopReason: "refusal"`. Without
-    // this branch, the terminal would `accept` and the loop would persist
-    // an empty assistant bubble to the user.
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [],
-        toolUseBlocksLength: 0,
-        toolUseTurns: 0,
-        emptyResponseRetries: 0,
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-    expect(decision.action).toBe("nudge");
-    expect(decision.nudgeText).toBe(CANONICAL_REFUSAL_NUDGE_TEXT);
-  });
-
-  test("stopReason='refusal' with a thinking-only block still nudges", async () => {
-    // Thinking blocks aren't visible text — the user sees nothing. A
-    // refusal with only thinking content matches the same shape the
-    // production log captured (`contentBlocks: 1, toolUseCount: 0`).
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [
-          {
-            type: "thinking",
-            thinking: "...",
-            signature: "sig",
-          } as ContentBlock,
-        ],
-        toolUseBlocksLength: 0,
-        toolUseTurns: 0,
-      }),
-    );
-    expect(decision.action).toBe("nudge");
-    expect(decision.nudgeText).toBe(CANONICAL_REFUSAL_NUDGE_TEXT);
-  });
-
-  test("stopReason='refusal' but visible text present → accept (model recovered)", async () => {
-    // The classifier can flag a partial response; if the model already
-    // delivered some visible text before refusing, the user has something
-    // to see. Accept.
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [{ type: "text", text: "partial answer" }],
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("stopReason='refusal' but tool_use blocks present → accept", async () => {
-    // A refusal with tool_use blocks is unusual (the model wouldn't normally
-    // issue tools after a classifier hit) but we still shouldn't nudge —
-    // the loop will execute the tools and the model will get another shot.
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [
-          {
-            type: "tool_use",
-            id: "tu-1",
-            name: "read",
-            input: { path: "/tmp/x" },
-          } as ContentBlock,
-        ],
-        toolUseBlocksLength: 1,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("stopReason='refusal' but retries exhausted → accept (no infinite loop)", async () => {
-    // Persistent classifier hit shouldn't burn turns indefinitely. Once
-    // we've used our retry budget, accept (the user will see an empty
-    // bubble, but the loop terminates).
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [],
-        toolUseTurns: 0,
-        emptyResponseRetries: 1,
-        maxEmptyResponseRetries: 1,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-  });
-
-  test("stopReason='refusal' beats post-tool-empty nudge text (refusal-specific wording)", async () => {
-    // When both branches would fire, refusal wins because the refusal
-    // text is more accurate ("safety classifier zeroed the response"
-    // vs. "summary of what you found or did"). This guards against a
-    // future refactor that orders the branches differently.
-    const decision = await runEmpty(
-      makeArgs({
-        stopReason: "refusal",
-        responseContent: [],
-        toolUseBlocksLength: 0,
-        toolUseTurns: 2, // would trip the post-tool branch too
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-    expect(decision.action).toBe("nudge");
-    expect(decision.nudgeText).toBe(CANONICAL_REFUSAL_NUDGE_TEXT);
-  });
-});
-
-describe("emptyResponse pipeline — custom middleware overrides", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-  });
-
-  test("plugin returning action:accept suppresses the nudge", async () => {
-    // Build a plugin whose middleware short-circuits with accept. Register it
-    // as the ONLY plugin so its decision is authoritative. The loop-side
-    // effect (no nudge appended) is covered by integration tests; here we
-    // assert the pipeline returns what the plugin returned.
-    const acceptPlugin: Plugin = {
-      manifest: {
-        name: "force-accept",
-        version: "1.0.0",
-      },
-      middleware: {
-        emptyResponse: async () => ({ action: "accept" }),
-      },
-    };
-    registerPlugin(acceptPlugin);
-
-    const decision = await runEmpty(
-      makeArgs({
-        // Conditions the default would nudge on — but the custom plugin wins.
-        responseContent: [],
-        toolUseTurns: 2,
-        emptyResponseRetries: 0,
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-    expect(decision.action).toBe("accept");
-    // `nudgeText` must not leak from the acceptance branch.
-    expect(decision.nudgeText).toBeUndefined();
-  });
-
-  test("plugin returning action:error is propagated to the caller", async () => {
-    const errorPlugin: Plugin = {
-      manifest: {
-        name: "force-error",
-        version: "1.0.0",
-      },
-      middleware: {
-        emptyResponse: async () => ({ action: "error" }),
-      },
-    };
-    registerPlugin(errorPlugin);
-
-    const decision = await runEmpty(makeArgs());
-    expect(decision.action).toBe("error");
-  });
-
-  test("plugin overriding default nudge text changes the returned text", async () => {
-    // Exercises the wrapping semantics: the custom plugin observes the
-    // default's decision via `next(args)` and rewrites only the text. This
-    // is the canonical "plugin wraps default" pattern.
-    const rewriterPlugin: Plugin = {
-      manifest: {
-        name: "rewrite-nudge",
-        version: "1.0.0",
-      },
-      middleware: {
-        emptyResponse: async (args, next, ctx) => {
-          const downstream = await next(args);
-          if (downstream.action !== "nudge") return downstream;
-          void ctx; // silence lint
-          return { action: "nudge", nudgeText: "ALTERED_NUDGE" };
-        },
-      },
-    };
-    // Register the custom plugin FIRST so it is the outermost middleware; the
-    // default registers second and acts as the inner decision maker.
-    registerPlugin(rewriterPlugin);
-    registerPlugin(defaultEmptyResponsePlugin);
-
-    const decision = await runEmpty(
-      makeArgs({
-        responseContent: [],
-        toolUseTurns: 2,
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-    expect(decision.action).toBe("nudge");
-    expect(decision.nudgeText).toBe("ALTERED_NUDGE");
-  });
-
-  test("user plugin registered AFTER the default still runs (no shadowing)", async () => {
-    // Production registration order: defaults load first via the side-effect
-    // imports in `defaults/index.ts`, then user plugins register on top via
-    // `bootstrapPlugins()`. The user's middleware ends up at a deeper onion
-    // layer than the default. If the default's middleware were to bypass
-    // `next` and decide directly, the user middleware would never run — this
-    // test guards against that regression.
-    registerPlugin(defaultEmptyResponsePlugin);
-
-    let userMiddlewareRan = false;
-    const userMiddleware: Middleware<
-      EmptyResponseArgs,
-      EmptyResponseDecision
-    > = async (args, next) => {
-      userMiddlewareRan = true;
-      return next(args);
-    };
-    registerPlugin({
-      manifest: {
-        name: "late-user-empty-response",
-        version: "0.0.1",
-      },
-      middleware: { emptyResponse: userMiddleware },
-    });
-
-    await runEmpty(
-      makeArgs({
-        responseContent: [],
-        toolUseTurns: 2,
-        priorAssistantHadVisibleText: false,
-      }),
-    );
-
-    expect(userMiddlewareRan).toBe(true);
-  });
-});

package/src/__tests__/llm-call-pipeline.test.ts

@@ -1,287 +0,0 @@
-/**
- * Unit tests for the `llmCall` pipeline wrapping (PR 15).
- *
- * Exercises the three behaviors the plan calls out:
- *
- * 1. The default `llmCall` pipeline delegates to `provider.sendMessage(...)`
- *    and returns its response unchanged.
- * 2. A spy middleware registered for `llmCall` observes the full argument
- *    payload before the provider is called.
- * 3. A short-circuit middleware synthesizes a `ProviderResponse` and prevents
- *    the real `provider.sendMessage` from running.
- */
-
-import { afterAll, beforeEach, describe, expect, test } from "bun:test";
-
-import type { TrustContext } from "../daemon/trust-context.js";
-import { defaultLlmCallPlugin } from "../plugins/defaults/llm-call/register.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import type {
-  LLMCallArgs,
-  LLMCallResult,
-  Middleware,
-  Plugin,
-  TurnContext,
-} from "../plugins/types.js";
-import type {
-  Message,
-  Provider,
-  ProviderResponse,
-  SendMessageOptions,
-  ToolDefinition,
-} from "../providers/types.js";
-
-// ─── Fixtures ───────────────────────────────────────────────────────────────
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
-  return {
-    requestId: "req-test",
-    conversationId: "conv-test",
-    turnIndex: 0,
-    trust,
-    ...overrides,
-  };
-}
-
-function makeResponse(
-  overrides: Partial<ProviderResponse> = {},
-): ProviderResponse {
-  return {
-    content: [{ type: "text", text: "hello from provider" }],
-    model: "fake-model",
-    usage: {
-      inputTokens: 10,
-      outputTokens: 5,
-    },
-    stopReason: "end_turn",
-    ...overrides,
-  };
-}
-
-type FakeProviderCall = {
-  messages: Message[];
-  tools?: ToolDefinition[];
-  systemPrompt?: string;
-};
-
-function makeFakeProvider(
-  response: ProviderResponse = makeResponse(),
-): Provider & { calls: FakeProviderCall[] } {
-  const calls: FakeProviderCall[] = [];
-  return {
-    name: "fake-provider",
-    calls,
-    async sendMessage(messages: Message[], options?: SendMessageOptions) {
-      calls.push({
-        messages,
-        tools: options?.tools,
-        systemPrompt: options?.systemPrompt,
-      });
-      return response;
-    },
-  };
-}
-
-function makeArgs(
-  provider: Provider,
-  overrides: Partial<LLMCallArgs> = {},
-): LLMCallArgs {
-  return {
-    provider,
-    messages: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
-    options: { systemPrompt: "you are a helpful assistant", config: {} },
-    ...overrides,
-  };
-}
-
-// The terminal passed into `runPipeline` matches the one in `agent/loop.ts`:
-// it delegates straight to `args.provider.sendMessage(...)` with no
-// transformation. Keeping it identical here means the test exercises the
-// exact call shape the real loop uses.
-const terminal = (args: LLMCallArgs): Promise<LLMCallResult> =>
-  args.provider.sendMessage(args.messages, args.options);
-
-// ─── Tests ──────────────────────────────────────────────────────────────────
-
-describe("llmCall pipeline", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-  });
-
-  // Clear the registry on the way out too so later test files in the same
-  // `bun test` run don't inherit `llmCall` middleware from our final test.
-  // Bun runs files sequentially within a process; `beforeEach` only clears
-  // at the start of each case, leaving whatever the final test registered
-  // in place for the next file.
-  afterAll(() => {
-    resetPluginRegistryForTests();
-  });
-
-  test("default pipeline invokes provider.sendMessage and returns its response", async () => {
-    registerPlugin(defaultLlmCallPlugin);
-
-    const expected = makeResponse({ model: "expected-model" });
-    const provider = makeFakeProvider(expected);
-    const args = makeArgs(provider);
-
-    const result = await runPipeline<LLMCallArgs, LLMCallResult>(
-      "llmCall",
-      getMiddlewaresFor("llmCall"),
-      terminal,
-      args,
-      makeCtx(),
-      DEFAULT_TIMEOUTS.llmCall,
-    );
-
-    expect(result).toBe(expected);
-    expect(provider.calls).toHaveLength(1);
-    expect(provider.calls[0]!.messages).toBe(args.messages);
-    expect(provider.calls[0]!.systemPrompt).toBe("you are a helpful assistant");
-  });
-
-  test("spy middleware records the full invocation arguments", async () => {
-    const observed: LLMCallArgs[] = [];
-    const spyPlugin: Plugin = {
-      manifest: {
-        name: "spy-llm",
-        version: "0.0.1",
-      },
-      middleware: {
-        llmCall: async (args, next, _ctx) => {
-          observed.push(args);
-          return next(args);
-        },
-      },
-    };
-
-    registerPlugin(spyPlugin);
-    registerPlugin(defaultLlmCallPlugin);
-
-    const provider = makeFakeProvider();
-    const tools: ToolDefinition[] = [
-      {
-        name: "echo",
-        description: "echoes its input",
-        input_schema: { type: "object" },
-      },
-    ];
-    const args = makeArgs(provider, {
-      options: {
-        tools,
-        systemPrompt: "you are a helpful assistant",
-        config: {},
-      },
-    });
-
-    await runPipeline<LLMCallArgs, LLMCallResult>(
-      "llmCall",
-      getMiddlewaresFor("llmCall"),
-      terminal,
-      args,
-      makeCtx(),
-      DEFAULT_TIMEOUTS.llmCall,
-    );
-
-    expect(observed).toHaveLength(1);
-    expect(observed[0]!.provider).toBe(provider);
-    expect(observed[0]!.messages).toBe(args.messages);
-    expect(observed[0]!.options?.tools).toBe(tools);
-    expect(observed[0]!.options?.systemPrompt).toBe(
-      "you are a helpful assistant",
-    );
-    expect(provider.calls).toHaveLength(1);
-  });
-
-  test("default registered first does not shadow later-registered user middleware", async () => {
-    // The default plugin registers at module load (before `bootstrapPlugins()`
-    // loads user plugins), so it sits at the outermost layer in the onion.
-    // This test registers the default FIRST (matching production ordering)
-    // and asserts that a user-registered spy still runs — confirming that
-    // the outermost middleware forwards via `next(args)` rather than
-    // short-circuiting the chain.
-    const observed: LLMCallArgs[] = [];
-    const spyPlugin: Plugin = {
-      manifest: {
-        name: "spy-llm-after-default",
-        version: "0.0.1",
-      },
-      middleware: {
-        llmCall: async (args, next, _ctx) => {
-          observed.push(args);
-          return next(args);
-        },
-      },
-    };
-
-    registerPlugin(defaultLlmCallPlugin);
-    registerPlugin(spyPlugin);
-
-    const provider = makeFakeProvider();
-    const args = makeArgs(provider);
-
-    await runPipeline<LLMCallArgs, LLMCallResult>(
-      "llmCall",
-      getMiddlewaresFor("llmCall"),
-      terminal,
-      args,
-      makeCtx(),
-      DEFAULT_TIMEOUTS.llmCall,
-    );
-
-    expect(observed).toHaveLength(1);
-    expect(observed[0]!.provider).toBe(provider);
-    expect(provider.calls).toHaveLength(1);
-  });
-
-  test("short-circuit middleware prevents the real provider call", async () => {
-    const synthetic = makeResponse({
-      model: "synthetic-model",
-      content: [{ type: "text", text: "synthesized" }],
-    });
-
-    const shortCircuit: Middleware<LLMCallArgs, LLMCallResult> = async (
-      _args,
-      _next,
-      _ctx,
-    ) => synthetic;
-
-    const shortCircuitPlugin: Plugin = {
-      manifest: {
-        name: "short-circuit-llm",
-        version: "0.0.1",
-      },
-      middleware: { llmCall: shortCircuit },
-    };
-
-    registerPlugin(shortCircuitPlugin);
-    registerPlugin(defaultLlmCallPlugin);
-
-    const provider = makeFakeProvider();
-    const args = makeArgs(provider);
-
-    const result = await runPipeline<LLMCallArgs, LLMCallResult>(
-      "llmCall",
-      getMiddlewaresFor("llmCall"),
-      terminal,
-      args,
-      makeCtx(),
-      DEFAULT_TIMEOUTS.llmCall,
-    );
-
-    expect(result).toBe(synthetic);
-    // The short-circuit middleware never calls `next`, so the terminal and
-    // every downstream middleware (including the default) are skipped and
-    // the provider is never contacted.
-    expect(provider.calls).toHaveLength(0);
-  });
-});