@vellumai/assistant 0.8.7 → 0.8.8

package/src/__tests__/title-generate-pipeline.test.ts

@@ -1,211 +0,0 @@
-/**
- * Tests for the `titleGenerate` pipeline (PR 28).
- *
- * The title-generation side effect used to be a direct call to
- * `queueGenerateConversationTitle` inside `conversation-agent-loop.ts`.
- * After PR 28 the assistant routes that call through the plugin pipeline
- * runner, giving plugins an opportunity to observe/replace the default
- * implementation.
- *
- * Covers:
- * - The default plugin's terminal delegates to
- *   `queueGenerateConversationTitle` with the same arguments the agent
- *   loop constructs.
- * - A custom plugin can install a short-circuit middleware that replaces
- *   the terminal with a deterministic generator. The default terminal is
- *   NOT invoked in that case.
- *
- * Mocks `memory/conversation-title-service.js` so the tests don't touch
- * the real provider stack, and resets the plugin registry between cases.
- */
-
-import { beforeEach, describe, expect, mock, test } from "bun:test";
-
-// Stub the title-generation service before importing anything that binds
-// to it, so both the default plugin and the agent loop capture the
-// stubbed binding.
-const queueGenerateConversationTitleMock = mock(
-  (_params: {
-    conversationId: string;
-    provider?: unknown;
-    userMessage?: string;
-    onTitleUpdated?: (title: string) => void;
-  }): void => undefined,
-);
-mock.module("../memory/conversation-title-service.js", () => ({
-  queueGenerateConversationTitle: queueGenerateConversationTitleMock,
-}));
-
-import { defaultTitleGenerateTerminal } from "../plugins/defaults/title-generate/terminal.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import type {
-  Middleware,
-  Plugin,
-  TitleArgs,
-  TitleResult,
-  TurnContext,
-} from "../plugins/types.js";
-
-function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
-  return {
-    requestId: "req-1",
-    conversationId: "conv-1",
-    turnIndex: 0,
-    trust: { sourceChannel: "vellum", trustClass: "unknown" },
-    ...overrides,
-  };
-}
-
-function makeArgs(overrides: Partial<TitleArgs> = {}): TitleArgs {
-  return {
-    conversationId: "conv-1",
-    userMessage: "hello world",
-    ...overrides,
-  };
-}
-
-describe("titleGenerate pipeline", () => {
-  beforeEach(() => {
-    resetPluginRegistryForTests();
-    queueGenerateConversationTitleMock.mockReset();
-    queueGenerateConversationTitleMock.mockImplementation(() => undefined);
-    // Re-register the default plugin after the registry reset so tests see
-    // the same shape the daemon sees at runtime.
-    registerPlugin({
-      manifest: {
-        name: "default-title-generate",
-        version: "1.0.0",
-      },
-    });
-  });
-
-  test("default: pipeline terminal queues a title-generation job", async () => {
-    const ctx = makeCtx();
-    const onTitleUpdated = mock((_title: string) => undefined);
-    const args = makeArgs({
-      conversationId: "conv-1",
-      userMessage: "first message",
-      onTitleUpdated,
-    });
-
-    await runPipeline(
-      "titleGenerate",
-      getMiddlewaresFor("titleGenerate"),
-      defaultTitleGenerateTerminal,
-      args,
-      ctx,
-      DEFAULT_TIMEOUTS.titleGenerate,
-    );
-
-    // The default terminal must have delegated to queueGenerateConversationTitle
-    // with every argument the caller supplied, including the callback.
-    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
-    const call = queueGenerateConversationTitleMock.mock.calls[0]?.[0];
-    expect(call?.conversationId).toBe("conv-1");
-    expect(call?.userMessage).toBe("first message");
-    expect(call?.onTitleUpdated).toBe(onTitleUpdated);
-  });
-
-  test("default: pipeline result is the empty object from the terminal", async () => {
-    const ctx = makeCtx();
-    const result = await runPipeline(
-      "titleGenerate",
-      getMiddlewaresFor("titleGenerate"),
-      defaultTitleGenerateTerminal,
-      makeArgs(),
-      ctx,
-      DEFAULT_TIMEOUTS.titleGenerate,
-    );
-    expect(result).toEqual({});
-  });
-
-  test("custom plugin: short-circuit middleware replaces the default with a deterministic generator", async () => {
-    // A custom plugin installs middleware that fabricates a title
-    // deterministically and never calls `next`, so the default terminal
-    // (which would delegate to queueGenerateConversationTitle) is
-    // skipped entirely.
-    const observedTitles: string[] = [];
-
-    const deterministicMw: Middleware<TitleArgs, TitleResult> = async (
-      args,
-    ) => {
-      const fabricated = `[deterministic] ${args.userMessage}`;
-      args.onTitleUpdated?.(fabricated);
-      observedTitles.push(fabricated);
-      return {};
-    };
-
-    const customPlugin: Plugin = {
-      manifest: {
-        name: "custom-deterministic-title",
-        version: "0.0.1",
-      },
-      middleware: { titleGenerate: deterministicMw },
-    };
-    registerPlugin(customPlugin);
-
-    const receivedTitle: string[] = [];
-    const args = makeArgs({
-      userMessage: "what is the weather",
-      onTitleUpdated: (title) => {
-        receivedTitle.push(title);
-      },
-    });
-
-    await runPipeline(
-      "titleGenerate",
-      getMiddlewaresFor("titleGenerate"),
-      defaultTitleGenerateTerminal,
-      args,
-      makeCtx(),
-      DEFAULT_TIMEOUTS.titleGenerate,
-    );
-
-    // Deterministic middleware produced the expected title and invoked
-    // the caller's callback.
-    expect(observedTitles).toEqual(["[deterministic] what is the weather"]);
-    expect(receivedTitle).toEqual(["[deterministic] what is the weather"]);
-    // The default terminal must NOT have been reached — it would have
-    // called the real title-service stub.
-    expect(queueGenerateConversationTitleMock).not.toHaveBeenCalled();
-  });
-
-  test("custom plugin: passthrough middleware leaves the default in charge", async () => {
-    // A plugin that always calls `next` just observes — the default
-    // terminal still runs and queues the title-generation job.
-    let middlewareSawArgs = false;
-
-    const passthroughMw: Middleware<TitleArgs, TitleResult> = async (
-      args,
-      next,
-    ) => {
-      middlewareSawArgs = true;
-      return next(args);
-    };
-
-    registerPlugin({
-      manifest: {
-        name: "observer",
-        version: "0.0.1",
-      },
-      middleware: { titleGenerate: passthroughMw },
-    });
-
-    await runPipeline(
-      "titleGenerate",
-      getMiddlewaresFor("titleGenerate"),
-      defaultTitleGenerateTerminal,
-      makeArgs(),
-      makeCtx(),
-      DEFAULT_TIMEOUTS.titleGenerate,
-    );
-
-    expect(middlewareSawArgs).toBe(true);
-    expect(queueGenerateConversationTitleMock).toHaveBeenCalledTimes(1);
-  });
-});

package/src/__tests__/token-estimate-pipeline.test.ts

@@ -1,479 +0,0 @@
-/**
- * Tests for the `tokenEstimate` plugin pipeline (PR 22 of the
- * agent-plugin-system plan).
- *
- * Covers:
- * - The default plugin's terminal middleware matches
- *   {@link estimatePromptTokens} output exactly across a set of golden
- *   inputs (empty history, text-only, tools, provider-specific image sizing).
- * - Running the pipeline end-to-end with the default registered produces
- *   the same numeric result as calling `estimatePromptTokens` directly.
- * - A custom plugin that short-circuits the chain can override the default,
- *   proving the extension point works.
- * - When a non-1.0 EWMA calibration sample has been recorded, the terminal's
- *   output reflects that correction rather than the raw estimate.
- *
- * These tests exercise the registry + runner directly. They do not touch
- * `bootstrapPlugins` — the default registration path is covered by the
- * bootstrap suite.
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-
-import {
-  recordEstimate,
-  resetCalibrations,
-} from "../context/estimator-calibration.js";
-import {
-  estimatePromptTokens,
-  estimatePromptTokensRaw,
-  estimateToolsTokens,
-} from "../context/token-estimator.js";
-import type { TrustContext } from "../daemon/trust-context.js";
-import { defaultTokenEstimatePlugin } from "../plugins/defaults/token-estimate/register.js";
-import { defaultTokenEstimateTerminal } from "../plugins/defaults/token-estimate/terminal.js";
-import { DEFAULT_TIMEOUTS, runPipeline } from "../plugins/pipeline.js";
-import {
-  getMiddlewaresFor,
-  registerPlugin,
-  resetPluginRegistryForTests,
-} from "../plugins/registry.js";
-import type {
-  EstimateArgs,
-  EstimateResult,
-  Middleware,
-  Plugin,
-  TurnContext,
-} from "../plugins/types.js";
-import type { Message, ToolDefinition } from "../providers/types.js";
-
-// ── Fixtures ─────────────────────────────────────────────────────────────
-
-const trust: TrustContext = {
-  sourceChannel: "vellum",
-  trustClass: "guardian",
-};
-
-function makeCtx(overrides: Partial<TurnContext> = {}): TurnContext {
-  return {
-    requestId: "req-token-estimate-test",
-    conversationId: "conv-token-estimate-test",
-    turnIndex: 0,
-    trust,
-    ...overrides,
-  };
-}
-
-const EMPTY_HISTORY: Message[] = [];
-
-const TEXT_HISTORY: Message[] = [
-  { role: "user", content: [{ type: "text", text: "hello there" }] },
-  {
-    role: "assistant",
-    content: [
-      { type: "text", text: "hi! how can I help you today?" },
-      { type: "text", text: "a second text block for good measure" },
-    ],
-  },
-];
-
-const TOOL_USE_HISTORY: Message[] = [
-  { role: "user", content: [{ type: "text", text: "what's in the log?" }] },
-  {
-    role: "assistant",
-    content: [
-      {
-        type: "tool_use",
-        id: "tu-1",
-        name: "bash",
-        input: { command: "tail -n 5 server.log" },
-      },
-    ],
-  },
-  {
-    role: "user",
-    content: [
-      {
-        type: "tool_result",
-        tool_use_id: "tu-1",
-        content: "line1\nline2\nline3",
-      },
-    ],
-  },
-];
-
-const SYSTEM_PROMPT = "You are a helpful assistant with a long preamble.";
-
-const SAMPLE_TOOLS: ToolDefinition[] = [
-  {
-    name: "bash",
-    description: "Execute a shell command and return its output.",
-    input_schema: {
-      type: "object",
-      properties: { command: { type: "string" } },
-      required: ["command"],
-    },
-  },
-  {
-    name: "file_read",
-    description: "Read a file from the workspace.",
-    input_schema: {
-      type: "object",
-      properties: { path: { type: "string" } },
-      required: ["path"],
-    },
-  },
-];
-
-// ── Helpers ──────────────────────────────────────────────────────────────
-
-function registerDefault(): void {
-  registerPlugin(defaultTokenEstimatePlugin);
-}
-
-function calibratedEstimate(
-  args: Pick<EstimateArgs, "history" | "systemPrompt" | "providerName"> & {
-    tools: ToolDefinition[];
-  },
-): number {
-  const toolTokenBudget =
-    args.tools.length > 0 ? estimateToolsTokens(args.tools) : 0;
-  return estimatePromptTokens(args.history, args.systemPrompt, {
-    providerName: args.providerName,
-    toolTokenBudget,
-  });
-}
-
-function rawEstimate(
-  args: Pick<EstimateArgs, "history" | "systemPrompt" | "providerName"> & {
-    tools: ToolDefinition[];
-  },
-): number {
-  const toolTokenBudget =
-    args.tools.length > 0 ? estimateToolsTokens(args.tools) : 0;
-  return estimatePromptTokensRaw(args.history, args.systemPrompt, {
-    providerName: args.providerName,
-    toolTokenBudget,
-  });
-}
-
-async function runViaPipeline(args: EstimateArgs): Promise<EstimateResult> {
-  return runPipeline<EstimateArgs, EstimateResult>(
-    "tokenEstimate",
-    getMiddlewaresFor("tokenEstimate"),
-    // Mirror the production wiring in `daemon/conversation-agent-loop.ts`:
-    // the default plugin's middleware is a passthrough, so the terminal is
-    // wired in by the call site. Using the same terminal here means the
-    // tests exercise the exact composition shape that ships.
-    defaultTokenEstimateTerminal,
-    args,
-    makeCtx(),
-    DEFAULT_TIMEOUTS.tokenEstimate,
-  );
-}
-
-// ── Tests ────────────────────────────────────────────────────────────────
-
-beforeEach(() => {
-  resetPluginRegistryForTests();
-  resetCalibrations();
-});
-
-afterEach(() => {
-  resetPluginRegistryForTests();
-  resetCalibrations();
-});
-
-describe("tokenEstimate pipeline — default plugin parity", () => {
-  test("default matches estimatePromptTokens on empty history", async () => {
-    registerDefault();
-    const args: EstimateArgs = {
-      history: EMPTY_HISTORY,
-      systemPrompt: undefined,
-      tools: [],
-      providerName: undefined,
-    };
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(calibratedEstimate(args));
-  });
-
-  test("default matches estimatePromptTokens on text-only history", async () => {
-    registerDefault();
-    const args: EstimateArgs = {
-      history: TEXT_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: [],
-      providerName: "anthropic",
-    };
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(calibratedEstimate(args));
-    // Sanity: the system prompt adds real token cost, so the number is
-    // strictly larger than the bare-history estimate.
-    expect(pipelineResult).toBeGreaterThan(
-      calibratedEstimate({
-        history: TEXT_HISTORY,
-        systemPrompt: undefined,
-        tools: [],
-        providerName: "anthropic",
-      }),
-    );
-  });
-
-  test("default matches estimatePromptTokens with tool_use/tool_result blocks", async () => {
-    registerDefault();
-    const args: EstimateArgs = {
-      history: TOOL_USE_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: SAMPLE_TOOLS,
-      providerName: "anthropic",
-    };
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(calibratedEstimate(args));
-  });
-
-  test("default folds tool definition tokens into the result", async () => {
-    registerDefault();
-    const baseArgs: EstimateArgs = {
-      history: TEXT_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: [],
-      providerName: "anthropic",
-    };
-    const withoutTools = await runViaPipeline(baseArgs);
-    const withTools = await runViaPipeline({
-      ...baseArgs,
-      tools: SAMPLE_TOOLS,
-    });
-    // Tools contribute non-zero overhead; the pipeline result must grow.
-    const toolBudget = estimateToolsTokens(SAMPLE_TOOLS);
-    expect(toolBudget).toBeGreaterThan(0);
-    expect(withTools - withoutTools).toBe(toolBudget);
-  });
-
-  test("provider-specific image sizing flows through the default", async () => {
-    registerDefault();
-    // Two providers see different image token costs for the same content —
-    // the raw estimator is the source of truth, so the pipeline must agree
-    // under both provider names.
-    const imageHistory: Message[] = [
-      {
-        role: "user",
-        content: [
-          {
-            type: "image",
-            source: {
-              type: "base64",
-              media_type: "image/png",
-              // Small fake PNG-ish payload; the estimator's fallback path
-              // kicks in when parseImageDimensions fails, which is fine —
-              // the two providers still diverge on overhead.
-              data: "a".repeat(128),
-            },
-          },
-        ],
-      },
-    ];
-    const anthropicArgs: EstimateArgs = {
-      history: imageHistory,
-      systemPrompt: undefined,
-      tools: [],
-      providerName: "anthropic",
-    };
-    const openaiArgs: EstimateArgs = {
-      ...anthropicArgs,
-      providerName: "openai",
-    };
-    const anthropicResult = await runViaPipeline(anthropicArgs);
-    const openaiResult = await runViaPipeline(openaiArgs);
-    expect(anthropicResult).toBe(calibratedEstimate(anthropicArgs));
-    expect(openaiResult).toBe(calibratedEstimate(openaiArgs));
-  });
-});
-
-describe("tokenEstimate pipeline — calibration correction", () => {
-  // Large-ish synthetic history so the raw estimate clears the
-  // MIN_SAMPLE_MAGNITUDE (500) guard in the calibrator — otherwise
-  // `recordEstimate` drops the sample as noise and the correction stays 1.0.
-  const LARGE_TEXT = "lorem ipsum dolor sit amet ".repeat(500);
-  const LARGE_HISTORY: Message[] = [
-    { role: "user", content: [{ type: "text", text: LARGE_TEXT }] },
-  ];
-
-  test("seeded EWMA sample shifts the terminal's output off the raw estimate", async () => {
-    registerDefault();
-    const args: EstimateArgs = {
-      history: LARGE_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: [],
-      providerName: "anthropic",
-    };
-    const raw = rawEstimate(args);
-    // Provider reports ~30% more tokens than we estimated — a plausible
-    // under-count bias. Seed the aggregate (provider, "") key that the
-    // terminal consults.
-    const actual = Math.round(raw * 1.3);
-    recordEstimate("anthropic", "", raw, actual);
-
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(calibratedEstimate(args));
-    expect(pipelineResult).not.toBe(raw);
-    expect(pipelineResult).toBeGreaterThan(raw);
-  });
-});
-
-describe("tokenEstimate pipeline — custom override", () => {
-  test("custom plugin short-circuit returns a different value than the default", async () => {
-    // A plugin that completely replaces the default with a fixed value,
-    // proving plugins can substitute provider-native tokenizers (e.g.
-    // `countTokens`) without touching orchestrator code.
-    const FIXED = 424242;
-    const override: Middleware<EstimateArgs, EstimateResult> = async (
-      _args,
-      _next,
-      _ctx,
-    ) => FIXED;
-    const customPlugin: Plugin = {
-      manifest: {
-        name: "custom-token-estimate",
-        version: "1.0.0",
-      },
-      middleware: { tokenEstimate: override },
-    };
-
-    // Register the custom plugin FIRST so it sits outermost and short-
-    // circuits before the default's terminal runs.
-    registerPlugin(customPlugin);
-    registerDefault();
-
-    const args: EstimateArgs = {
-      history: TEXT_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: SAMPLE_TOOLS,
-      providerName: "anthropic",
-    };
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(FIXED);
-    // And for contrast: the default alone would have given the calibrated value.
-    expect(pipelineResult).not.toBe(calibratedEstimate(args));
-  });
-
-  test("wrapper middleware that scales the downstream result composes with the default", async () => {
-    // A plugin that wraps the downstream estimate, doubling it. This
-    // exercises the onion composition: outer middleware sees the raw
-    // default result and returns its own modification.
-    const doubler: Middleware<EstimateArgs, EstimateResult> = async (
-      args,
-      next,
-      _ctx,
-    ) => {
-      const inner = await next(args);
-      return inner * 2;
-    };
-    const wrapperPlugin: Plugin = {
-      manifest: {
-        name: "doubling-token-estimate",
-        version: "1.0.0",
-      },
-      middleware: { tokenEstimate: doubler },
-    };
-
-    registerPlugin(wrapperPlugin);
-    registerDefault();
-
-    const args: EstimateArgs = {
-      history: TEXT_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: SAMPLE_TOOLS,
-      providerName: "anthropic",
-    };
-    const pipelineResult = await runViaPipeline(args);
-    expect(pipelineResult).toBe(calibratedEstimate(args) * 2);
-  });
-});
-
-describe("tokenEstimate pipeline — default does not shadow late plugins", () => {
-  test("user middleware registered AFTER the default still runs", async () => {
-    // Regression test for the default-first shadowing hazard: defaults are
-    // registered before user plugins in `bootstrapPlugins()`, putting the
-    // default at the OUTERMOST onion position. If the default middleware
-    // runs the estimate directly instead of calling `next(args)`, any user
-    // plugin loaded afterward is invisible. The default is a passthrough —
-    // this test fails loudly if that invariant ever regresses.
-    registerDefault();
-    const observed: EstimateArgs[] = [];
-    const observer: Middleware<EstimateArgs, EstimateResult> = async (
-      args,
-      next,
-      _ctx,
-    ) => {
-      observed.push(args);
-      // Return a sentinel so we can distinguish the observer's result from
-      // the default's output.
-      await next(args);
-      return 999_999;
-    };
-    const userPlugin: Plugin = {
-      manifest: {
-        name: "late-registered-observer",
-        version: "1.0.0",
-      },
-      middleware: { tokenEstimate: observer },
-    };
-    registerPlugin(userPlugin);
-
-    const args: EstimateArgs = {
-      history: TEXT_HISTORY,
-      systemPrompt: SYSTEM_PROMPT,
-      tools: [],
-      providerName: "anthropic",
-    };
-    const result = await runViaPipeline(args);
-    expect(observed.length).toBe(1);
-    expect(result).toBe(999_999);
-  });
-});
-
-describe("tokenEstimate pipeline — args are immutable to middleware", () => {
-  test("frozen history/tools reject in-place mutation attempts", () => {
-    // The call site freezes shallow clones of `history` and `tools` before
-    // handing them to the pipeline. This mirrors the runtime protection
-    // that stops a misbehaving middleware from trimming `args.history` in
-    // place — which would silently drop prompt context from the
-    // orchestrator's live `runMessages` array before the provider call.
-    const frozenHistory = Object.freeze([...TEXT_HISTORY]);
-    const frozenTools = Object.freeze([...SAMPLE_TOOLS]);
-    expect(() => {
-      (frozenHistory as Message[]).pop();
-    }).toThrow(TypeError);
-    expect(() => {
-      (frozenTools as ToolDefinition[]).push({
-        name: "extra",
-        description: "",
-        input_schema: { type: "object", properties: {} },
-      });
-    }).toThrow(TypeError);
-  });
-});
-
-describe("tokenEstimate pipeline — empty registry fallback", () => {
-  test("without any plugin registered, the terminal receives the call", async () => {
-    // `runViaPipeline` uses a throwing terminal, so here we run the
-    // pipeline with an explicit terminal that returns a sentinel to prove
-    // that an empty middleware list falls through.
-    const SENTINEL = 12345;
-    const result = await runPipeline<EstimateArgs, EstimateResult>(
-      "tokenEstimate",
-      getMiddlewaresFor("tokenEstimate"),
-      async () => SENTINEL,
-      {
-        history: TEXT_HISTORY,
-        systemPrompt: SYSTEM_PROMPT,
-        tools: [],
-        providerName: "anthropic",
-      },
-      makeCtx(),
-      DEFAULT_TIMEOUTS.tokenEstimate,
-    );
-    expect(result).toBe(SENTINEL);
-  });
-});