@vellumai/assistant 0.9.0 → 0.10.0-staging.2

package/src/plugins/defaults/advisor/__tests__/advisor-state-store.test.ts

@@ -0,0 +1,43 @@
+import { afterEach, describe, expect, test } from "bun:test";
+
+import type { Message } from "../../../../providers/types.js";
+import {
+  getCapture,
+  recordMessages,
+  recordSystemPrompt,
+  resetAdvisorStateForTests,
+  seedCapture,
+} from "../advisor-state-store.js";
+
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+afterEach(() => {
+  resetAdvisorStateForTests();
+});
+
+describe("advisor state store", () => {
+  test("records system prompt and messages independently per conversation", () => {
+    recordSystemPrompt("c1", "system A");
+    recordMessages("c1", [userMsg("hello")]);
+    recordSystemPrompt("c2", "system B");
+
+    expect(getCapture("c1")?.systemPrompt).toBe("system A");
+    expect(getCapture("c1")?.messages).toEqual([userMsg("hello")]);
+    expect(getCapture("c2")?.systemPrompt).toBe("system B");
+    expect(getCapture("c2")?.messages).toEqual([]);
+  });
+
+  test("seedCapture snapshots (copies) the array", () => {
+    const live: Message[] = [userMsg("one")];
+    seedCapture("c1", live);
+    live.push(userMsg("two"));
+    expect(getCapture("c1")?.messages).toEqual([userMsg("one")]);
+  });
+
+  test("getCapture returns undefined for an unseen conversation", () => {
+    expect(getCapture("nope")).toBeUndefined();
+  });
+});

package/src/plugins/defaults/advisor/__tests__/agent-loop-integration.test.ts

@@ -0,0 +1,137 @@
+/**
+ * End-to-end integration test: drives the REAL agent loop with all first-party
+ * defaults registered (so the advisor plugin's hooks + tool are live), and lets
+ * the REAL consult run. Only the provider boundary is stubbed:
+ *   - the executor's provider (a scripted mock provider), and
+ *   - `getConfiguredProvider`, so the advisor sub-call resolves to a second
+ *     scripted mock provider instead of a configured one.
+ *
+ * The advisor's inference genuinely runs through `consult` → `sendMessage`
+ * against the injected advisor provider, exercising the full hook-capture →
+ * tool.execute → consult → routed-inference path through the loop.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import {
+  createMockProvider,
+  textResponse,
+} from "../../../../__tests__/helpers/mock-provider.js";
+import { AgentLoop } from "../../../../agent/loop.js";
+import type {
+  ContentBlock,
+  Message,
+  ProviderResponse,
+} from "../../../../providers/types.js";
+
+const ADVICE = "ADVICE: use a channel-based worker pool with graceful drain.";
+let advisorProvider: ReturnType<typeof createMockProvider> | null = null;
+
+const realProviderSendMessage =
+  await import("../../../../providers/provider-send-message.js");
+mock.module("../../../../providers/provider-send-message.js", () => ({
+  ...realProviderSendMessage,
+  getConfiguredProvider: async () => advisorProvider?.provider ?? null,
+}));
+
+const { resetPluginRegistryAndRegisterDefaults } =
+  await import("../../index.js");
+const advisorTool = (await import("../tools/advisor.js")).default;
+
+const userTurn: Message = {
+  role: "user",
+  content: [{ type: "text", text: "build a worker pool" }],
+};
+
+function textOf(content: ReadonlyArray<ContentBlock>): string {
+  return content.map((b) => (b.type === "text" ? b.text : "")).join("");
+}
+
+describe("advisor — agent-loop integration", () => {
+  beforeEach(() => {
+    resetPluginRegistryAndRegisterDefaults();
+    advisorProvider = createMockProvider([textResponse(ADVICE)]);
+  });
+
+  test("model calls advisor → hooks capture → consult routes through inference → advice returns", async () => {
+    const consultThenText: ProviderResponse = {
+      content: [
+        { type: "text", text: "Let me consult the advisor." },
+        { type: "tool_use", id: "call-1", name: "advisor", input: {} },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const { provider } = createMockProvider([
+      consultThenText,
+      textResponse("Done — implemented the worker pool."),
+    ]);
+
+    const conversationId = "advisor-itest";
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "You are a coding agent.",
+      conversationId,
+      tools: [
+        {
+          name: "advisor",
+          description: "",
+          input_schema: { type: "object", properties: {} },
+        },
+      ],
+      toolExecutor: async (name, input) =>
+        name === "advisor"
+          ? advisorTool.execute!(input, { conversationId } as never)
+          : { content: `unknown tool ${name}`, isError: true },
+    });
+
+    const { history } = await loop.run({
+      requestId: "req-1",
+      messages: [userTurn],
+      onEvent: () => {},
+      callSite: "mainAgent",
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The advisor sub-call happened, routed through the dedicated advisor call site.
+    expect(advisorProvider!.calls).toHaveLength(1);
+    const sub = advisorProvider!.calls[0];
+    expect(sub.options?.config?.callSite).toBe("advisor");
+    // No `advisorProfile` configured in the default test config, so no override
+    // is passed and the `advisor` call site resolves its own default profile.
+    expect(sub.options?.config?.overrideProfile).toBeUndefined();
+    expect(sub.options?.config?.tool_choice).toEqual({ type: "none" });
+    // No advisor-specific output cap — the resolver applies the profile budget.
+    expect(sub.options?.config?.max_tokens).toBeUndefined();
+
+    // The advisor saw the captured transcript (task present; pending tool_use stripped).
+    expect(textOf(sub.messages[0].content)).toContain("build a worker pool");
+    expect(textOf(sub.messages[sub.messages.length - 1].content)).toContain(
+      "focused strategic guidance",
+    );
+    // It also saw the model's CURRENT turn — the text it wrote right before the
+    // `advisor` tool_use — which `post-model-call` lifts out of `ctx.content`.
+    const transcript = sub.messages.map((m) => textOf(m.content)).join("\n");
+    expect(transcript).toContain("Let me consult the advisor.");
+
+    // The advisor saw the executor's system prompt (via pre-model-call).
+    expect(sub.options?.systemPrompt).toContain("senior advisor");
+    expect(sub.options?.systemPrompt).toContain("You are a coding agent.");
+
+    // The advice flowed back into the executor's history as the tool result.
+    const toolResult = history
+      .flatMap((m) => m.content)
+      .find((b) => b.type === "tool_result");
+    expect((toolResult as { content: string }).content).toContain(
+      "channel-based worker pool",
+    );
+
+    // The loop completed with the final answer.
+    const lastAssistant = [...history]
+      .reverse()
+      .find((m) => m.role === "assistant")!;
+    expect(textOf(lastAssistant.content)).toContain(
+      "implemented the worker pool",
+    );
+  });
+});

package/src/plugins/defaults/advisor/__tests__/consult.test.ts

@@ -0,0 +1,153 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { Message } from "../../../../providers/types.js";
+
+// Stub the provider resolution; spread the real module so `extractAllText` /
+// `userMessage` (which the consult also uses) keep working.
+let sendMessageArgs: Record<string, unknown> | null = null;
+let responseText = "Use a channel-based worker pool; drain on shutdown.";
+let sendMessageError: Error | null = null;
+let providerResolves = true;
+
+const fakeProvider = {
+  name: "mock-advisor-provider",
+  async sendMessage(messages: unknown, options: unknown) {
+    sendMessageArgs = { messages, options } as Record<string, unknown>;
+    if (sendMessageError) throw sendMessageError;
+    return {
+      content: [{ type: "text", text: responseText }],
+      model: "mock-model",
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: "end_turn",
+    };
+  },
+};
+
+const realPsm = await import("../../../../providers/provider-send-message.js");
+mock.module("../../../../providers/provider-send-message.js", () => ({
+  ...realPsm,
+  getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
+}));
+
+const { consultAdvisor } = await import("../consult.js");
+const advisorTool = (await import("../tools/advisor.js")).default;
+const { recordSystemPrompt, recordMessages, resetAdvisorStateForTests } =
+  await import("../advisor-state-store.js");
+
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+function optionConfig(): Record<string, unknown> {
+  const options = sendMessageArgs?.options as Record<string, unknown>;
+  return options.config as Record<string, unknown>;
+}
+
+beforeEach(() => {
+  sendMessageArgs = null;
+  responseText = "Use a channel-based worker pool; drain on shutdown.";
+  sendMessageError = null;
+  providerResolves = true;
+  resetAdvisorStateForTests();
+});
+
+describe("consultAdvisor", () => {
+  test("routes through the advisor call site, tools off, returns advice", async () => {
+    const messages: Message[] = [
+      userMsg("build a worker pool"),
+      {
+        role: "assistant",
+        content: [
+          { type: "thinking", thinking: "secret", signature: "s" },
+          { type: "text", text: "let me consult the advisor" },
+          { type: "tool_use", id: "t1", name: "advisor", input: {} },
+        ],
+      },
+    ];
+
+    const advice = await consultAdvisor({
+      systemPrompt: "You are a coding agent.",
+      messages,
+    });
+
+    expect(advice).toBe(responseText);
+
+    const config = optionConfig();
+    expect(config.callSite).toBe("advisor");
+    // No `advisorProfile` is configured in the default test config, so the
+    // consult passes no override and the `advisor` call site resolves to its
+    // default profile (`quality-optimized`).
+    expect(config.overrideProfile).toBeUndefined();
+    expect(config.tool_choice).toEqual({ type: "none" });
+    // No advisor-specific output cap — the resolver applies the profile budget.
+    expect(config.max_tokens).toBeUndefined();
+
+    const sent = sendMessageArgs?.messages as Message[];
+    expect(sent[0]).toEqual(userMsg("build a worker pool"));
+    expect(sent[1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "let me consult the advisor" }],
+    });
+    const lastText = (sent[sent.length - 1].content[0] as { text: string })
+      .text;
+    expect(lastText).toContain("focused strategic guidance");
+    // The request carries no word limit.
+    expect(lastText).not.toContain("words");
+
+    const options = sendMessageArgs?.options as { systemPrompt: string };
+    expect(options.systemPrompt).toContain("senior advisor");
+    expect(options.systemPrompt).toContain("You are a coding agent.");
+  });
+
+  test("soft-fails when no provider is configured", async () => {
+    providerResolves = false;
+    const advice = await consultAdvisor({
+      systemPrompt: null,
+      messages: [userMsg("hi")],
+    });
+    expect(advice).toContain("no inference provider");
+  });
+
+  test("returns a notice when there is no usable transcript", async () => {
+    const advice = await consultAdvisor({ systemPrompt: null, messages: [] });
+    expect(advice).toContain("no conversation context");
+    expect(sendMessageArgs).toBeNull();
+  });
+
+  test("falls back to a notice when the advisor returns blank text", async () => {
+    responseText = "   ";
+    const advice = await consultAdvisor({
+      systemPrompt: null,
+      messages: [userMsg("hi")],
+    });
+    expect(advice).toContain("no guidance");
+  });
+});
+
+describe("advisor tool.execute", () => {
+  test("reads the captured transcript and returns guidance as a non-error result", async () => {
+    recordSystemPrompt("c1", "You are a coding agent.");
+    recordMessages("c1", [userMsg("build a worker pool")]);
+
+    const result = await advisorTool.execute?.({}, {
+      conversationId: "c1",
+    } as never);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.content).toBe(responseText);
+  });
+
+  test("degrades to a benign result (never throws) when the consult fails", async () => {
+    recordMessages("c2", [userMsg("hi")]);
+    sendMessageError = new Error("kaboom");
+
+    const result = await advisorTool.execute?.({}, {
+      conversationId: "c2",
+    } as never);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.content).toContain("advisor unavailable");
+    expect(result?.content).toContain("kaboom");
+  });
+});

package/src/plugins/defaults/advisor/__tests__/hooks.test.ts

@@ -0,0 +1,138 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type {
+  Message,
+  PostModelCallContext,
+  PreModelCallContext,
+  UserPromptSubmitContext,
+} from "@vellumai/plugin-api";
+
+import {
+  getCapture,
+  resetAdvisorStateForTests,
+} from "../advisor-state-store.js";
+import postModelCall from "../hooks/post-model-call.js";
+import preModelCall from "../hooks/pre-model-call.js";
+import userPromptSubmit from "../hooks/user-prompt-submit.js";
+import { STEERING_MARKER } from "../steering.js";
+
+const logger = { info() {}, warn() {}, error() {}, debug() {} };
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+beforeEach(() => {
+  resetAdvisorStateForTests();
+});
+
+describe("pre-model-call hook", () => {
+  test("records the original system prompt and injects steering on mainAgent", async () => {
+    const ctx = {
+      conversationId: "c1",
+      callSite: "mainAgent",
+      systemPrompt: "BASE PROMPT",
+      deferAssistantOutput: false,
+      logger,
+    } as unknown as PreModelCallContext;
+
+    await preModelCall(ctx);
+
+    expect(ctx.systemPrompt).toContain(STEERING_MARKER);
+    expect(ctx.systemPrompt?.startsWith("BASE PROMPT")).toBe(true);
+    expect(getCapture("c1")?.systemPrompt).toBe("BASE PROMPT");
+  });
+
+  test("ignores non-mainAgent calls", async () => {
+    const ctx = {
+      conversationId: "bg",
+      callSite: "inference",
+      systemPrompt: "BG",
+      deferAssistantOutput: false,
+      logger,
+    } as unknown as PreModelCallContext;
+
+    await preModelCall(ctx);
+    expect(ctx.systemPrompt).toBe("BG");
+    expect(getCapture("bg")).toBeUndefined();
+  });
+});
+
+describe("post-model-call hook", () => {
+  const base = {
+    content: [],
+    stopReason: "end_turn",
+    decision: "stop" as const,
+    logger,
+  };
+
+  test("snapshots the transcript on mainAgent", async () => {
+    const messages = [userMsg("hi")];
+    await postModelCall({
+      ...base,
+      conversationId: "c1",
+      callSite: "mainAgent",
+      messages,
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c1")?.messages).toEqual(messages);
+  });
+
+  test("appends the model's current turn (ctx.content) to the snapshot", async () => {
+    const messages = [userMsg("task")];
+    const assistantTurn: Message = {
+      role: "assistant",
+      content: [
+        { type: "text", text: "thinking about it" },
+        { type: "tool_use", id: "t1", name: "advisor", input: {} },
+      ],
+    };
+    await postModelCall({
+      ...base,
+      content: assistantTurn.content,
+      conversationId: "c4",
+      callSite: "mainAgent",
+      messages,
+    } as unknown as PostModelCallContext);
+    const captured = getCapture("c4")?.messages;
+    expect(captured).toHaveLength(2);
+    expect(captured?.[1]).toEqual(assistantTurn);
+  });
+
+  test("ignores the advisor's own inference sub-call (recursion safety) and errors", async () => {
+    await postModelCall({
+      ...base,
+      conversationId: "c2",
+      callSite: "inference",
+      messages: [userMsg("hi")],
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c2")).toBeUndefined();
+
+    await postModelCall({
+      ...base,
+      conversationId: "c3",
+      callSite: "mainAgent",
+      messages: [userMsg("hi")],
+      error: new Error("boom"),
+      stopReason: null,
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c3")).toBeUndefined();
+  });
+});
+
+describe("user-prompt-submit hook", () => {
+  test("seeds the capture with the inbound history", async () => {
+    const messages = [userMsg("task")];
+    await userPromptSubmit({
+      conversationId: "c1",
+      latestMessages: messages,
+      originalMessages: messages,
+      prompt: "task",
+      userMessageId: "u1",
+      requestId: "r1",
+      modelProfileKey: null,
+      isNonInteractive: false,
+      logger,
+    } as unknown as UserPromptSubmitContext);
+    expect(getCapture("c1")?.messages).toEqual(messages);
+  });
+});

package/src/plugins/defaults/advisor/__tests__/transcript.test.ts

@@ -0,0 +1,147 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ContentBlock, Message } from "../../../../providers/types.js";
+import { toAdvisorMessages } from "../transcript.js";
+
+const text = (t: string): ContentBlock => ({ type: "text", text: t });
+
+describe("toAdvisorMessages", () => {
+  test("drops thinking and redacted-thinking blocks", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("do the task")] },
+      {
+        role: "assistant",
+        content: [
+          { type: "thinking", thinking: "secret", signature: "sig" },
+          { type: "redacted_thinking", data: "blob" },
+          text("here is my answer"),
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out).toHaveLength(2);
+    expect(out[1].content).toEqual([text("here is my answer")]);
+  });
+
+  test("strips the pending advisor tool_use from the final assistant turn", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("task")] },
+      {
+        role: "assistant",
+        content: [
+          text("let me consult the advisor"),
+          { type: "tool_use", id: "t1", name: "advisor", input: {} },
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[1].content).toEqual([text("let me consult the advisor")]);
+  });
+
+  test("preserves completed client tool_use / tool_result pairs in earlier turns", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("task")] },
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: "a", name: "bash", input: {} }],
+      },
+      {
+        role: "user",
+        content: [{ type: "tool_result", tool_use_id: "a", content: "output" }],
+      },
+      { role: "assistant", content: [text("done")] },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[1].content[0]).toEqual({
+      type: "tool_use",
+      id: "a",
+      name: "bash",
+      input: {},
+    });
+    expect(out[2].content[0]).toEqual({
+      type: "tool_result",
+      tool_use_id: "a",
+      content: "output",
+    });
+  });
+
+  test("drops a web-search server_tool_use AND its result together — no orphan", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("look it up")] },
+      {
+        role: "assistant",
+        content: [
+          text("searching"),
+          { type: "server_tool_use", id: "ws1", name: "web_search", input: {} },
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "web_search_tool_result",
+            tool_use_id: "ws1",
+            content: [{ title: "x", url: "y" }],
+          },
+        ],
+      },
+      { role: "assistant", content: [text("found it; here is the answer")] },
+    ];
+    const out = toAdvisorMessages(messages);
+    const flat = out.flatMap((m) => m.content);
+    expect(flat.some((b) => b.type === "server_tool_use")).toBe(false);
+    expect(flat.some((b) => b.type === "web_search_tool_result")).toBe(false);
+    expect(flat).toContainEqual(text("searching"));
+    expect(flat).toContainEqual(text("found it; here is the answer"));
+  });
+
+  test("keeps top-level image blocks", () => {
+    const img: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "x" },
+    };
+    const messages: Message[] = [
+      { role: "user", content: [text("look at this"), img] },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[0].content).toEqual([text("look at this"), img]);
+  });
+
+  test("keeps image contentBlocks on a tool_result, dropping disallowed ones", () => {
+    const img: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "x" },
+    };
+    const messages: Message[] = [
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "a",
+            content: "screenshot",
+            contentBlocks: [
+              img,
+              {
+                type: "file",
+                source: {
+                  type: "base64",
+                  media_type: "application/pdf",
+                  data: "z",
+                  filename: "f.pdf",
+                },
+              },
+            ],
+          },
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[0].content[0]).toEqual({
+      type: "tool_result",
+      tool_use_id: "a",
+      content: "screenshot",
+      contentBlocks: [img],
+    });
+  });
+});

package/src/plugins/defaults/advisor/advisor-gate.ts

@@ -0,0 +1,29 @@
+/**
+ * Per-chat-profile gate for the advisor.
+ *
+ * The advisor is active or not depending on the chat profile the turn routes
+ * to (`ProfileEntry.advisorEnabled`). Default-on: the advisor runs unless a
+ * profile sets `advisorEnabled: false` explicitly — absent/null both mean on,
+ * preserving the prior always-on behavior for profiles that never set it.
+ */
+
+import { resolveDefaultProfileKey } from "../../../config/llm-resolver.js";
+import { getConfig } from "../../../config/loader.js";
+
+/**
+ * Whether the advisor is enabled for the chat profile this turn uses.
+ *
+ * `modelProfile` is the call's already-resolved profile — `PreModelCallContext.modelProfile`
+ * from the steering hook, or `ToolContext.overrideProfile` from the tool. Pass
+ * `null` when no per-turn override applies, in which case the workspace active
+ * profile — or the `mainAgent` call-site default — applies.
+ */
+export function advisorEnabledForProfile(modelProfile: string | null): boolean {
+  const { llm } = getConfig();
+  const key =
+    modelProfile ??
+    llm.activeProfile ??
+    resolveDefaultProfileKey("mainAgent", llm);
+  const entry = key ? llm.profiles[key] : undefined;
+  return entry?.advisorEnabled !== false;
+}