@vellumai/assistant 0.9.0-dev.202606181513.fd39213 → 0.9.0-dev.202606181735.ae6e1d8

package/src/plugins/defaults/advisor/__tests__/agent-loop-integration.test.ts

@@ -0,0 +1,134 @@
+/**
+ * End-to-end integration test: drives the REAL agent loop with all first-party
+ * defaults registered (so the advisor plugin's hooks + tool are live), and lets
+ * the REAL consult run. Only the provider boundary is stubbed:
+ *   - the executor's provider (a scripted mock provider), and
+ *   - `getConfiguredProvider`, so the advisor sub-call resolves to a second
+ *     scripted mock provider instead of a configured one.
+ *
+ * The advisor's inference genuinely runs through `consult` → `sendMessage`
+ * against the injected advisor provider, exercising the full hook-capture →
+ * tool.execute → consult → routed-inference path through the loop.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import {
+  createMockProvider,
+  textResponse,
+} from "../../../../__tests__/helpers/mock-provider.js";
+import { AgentLoop } from "../../../../agent/loop.js";
+import type {
+  ContentBlock,
+  Message,
+  ProviderResponse,
+} from "../../../../providers/types.js";
+
+const ADVICE = "ADVICE: use a channel-based worker pool with graceful drain.";
+let advisorProvider: ReturnType<typeof createMockProvider> | null = null;
+
+const realProviderSendMessage =
+  await import("../../../../providers/provider-send-message.js");
+mock.module("../../../../providers/provider-send-message.js", () => ({
+  ...realProviderSendMessage,
+  getConfiguredProvider: async () => advisorProvider?.provider ?? null,
+}));
+
+const { resetPluginRegistryAndRegisterDefaults } =
+  await import("../../index.js");
+const advisorTool = (await import("../tools/advisor.js")).default;
+
+const userTurn: Message = {
+  role: "user",
+  content: [{ type: "text", text: "build a worker pool" }],
+};
+
+function textOf(content: ReadonlyArray<ContentBlock>): string {
+  return content.map((b) => (b.type === "text" ? b.text : "")).join("");
+}
+
+describe("advisor — agent-loop integration", () => {
+  beforeEach(() => {
+    resetPluginRegistryAndRegisterDefaults();
+    advisorProvider = createMockProvider([textResponse(ADVICE)]);
+  });
+
+  test("model calls advisor → hooks capture → consult routes through inference → advice returns", async () => {
+    const consultThenText: ProviderResponse = {
+      content: [
+        { type: "text", text: "Let me consult the advisor." },
+        { type: "tool_use", id: "call-1", name: "advisor", input: {} },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const { provider } = createMockProvider([
+      consultThenText,
+      textResponse("Done — implemented the worker pool."),
+    ]);
+
+    const conversationId = "advisor-itest";
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "You are a coding agent.",
+      conversationId,
+      tools: [
+        {
+          name: "advisor",
+          description: "",
+          input_schema: { type: "object", properties: {} },
+        },
+      ],
+      toolExecutor: async (name, input) =>
+        name === "advisor"
+          ? advisorTool.execute!(input, { conversationId } as never)
+          : { content: `unknown tool ${name}`, isError: true },
+    });
+
+    const { history } = await loop.run({
+      requestId: "req-1",
+      messages: [userTurn],
+      onEvent: () => {},
+      callSite: "mainAgent",
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The advisor sub-call happened, routed through the inference call site.
+    expect(advisorProvider!.calls).toHaveLength(1);
+    const sub = advisorProvider!.calls[0];
+    expect(sub.options?.config?.callSite).toBe("inference");
+    expect(sub.options?.config?.overrideProfile).toBe("quality-optimized");
+    expect(sub.options?.config?.tool_choice).toEqual({ type: "none" });
+    expect(sub.options?.config?.max_tokens).toBe(2048);
+
+    // The advisor saw the captured transcript (task present; pending tool_use stripped).
+    expect(textOf(sub.messages[0].content)).toContain("build a worker pool");
+    expect(textOf(sub.messages[sub.messages.length - 1].content)).toContain(
+      "80 words",
+    );
+    // It also saw the model's CURRENT turn — the text it wrote right before the
+    // `advisor` tool_use — which `post-model-call` lifts out of `ctx.content`.
+    const transcript = sub.messages.map((m) => textOf(m.content)).join("\n");
+    expect(transcript).toContain("Let me consult the advisor.");
+
+    // The advisor saw the executor's system prompt (via pre-model-call).
+    expect(sub.options?.systemPrompt).toContain("senior technical advisor");
+    expect(sub.options?.systemPrompt).toContain("You are a coding agent.");
+
+    // The advice flowed back into the executor's history as the tool result.
+    const toolResult = history
+      .flatMap((m) => m.content)
+      .find((b) => b.type === "tool_result");
+    expect((toolResult as { content: string }).content).toContain(
+      "channel-based worker pool",
+    );
+
+    // The loop completed with the final answer.
+    const lastAssistant = [...history]
+      .reverse()
+      .find((m) => m.role === "assistant")!;
+    expect(textOf(lastAssistant.content)).toContain(
+      "implemented the worker pool",
+    );
+  });
+});

package/src/plugins/defaults/advisor/__tests__/consult.test.ts

@@ -0,0 +1,147 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { Message } from "../../../../providers/types.js";
+
+// Stub the provider resolution; spread the real module so `extractAllText` /
+// `userMessage` (which the consult also uses) keep working.
+let sendMessageArgs: Record<string, unknown> | null = null;
+let responseText = "Use a channel-based worker pool; drain on shutdown.";
+let sendMessageError: Error | null = null;
+let providerResolves = true;
+
+const fakeProvider = {
+  name: "mock-advisor-provider",
+  async sendMessage(messages: unknown, options: unknown) {
+    sendMessageArgs = { messages, options } as Record<string, unknown>;
+    if (sendMessageError) throw sendMessageError;
+    return {
+      content: [{ type: "text", text: responseText }],
+      model: "mock-model",
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: "end_turn",
+    };
+  },
+};
+
+const realPsm = await import("../../../../providers/provider-send-message.js");
+mock.module("../../../../providers/provider-send-message.js", () => ({
+  ...realPsm,
+  getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
+}));
+
+const { consultAdvisor } = await import("../consult.js");
+const advisorTool = (await import("../tools/advisor.js")).default;
+const { recordSystemPrompt, recordMessages, resetAdvisorStateForTests } =
+  await import("../advisor-state-store.js");
+
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+function optionConfig(): Record<string, unknown> {
+  const options = sendMessageArgs?.options as Record<string, unknown>;
+  return options.config as Record<string, unknown>;
+}
+
+beforeEach(() => {
+  sendMessageArgs = null;
+  responseText = "Use a channel-based worker pool; drain on shutdown.";
+  sendMessageError = null;
+  providerResolves = true;
+  resetAdvisorStateForTests();
+});
+
+describe("consultAdvisor", () => {
+  test("routes through the inference call site, tools off, capped, returns advice", async () => {
+    const messages: Message[] = [
+      userMsg("build a worker pool"),
+      {
+        role: "assistant",
+        content: [
+          { type: "thinking", thinking: "secret", signature: "s" },
+          { type: "text", text: "let me consult the advisor" },
+          { type: "tool_use", id: "t1", name: "advisor", input: {} },
+        ],
+      },
+    ];
+
+    const advice = await consultAdvisor({
+      systemPrompt: "You are a coding agent.",
+      messages,
+    });
+
+    expect(advice).toBe(responseText);
+
+    const config = optionConfig();
+    expect(config.callSite).toBe("inference");
+    expect(config.overrideProfile).toBe("quality-optimized");
+    expect(config.tool_choice).toEqual({ type: "none" });
+    expect(config.max_tokens).toBe(2048);
+
+    const sent = sendMessageArgs?.messages as Message[];
+    expect(sent[0]).toEqual(userMsg("build a worker pool"));
+    expect(sent[1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "let me consult the advisor" }],
+    });
+    const lastText = (sent[sent.length - 1].content[0] as { text: string })
+      .text;
+    expect(lastText).toContain("80 words");
+
+    const options = sendMessageArgs?.options as { systemPrompt: string };
+    expect(options.systemPrompt).toContain("senior technical advisor");
+    expect(options.systemPrompt).toContain("You are a coding agent.");
+  });
+
+  test("soft-fails when no provider is configured", async () => {
+    providerResolves = false;
+    const advice = await consultAdvisor({
+      systemPrompt: null,
+      messages: [userMsg("hi")],
+    });
+    expect(advice).toContain("no inference provider");
+  });
+
+  test("returns a notice when there is no usable transcript", async () => {
+    const advice = await consultAdvisor({ systemPrompt: null, messages: [] });
+    expect(advice).toContain("no conversation context");
+    expect(sendMessageArgs).toBeNull();
+  });
+
+  test("falls back to a notice when the advisor returns blank text", async () => {
+    responseText = "   ";
+    const advice = await consultAdvisor({
+      systemPrompt: null,
+      messages: [userMsg("hi")],
+    });
+    expect(advice).toContain("no guidance");
+  });
+});
+
+describe("advisor tool.execute", () => {
+  test("reads the captured transcript and returns guidance as a non-error result", async () => {
+    recordSystemPrompt("c1", "You are a coding agent.");
+    recordMessages("c1", [userMsg("build a worker pool")]);
+
+    const result = await advisorTool.execute?.({}, {
+      conversationId: "c1",
+    } as never);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.content).toBe(responseText);
+  });
+
+  test("degrades to a benign result (never throws) when the consult fails", async () => {
+    recordMessages("c2", [userMsg("hi")]);
+    sendMessageError = new Error("kaboom");
+
+    const result = await advisorTool.execute?.({}, {
+      conversationId: "c2",
+    } as never);
+
+    expect(result?.isError).toBe(false);
+    expect(result?.content).toContain("advisor unavailable");
+    expect(result?.content).toContain("kaboom");
+  });
+});

package/src/plugins/defaults/advisor/__tests__/hooks.test.ts

@@ -0,0 +1,138 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type {
+  Message,
+  PostModelCallContext,
+  PreModelCallContext,
+  UserPromptSubmitContext,
+} from "@vellumai/plugin-api";
+
+import {
+  getCapture,
+  resetAdvisorStateForTests,
+} from "../advisor-state-store.js";
+import postModelCall from "../hooks/post-model-call.js";
+import preModelCall from "../hooks/pre-model-call.js";
+import userPromptSubmit from "../hooks/user-prompt-submit.js";
+import { STEERING_MARKER } from "../steering.js";
+
+const logger = { info() {}, warn() {}, error() {}, debug() {} };
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+beforeEach(() => {
+  resetAdvisorStateForTests();
+});
+
+describe("pre-model-call hook", () => {
+  test("records the original system prompt and injects steering on mainAgent", async () => {
+    const ctx = {
+      conversationId: "c1",
+      callSite: "mainAgent",
+      systemPrompt: "BASE PROMPT",
+      deferAssistantOutput: false,
+      logger,
+    } as unknown as PreModelCallContext;
+
+    await preModelCall(ctx);
+
+    expect(ctx.systemPrompt).toContain(STEERING_MARKER);
+    expect(ctx.systemPrompt?.startsWith("BASE PROMPT")).toBe(true);
+    expect(getCapture("c1")?.systemPrompt).toBe("BASE PROMPT");
+  });
+
+  test("ignores non-mainAgent calls", async () => {
+    const ctx = {
+      conversationId: "bg",
+      callSite: "inference",
+      systemPrompt: "BG",
+      deferAssistantOutput: false,
+      logger,
+    } as unknown as PreModelCallContext;
+
+    await preModelCall(ctx);
+    expect(ctx.systemPrompt).toBe("BG");
+    expect(getCapture("bg")).toBeUndefined();
+  });
+});
+
+describe("post-model-call hook", () => {
+  const base = {
+    content: [],
+    stopReason: "end_turn",
+    decision: "stop" as const,
+    logger,
+  };
+
+  test("snapshots the transcript on mainAgent", async () => {
+    const messages = [userMsg("hi")];
+    await postModelCall({
+      ...base,
+      conversationId: "c1",
+      callSite: "mainAgent",
+      messages,
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c1")?.messages).toEqual(messages);
+  });
+
+  test("appends the model's current turn (ctx.content) to the snapshot", async () => {
+    const messages = [userMsg("task")];
+    const assistantTurn: Message = {
+      role: "assistant",
+      content: [
+        { type: "text", text: "thinking about it" },
+        { type: "tool_use", id: "t1", name: "advisor", input: {} },
+      ],
+    };
+    await postModelCall({
+      ...base,
+      content: assistantTurn.content,
+      conversationId: "c4",
+      callSite: "mainAgent",
+      messages,
+    } as unknown as PostModelCallContext);
+    const captured = getCapture("c4")?.messages;
+    expect(captured).toHaveLength(2);
+    expect(captured?.[1]).toEqual(assistantTurn);
+  });
+
+  test("ignores the advisor's own inference sub-call (recursion safety) and errors", async () => {
+    await postModelCall({
+      ...base,
+      conversationId: "c2",
+      callSite: "inference",
+      messages: [userMsg("hi")],
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c2")).toBeUndefined();
+
+    await postModelCall({
+      ...base,
+      conversationId: "c3",
+      callSite: "mainAgent",
+      messages: [userMsg("hi")],
+      error: new Error("boom"),
+      stopReason: null,
+    } as unknown as PostModelCallContext);
+    expect(getCapture("c3")).toBeUndefined();
+  });
+});
+
+describe("user-prompt-submit hook", () => {
+  test("seeds the capture with the inbound history", async () => {
+    const messages = [userMsg("task")];
+    await userPromptSubmit({
+      conversationId: "c1",
+      latestMessages: messages,
+      originalMessages: messages,
+      prompt: "task",
+      userMessageId: "u1",
+      requestId: "r1",
+      modelProfileKey: null,
+      isNonInteractive: false,
+      logger,
+    } as unknown as UserPromptSubmitContext);
+    expect(getCapture("c1")?.messages).toEqual(messages);
+  });
+});

package/src/plugins/defaults/advisor/__tests__/transcript.test.ts

@@ -0,0 +1,147 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ContentBlock, Message } from "../../../../providers/types.js";
+import { toAdvisorMessages } from "../transcript.js";
+
+const text = (t: string): ContentBlock => ({ type: "text", text: t });
+
+describe("toAdvisorMessages", () => {
+  test("drops thinking and redacted-thinking blocks", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("do the task")] },
+      {
+        role: "assistant",
+        content: [
+          { type: "thinking", thinking: "secret", signature: "sig" },
+          { type: "redacted_thinking", data: "blob" },
+          text("here is my answer"),
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out).toHaveLength(2);
+    expect(out[1].content).toEqual([text("here is my answer")]);
+  });
+
+  test("strips the pending advisor tool_use from the final assistant turn", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("task")] },
+      {
+        role: "assistant",
+        content: [
+          text("let me consult the advisor"),
+          { type: "tool_use", id: "t1", name: "advisor", input: {} },
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[1].content).toEqual([text("let me consult the advisor")]);
+  });
+
+  test("preserves completed client tool_use / tool_result pairs in earlier turns", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("task")] },
+      {
+        role: "assistant",
+        content: [{ type: "tool_use", id: "a", name: "bash", input: {} }],
+      },
+      {
+        role: "user",
+        content: [{ type: "tool_result", tool_use_id: "a", content: "output" }],
+      },
+      { role: "assistant", content: [text("done")] },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[1].content[0]).toEqual({
+      type: "tool_use",
+      id: "a",
+      name: "bash",
+      input: {},
+    });
+    expect(out[2].content[0]).toEqual({
+      type: "tool_result",
+      tool_use_id: "a",
+      content: "output",
+    });
+  });
+
+  test("drops a web-search server_tool_use AND its result together — no orphan", () => {
+    const messages: Message[] = [
+      { role: "user", content: [text("look it up")] },
+      {
+        role: "assistant",
+        content: [
+          text("searching"),
+          { type: "server_tool_use", id: "ws1", name: "web_search", input: {} },
+        ],
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "web_search_tool_result",
+            tool_use_id: "ws1",
+            content: [{ title: "x", url: "y" }],
+          },
+        ],
+      },
+      { role: "assistant", content: [text("found it; here is the answer")] },
+    ];
+    const out = toAdvisorMessages(messages);
+    const flat = out.flatMap((m) => m.content);
+    expect(flat.some((b) => b.type === "server_tool_use")).toBe(false);
+    expect(flat.some((b) => b.type === "web_search_tool_result")).toBe(false);
+    expect(flat).toContainEqual(text("searching"));
+    expect(flat).toContainEqual(text("found it; here is the answer"));
+  });
+
+  test("keeps top-level image blocks", () => {
+    const img: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "x" },
+    };
+    const messages: Message[] = [
+      { role: "user", content: [text("look at this"), img] },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[0].content).toEqual([text("look at this"), img]);
+  });
+
+  test("keeps image contentBlocks on a tool_result, dropping disallowed ones", () => {
+    const img: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: "x" },
+    };
+    const messages: Message[] = [
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "a",
+            content: "screenshot",
+            contentBlocks: [
+              img,
+              {
+                type: "file",
+                source: {
+                  type: "base64",
+                  media_type: "application/pdf",
+                  data: "z",
+                  filename: "f.pdf",
+                },
+              },
+            ],
+          },
+        ],
+      },
+    ];
+    const out = toAdvisorMessages(messages);
+    expect(out[0].content[0]).toEqual({
+      type: "tool_result",
+      tool_use_id: "a",
+      content: "screenshot",
+      contentBlocks: [img],
+    });
+  });
+});

package/src/plugins/defaults/advisor/advisor-state-store.ts

@@ -0,0 +1,94 @@
+/**
+ * Per-conversation capture store for the advisor plugin.
+ *
+ * A tool's `execute` does not receive the conversation transcript — only
+ * lifecycle hooks do. So the hooks snapshot what the executor saw (the system
+ * prompt from `pre-model-call`, the running transcript from `post-model-call`)
+ * into this module-level store, keyed by conversation id, and the `advisor`
+ * tool reads the latest snapshot when the model calls it. This reproduces the
+ * advisor-tool property that the model calls the tool with no arguments and the
+ * full context is supplied for it.
+ *
+ * Bounded by a small LRU so a long-lived daemon does not accumulate state for
+ * every conversation it has ever served.
+ */
+
+import type { Message } from "../../../providers/types.js";
+
+export interface AdvisorCapture {
+  /** The executor's system prompt (steering stripped), or null if unseen. */
+  systemPrompt: string | null;
+  /** The transcript the executor most recently saw on a `mainAgent` call. */
+  messages: Message[];
+  /** Last-write timestamp, used for LRU eviction. */
+  updatedAt: number;
+}
+
+const MAX_CONVERSATIONS = 200;
+const store = new Map<string, AdvisorCapture>();
+
+/** Move an entry to most-recently-used and evict the oldest past the cap. */
+function bump(conversationId: string, entry: AdvisorCapture): void {
+  entry.updatedAt = Date.now();
+  store.delete(conversationId);
+  store.set(conversationId, entry);
+  while (store.size > MAX_CONVERSATIONS) {
+    const oldest = store.keys().next().value;
+    if (oldest === undefined) break;
+    store.delete(oldest);
+  }
+}
+
+function ensure(conversationId: string): AdvisorCapture {
+  return (
+    store.get(conversationId) ?? {
+      systemPrompt: null,
+      messages: [],
+      updatedAt: Date.now(),
+    }
+  );
+}
+
+/**
+ * Seed the capture at the start of a user turn with the inbound history, so an
+ * advisor call on the very first model turn still has context even before
+ * `post-model-call` snapshots the running transcript.
+ */
+export function seedCapture(
+  conversationId: string,
+  messages: ReadonlyArray<Message>,
+): void {
+  const entry = ensure(conversationId);
+  entry.messages = [...messages];
+  bump(conversationId, entry);
+}
+
+/** Record the executor's system prompt (steering already stripped). */
+export function recordSystemPrompt(
+  conversationId: string,
+  systemPrompt: string | null,
+): void {
+  const entry = ensure(conversationId);
+  entry.systemPrompt = systemPrompt;
+  bump(conversationId, entry);
+}
+
+/** Snapshot the transcript the executor just saw (before tools run). */
+export function recordMessages(
+  conversationId: string,
+  messages: ReadonlyArray<Message>,
+): void {
+  const entry = ensure(conversationId);
+  entry.messages = [...messages];
+  bump(conversationId, entry);
+}
+
+/** The latest capture for a conversation, or `undefined` if none recorded. */
+export function getCapture(conversationId: string): AdvisorCapture | undefined {
+  return store.get(conversationId);
+}
+
+/** Test-only: clear all captured state. */
+export function resetAdvisorStateForTests(): void {
+  store.clear();
+}

package/src/plugins/defaults/advisor/config.ts

@@ -0,0 +1,26 @@
+/**
+ * Static behavioral configuration for the advisor plugin.
+ *
+ * The advisor *model* is the inference profile named by `profile`, applied as
+ * an `overrideProfile` on the general-purpose `inference` call site — so the
+ * consult routes through the workspace's configured provider (managed-proxy or
+ * BYOK) with no separate API key.
+ */
+export const ADVISOR_CONFIG = {
+  /**
+   * Inference profile the advisor consults. Defaults to the strongest managed
+   * profile; the resolver falls back to the active profile if it's absent.
+   */
+  profile: "quality-optimized",
+  /**
+   * Hard cap on advisor output tokens per consult. Mirrors the advisor tool's
+   * recommended 2048 (the provider floor is 1024).
+   */
+  maxTokens: 2048,
+  /** Soft word budget requested of the advisor, biasing toward a focused start. */
+  wordLimit: 80,
+  /** Abort the consult if the sub-call runs longer than this. */
+  timeoutMs: 60_000,
+  /** Inject the "call advisor before substantive work" steering. */
+  steeringEnabled: true,
+} as const;