@vellumai/assistant 0.10.3-staging.2 → 0.10.4-staging.1

package/src/plugins/defaults/advisor/__tests__/agent-loop-integration.test.ts

@@ -1,137 +0,0 @@
-/**
- * End-to-end integration test: drives the REAL agent loop with all first-party
- * defaults registered (so the advisor plugin's hooks + tool are live), and lets
- * the REAL consult run. Only the provider boundary is stubbed:
- *   - the executor's provider (a scripted mock provider), and
- *   - `getConfiguredProvider`, so the advisor sub-call resolves to a second
- *     scripted mock provider instead of a configured one.
- *
- * The advisor's inference genuinely runs through `consult` → `sendMessage`
- * against the injected advisor provider, exercising the full hook-capture →
- * tool.execute → consult → routed-inference path through the loop.
- */
-import { beforeEach, describe, expect, mock, test } from "bun:test";
-
-import {
-  createMockProvider,
-  textResponse,
-} from "../../../../__tests__/helpers/mock-provider.js";
-import { AgentLoop } from "../../../../agent/loop.js";
-import type {
-  ContentBlock,
-  Message,
-  ProviderResponse,
-} from "../../../../providers/types.js";
-
-const ADVICE = "ADVICE: use a channel-based worker pool with graceful drain.";
-let advisorProvider: ReturnType<typeof createMockProvider> | null = null;
-
-const realProviderSendMessage =
-  await import("../../../../providers/provider-send-message.js");
-mock.module("../../../../providers/provider-send-message.js", () => ({
-  ...realProviderSendMessage,
-  getConfiguredProvider: async () => advisorProvider?.provider ?? null,
-}));
-
-const { resetPluginRegistryAndRegisterDefaults } =
-  await import("../../index.js");
-const advisorTool = (await import("../tools/advisor.js")).default;
-
-const userTurn: Message = {
-  role: "user",
-  content: [{ type: "text", text: "build a worker pool" }],
-};
-
-function textOf(content: ReadonlyArray<ContentBlock>): string {
-  return content.map((b) => (b.type === "text" ? b.text : "")).join("");
-}
-
-describe("advisor — agent-loop integration", () => {
-  beforeEach(() => {
-    resetPluginRegistryAndRegisterDefaults();
-    advisorProvider = createMockProvider([textResponse(ADVICE)]);
-  });
-
-  test("model calls advisor → hooks capture → consult routes through inference → advice returns", async () => {
-    const consultThenText: ProviderResponse = {
-      content: [
-        { type: "text", text: "Let me consult the advisor." },
-        { type: "tool_use", id: "call-1", name: "advisor", input: {} },
-      ],
-      model: "mock-model",
-      usage: { inputTokens: 10, outputTokens: 5 },
-      stopReason: "tool_use",
-    };
-    const { provider } = createMockProvider([
-      consultThenText,
-      textResponse("Done — implemented the worker pool."),
-    ]);
-
-    const conversationId = "advisor-itest";
-    const loop = new AgentLoop({
-      provider,
-      systemPrompt: "You are a coding agent.",
-      conversationId,
-      tools: [
-        {
-          name: "advisor",
-          description: "",
-          input_schema: { type: "object", properties: {} },
-        },
-      ],
-      toolExecutor: async (name, input) =>
-        name === "advisor"
-          ? advisorTool.execute!(input, { conversationId } as never)
-          : { content: `unknown tool ${name}`, isError: true },
-    });
-
-    const { history } = await loop.run({
-      requestId: "req-1",
-      messages: [userTurn],
-      onEvent: () => {},
-      callSite: "mainAgent",
-      trust: { sourceChannel: "vellum", trustClass: "unknown" },
-    });
-
-    // The advisor sub-call happened, routed through the dedicated advisor call site.
-    expect(advisorProvider!.calls).toHaveLength(1);
-    const sub = advisorProvider!.calls[0];
-    expect(sub.options?.config?.callSite).toBe("advisor");
-    // No `advisorProfile` configured in the default test config, so no override
-    // is passed and the `advisor` call site resolves its own default profile.
-    expect(sub.options?.config?.overrideProfile).toBeUndefined();
-    expect(sub.options?.config?.tool_choice).toEqual({ type: "none" });
-    // No advisor-specific output cap — the resolver applies the profile budget.
-    expect(sub.options?.config?.max_tokens).toBeUndefined();
-
-    // The advisor saw the captured transcript (task present; pending tool_use stripped).
-    expect(textOf(sub.messages[0].content)).toContain("build a worker pool");
-    expect(textOf(sub.messages[sub.messages.length - 1].content)).toContain(
-      "focused strategic guidance",
-    );
-    // It also saw the model's CURRENT turn — the text it wrote right before the
-    // `advisor` tool_use — which `post-model-call` lifts out of `ctx.content`.
-    const transcript = sub.messages.map((m) => textOf(m.content)).join("\n");
-    expect(transcript).toContain("Let me consult the advisor.");
-
-    // The advisor saw the executor's system prompt (via pre-model-call).
-    expect(sub.options?.systemPrompt).toContain("senior advisor");
-    expect(sub.options?.systemPrompt).toContain("You are a coding agent.");
-
-    // The advice flowed back into the executor's history as the tool result.
-    const toolResult = history
-      .flatMap((m) => m.content)
-      .find((b) => b.type === "tool_result");
-    expect((toolResult as { content: string }).content).toContain(
-      "channel-based worker pool",
-    );
-
-    // The loop completed with the final answer.
-    const lastAssistant = [...history]
-      .reverse()
-      .find((m) => m.role === "assistant")!;
-    expect(textOf(lastAssistant.content)).toContain(
-      "implemented the worker pool",
-    );
-  });
-});

package/src/plugins/defaults/advisor/__tests__/consult.test.ts

@@ -1,314 +0,0 @@
-import { beforeEach, describe, expect, mock, test } from "bun:test";
-
-import type { Message } from "../../../../providers/types.js";
-
-// Stub the provider resolution; spread the real module so `extractAllText` /
-// `userMessage` (which the consult also uses) keep working.
-let sendMessageArgs: Record<string, unknown> | null = null;
-let responseText = "Use a channel-based worker pool; drain on shutdown.";
-let sendMessageError: Error | null = null;
-let providerResolves = true;
-let providerSupportsWeb = false;
-let streamDeltas: string[] = [];
-let streamEvents: Array<Record<string, unknown>> = [];
-
-const fakeProvider = {
-  name: "mock-advisor-provider",
-  get supportsNativeWebSearch() {
-    return providerSupportsWeb;
-  },
-  async sendMessage(messages: unknown, options: unknown) {
-    sendMessageArgs = { messages, options } as Record<string, unknown>;
-    if (sendMessageError) throw sendMessageError;
-    const onEvent = (
-      options as { onEvent?: (e: Record<string, unknown>) => void }
-    ).onEvent;
-    if (onEvent) {
-      // Activity (search/thinking) streams before the final advice text.
-      for (const ev of streamEvents) onEvent(ev);
-      for (const text of streamDeltas) onEvent({ type: "text_delta", text });
-    }
-    return {
-      content: [{ type: "text", text: responseText }],
-      model: "mock-model",
-      usage: { inputTokens: 1, outputTokens: 1 },
-      stopReason: "end_turn",
-    };
-  },
-};
-
-const realPsm = await import("../../../../providers/provider-send-message.js");
-mock.module("../../../../providers/provider-send-message.js", () => ({
-  ...realPsm,
-  getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
-}));
-
-// Keep the tool tests focused on the consult wiring: stub the context pack so
-// they don't reach into the registry / workspace / memory sources (those have
-// their own coverage). The consult itself never imports this module.
-mock.module("../context-pack.js", () => ({
-  buildAdvisorContext: async () => null,
-  deriveRecallQuery: () => null,
-}));
-
-const { consultAdvisor } = await import("../consult.js");
-const advisorTool = (await import("../tools/advisor.js")).default;
-const { recordSystemPrompt, recordMessages, resetAdvisorStateForTests } =
-  await import("../advisor-state-store.js");
-
-const userMsg = (t: string): Message => ({
-  role: "user",
-  content: [{ type: "text", text: t }],
-});
-
-function optionConfig(): Record<string, unknown> {
-  const options = sendMessageArgs?.options as Record<string, unknown>;
-  return options.config as Record<string, unknown>;
-}
-
-beforeEach(() => {
-  sendMessageArgs = null;
-  responseText = "Use a channel-based worker pool; drain on shutdown.";
-  sendMessageError = null;
-  providerResolves = true;
-  providerSupportsWeb = false;
-  streamDeltas = [];
-  streamEvents = [];
-  resetAdvisorStateForTests();
-});
-
-describe("consultAdvisor", () => {
-  test("routes through the advisor call site, tools off, returns advice", async () => {
-    const messages: Message[] = [
-      userMsg("build a worker pool"),
-      {
-        role: "assistant",
-        content: [
-          { type: "thinking", thinking: "secret", signature: "s" },
-          { type: "text", text: "let me consult the advisor" },
-          { type: "tool_use", id: "t1", name: "advisor", input: {} },
-        ],
-      },
-    ];
-
-    const advice = await consultAdvisor({
-      systemPrompt: "You are a coding agent.",
-      messages,
-    });
-
-    expect(advice).toBe(responseText);
-
-    const config = optionConfig();
-    expect(config.callSite).toBe("advisor");
-    // No `advisorProfile` is configured in the default test config, so the
-    // consult passes no override and the `advisor` call site resolves to its
-    // default profile (`quality-optimized`).
-    expect(config.overrideProfile).toBeUndefined();
-    expect(config.tool_choice).toEqual({ type: "none" });
-    // No advisor-specific output cap — the resolver applies the profile budget.
-    expect(config.max_tokens).toBeUndefined();
-
-    const sent = sendMessageArgs?.messages as Message[];
-    expect(sent[0]).toEqual(userMsg("build a worker pool"));
-    expect(sent[1]).toEqual({
-      role: "assistant",
-      content: [{ type: "text", text: "let me consult the advisor" }],
-    });
-    const lastText = (sent[sent.length - 1].content[0] as { text: string })
-      .text;
-    expect(lastText).toContain("focused strategic guidance");
-    // The request carries no word limit.
-    expect(lastText).not.toContain("words");
-
-    const options = sendMessageArgs?.options as { systemPrompt: string };
-    expect(options.systemPrompt).toContain("senior advisor");
-    expect(options.systemPrompt).toContain("You are a coding agent.");
-  });
-
-  test("stays tool-less when the provider has no native web search", async () => {
-    providerSupportsWeb = false;
-    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
-    const options = sendMessageArgs?.options as { tools?: unknown };
-    expect(options.tools).toBeUndefined();
-    expect(optionConfig().tool_choice).toEqual({ type: "none" });
-  });
-
-  test("enables native web search when the provider supports it", async () => {
-    providerSupportsWeb = true;
-    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
-
-    const options = sendMessageArgs?.options as {
-      tools?: Array<{ name: string }>;
-    };
-    expect(options.tools?.map((t) => t.name)).toEqual(["web_search"]);
-    // tool_choice must not be `none`, or the provider suppresses its server tool.
-    expect(optionConfig().tool_choice).toEqual({ type: "auto" });
-  });
-
-  test("streams web-search activity to onText, not just the final advice", async () => {
-    providerSupportsWeb = true;
-    streamEvents = [
-      { type: "server_tool_start", name: "web_search", toolUseId: "s1", input: {} },
-      {
-        type: "server_tool_complete",
-        toolUseId: "s1",
-        isError: false,
-        resolvedInput: { query: "vellum streaming" },
-      },
-    ];
-    streamDeltas = ["Here is ", "the advice."];
-    const chunks: string[] = [];
-
-    await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-      onText: (c) => chunks.push(c),
-    });
-
-    const joined = chunks.join("");
-    // The drawer isn't silent during the search prefix...
-    expect(joined).toContain("Searching the web");
-    expect(joined).toContain("Searched: vellum streaming");
-    // ...and the advice text still streams.
-    expect(joined).toContain("Here is the advice.");
-  });
-
-  test("surfaces a failure note (not 'Searched') when a web search errors", async () => {
-    providerSupportsWeb = true;
-    streamEvents = [
-      { type: "server_tool_start", name: "web_search", toolUseId: "s1", input: {} },
-      {
-        type: "server_tool_complete",
-        toolUseId: "s1",
-        isError: true,
-        errorCode: "query_too_long",
-        resolvedInput: { query: "an overly long query" },
-      },
-    ];
-    streamDeltas = ["Proceeding without search."];
-    const chunks: string[] = [];
-
-    await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-      onText: (c) => chunks.push(c),
-    });
-
-    const joined = chunks.join("");
-    expect(joined).toContain("Web search failed");
-    expect(joined).not.toContain("🔎 Searched:");
-    // The consult still continues and streams its guidance.
-    expect(joined).toContain("Proceeding without search.");
-  });
-
-  test("streams the model's reasoning summary to onText", async () => {
-    streamEvents = [{ type: "thinking_delta", thinking: "weighing tradeoffs" }];
-    const chunks: string[] = [];
-    await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-      onText: (c) => chunks.push(c),
-    });
-    expect(chunks.join("")).toContain("weighing tradeoffs");
-  });
-
-  test("embeds the runtime context in the advisor system prompt", async () => {
-    await consultAdvisor({
-      systemPrompt: "You are a coding agent.",
-      messages: [userMsg("hi")],
-      runtimeContext: "## Available tools\n- bash — run commands",
-    });
-    const options = sendMessageArgs?.options as { systemPrompt: string };
-    expect(options.systemPrompt).toContain("<agent_runtime_context>");
-    expect(options.systemPrompt).toContain("- bash — run commands");
-  });
-
-  test("soft-fails when no provider is configured", async () => {
-    providerResolves = false;
-    const advice = await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-    });
-    expect(advice).toContain("no inference provider");
-  });
-
-  test("returns a notice when there is no usable transcript", async () => {
-    const advice = await consultAdvisor({ systemPrompt: null, messages: [] });
-    expect(advice).toContain("no conversation context");
-    expect(sendMessageArgs).toBeNull();
-  });
-
-  test("falls back to a notice when the advisor returns blank text", async () => {
-    responseText = "   ";
-    const advice = await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-    });
-    expect(advice).toContain("no guidance");
-  });
-
-  test("streams the model's text deltas to `onText` as it generates", async () => {
-    streamDeltas = ["Use a ", "channel-based ", "worker pool."];
-    const chunks: string[] = [];
-
-    const advice = await consultAdvisor({
-      systemPrompt: null,
-      messages: [userMsg("hi")],
-      onText: (c) => chunks.push(c),
-    });
-
-    // Each visible delta is forwarded live...
-    expect(chunks).toEqual(["Use a ", "channel-based ", "worker pool."]);
-    // ...and the complete guidance is still returned.
-    expect(advice).toBe(responseText);
-  });
-
-  test("registers no `onEvent` sink when `onText` is absent", async () => {
-    streamDeltas = ["x"];
-    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
-    const options = sendMessageArgs?.options as { onEvent?: unknown };
-    expect(options.onEvent).toBeUndefined();
-  });
-});
-
-describe("advisor tool.execute", () => {
-  test("reads the captured transcript and returns guidance as a non-error result", async () => {
-    recordSystemPrompt("c1", "You are a coding agent.");
-    recordMessages("c1", [userMsg("build a worker pool")]);
-
-    const result = await advisorTool.execute?.({}, {
-      conversationId: "c1",
-    } as never);
-
-    expect(result?.isError).toBe(false);
-    expect(result?.content).toBe(responseText);
-  });
-
-  test("degrades to a benign result (never throws) when the consult fails", async () => {
-    recordMessages("c2", [userMsg("hi")]);
-    sendMessageError = new Error("kaboom");
-
-    const result = await advisorTool.execute?.({}, {
-      conversationId: "c2",
-    } as never);
-
-    expect(result?.isError).toBe(false);
-    expect(result?.content).toContain("advisor unavailable");
-    expect(result?.content).toContain("kaboom");
-  });
-
-  test("streams the consult live via `ctx.onOutput`", async () => {
-    recordMessages("c3", [userMsg("hi")]);
-    streamDeltas = ["plan: ", "do X"];
-    const out: string[] = [];
-
-    const result = await advisorTool.execute?.({}, {
-      conversationId: "c3",
-      onOutput: (c: string) => out.push(c),
-    } as never);
-
-    expect(out).toEqual(["plan: ", "do X"]);
-    expect(result?.isError).toBe(false);
-    expect(result?.content).toBe(responseText);
-  });
-});

package/src/plugins/defaults/advisor/__tests__/context-pack-gating.test.ts

@@ -1,106 +0,0 @@
-/**
- * Personal-memory gating for the advisor context pack: NOW.md and PKB must only
- * reach the advisor when the turn's trust admits personal memory (and, for
- * NOW.md, when the scratchpad-injection toggle is on) — the same policy the
- * runtime memory injectors apply. Without it, a low-risk advisor consult on a
- * remote/trusted-contact turn could forward private content the main agent
- * would never receive.
- *
- * Mocks are isolated to this file (the test runner runs each file in its own
- * process), so the broad module stubs here don't leak into other suites.
- */
-import { beforeEach, describe, expect, mock, test } from "bun:test";
-
-let personalAllowed = false;
-let scratchpadEnabled = true;
-let gateArg: unknown = null;
-
-mock.module("../../../../daemon/trust-context.js", () => ({
-  isPersonalMemoryAllowed: (trust: unknown) => {
-    gateArg = trust;
-    return personalAllowed;
-  },
-}));
-mock.module("../../../../daemon/now-scratchpad.js", () => ({
-  readNowScratchpad: () => "NOW-CONTENT",
-}));
-mock.module("../../../../memory/pkb/context.js", () => ({
-  readPkbContext: () => "PKB-CONTENT",
-}));
-mock.module("../../../../config/loader.js", () => ({
-  getConfig: () => ({
-    memory: {
-      retrieval: { scratchpadInjection: { enabled: scratchpadEnabled } },
-    },
-    llm: {},
-  }),
-}));
-// Keep every other section empty so the assertions isolate NOW.md / PKB.
-mock.module("../../../../daemon/conversation-workspace.js", () => ({
-  resolveWorkspaceTopLevelContext: () => null,
-}));
-mock.module("../../../../daemon/conversation-runtime-assembly.js", () => ({
-  buildActiveDocuments: () => null,
-}));
-mock.module("../../../../runtime/capabilities.js", () => ({
-  resolveCapabilities: () => ({ canAccessMemory: false }),
-}));
-mock.module("../../../../config/skills.js", () => ({
-  loadSkillCatalog: () => [],
-}));
-
-const { buildAdvisorContext } = await import("../context-pack.js");
-
-const sources = {
-  conversationId: "c1",
-  workingDir: "/tmp",
-  // A remote, non-guardian per-turn snapshot — the case the live-state read
-  // could have wrongly elevated.
-  trustClass: "unknown" as const,
-  sourceChannel: "telegram",
-  transcript: [],
-  allowedToolNames: new Set<string>(),
-};
-
-beforeEach(() => {
-  personalAllowed = false;
-  scratchpadEnabled = true;
-  gateArg = null;
-});
-
-describe("advisor context pack — personal-memory gating", () => {
-  test("withholds NOW.md and PKB when personal memory is disallowed", async () => {
-    personalAllowed = false;
-    const ctx = (await buildAdvisorContext(sources)) ?? "";
-    expect(ctx).not.toContain("NOW-CONTENT");
-    expect(ctx).not.toContain("PKB-CONTENT");
-  });
-
-  test("includes NOW.md and PKB when allowed and the scratchpad toggle is on", async () => {
-    personalAllowed = true;
-    scratchpadEnabled = true;
-    const ctx = await buildAdvisorContext(sources);
-    expect(ctx).toContain("NOW-CONTENT");
-    expect(ctx).toContain("PKB-CONTENT");
-  });
-
-  test("withholds NOW.md when the scratchpad toggle is off, PKB still allowed", async () => {
-    personalAllowed = true;
-    scratchpadEnabled = false;
-    const ctx = (await buildAdvisorContext(sources)) ?? "";
-    expect(ctx).not.toContain("NOW-CONTENT");
-    expect(ctx).toContain("PKB-CONTENT");
-  });
-
-  test("feeds the gate the per-turn trust snapshot, not live conversation state", async () => {
-    personalAllowed = true;
-    await buildAdvisorContext(sources);
-    // The gate must see exactly the snapshot threaded from ToolContext —
-    // trustClass + executionChannel — so a concurrent live-trust change can't
-    // elevate this invocation.
-    expect(gateArg).toEqual({
-      sourceChannel: "telegram",
-      trustClass: "unknown",
-    });
-  });
-});

package/src/plugins/defaults/advisor/__tests__/context-pack.test.ts

@@ -1,60 +0,0 @@
-import { describe, expect, test } from "bun:test";
-
-import type { Message } from "../../../../providers/types.js";
-import { buildAdvisorContext, deriveRecallQuery } from "../context-pack.js";
-
-const userMsg = (t: string): Message => ({
-  role: "user",
-  content: [{ type: "text", text: t }],
-});
-
-describe("deriveRecallQuery", () => {
-  test("returns the most recent user message text", () => {
-    const query = deriveRecallQuery([
-      userMsg("the original task"),
-      { role: "assistant", content: [{ type: "text", text: "ok" }] },
-      userMsg("the latest question"),
-    ]);
-    expect(query).toBe("the latest question");
-  });
-
-  test("returns null when there is no user text", () => {
-    expect(
-      deriveRecallQuery([
-        { role: "assistant", content: [{ type: "text", text: "hi" }] },
-      ]),
-    ).toBeNull();
-    expect(deriveRecallQuery([])).toBeNull();
-  });
-});
-
-describe("buildAdvisorContext", () => {
-  test("lists the agent's available tools, skipping the advisor itself", async () => {
-    const context = await buildAdvisorContext({
-      conversationId: "ctx-1",
-      workingDir: "/tmp/does-not-exist",
-      allowedToolNames: new Set(["bash", "advisor", "read_file"]),
-      trustClass: "unknown",
-      transcript: [userMsg("hi")],
-    });
-
-    expect(context).toContain("## Available tools");
-    expect(context).toContain("- bash");
-    expect(context).toContain("- read_file");
-    // The advisor advises; it never tells the agent to consult itself.
-    expect(context).not.toContain("- advisor");
-  });
-
-  test("omits the tools section when no tools are available", async () => {
-    const context = await buildAdvisorContext({
-      conversationId: "ctx-2",
-      workingDir: "/tmp/does-not-exist",
-      allowedToolNames: new Set(),
-      trustClass: "unknown",
-      transcript: [],
-    });
-    // Other sources (e.g. the skills catalog) may still contribute, but with no
-    // allowed tools the tools section must not appear.
-    if (context !== null) expect(context).not.toContain("## Available tools");
-  });
-});