@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/plugin-api/model-profiles.ts

@@ -1,11 +1,14 @@
 import { getConfig } from "../config/loader.js";
+import { isDispatchableProfile } from "../config/profile-dispatchability.js";
 import { orderProfileKeys } from "../config/profile-order.js";
 import type { ModelProfileInfo } from "./types.js";
 
 /**
  * List the workspace inference profiles a plugin can route to, in the order the
  * `/model` picker presents them (`llm.profileOrder` first, then the rest
- * alphabetically). Disabled profiles are included and flagged via
+ * alphabetically). Metadata-only entries without a provider, model, or mix are
+ * not routing targets, so plugins never see them. Disabled profiles are
+ * included and flagged via
  * {@link ModelProfileInfo.isDisabled}; weighted "mix" profiles are included and
  * flagged via {@link ModelProfileInfo.isMix}, since a mix is itself a valid
  * routing target (it resolves to one constituent per conversation).
@@ -20,6 +23,7 @@ export function getModelProfiles(): ModelProfileInfo[] {
   for (const key of orderProfileKeys(profiles, llm.profileOrder)) {
     const entry = profiles[key];
     if (entry == null) continue;
+    if (!isDispatchableProfile(entry)) continue;
     result.push({
       key,
       label: entry.label ?? key,

package/src/plugin-api/vision-support.test.ts

@@ -0,0 +1,173 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { ModelProfileInfo } from "./types.js";
+
+// ─── Mock config ────────────────────────────────────────────────────────────
+
+interface MockProfileEntry {
+  provider?: string;
+  model?: string;
+  status?: string;
+  mix?: Array<{ profile: string; weight: number }>;
+}
+
+let mockProfiles: Record<string, MockProfileEntry> = {};
+let mockDefault: { provider?: string; model?: string } = {
+  provider: "anthropic",
+  model: "claude-opus-4-6",
+};
+
+const realConfigLoader = await import("../config/loader.js");
+
+mock.module("../config/loader.js", () => ({
+  ...realConfigLoader,
+  getConfig: () => ({
+    llm: {
+      profiles: mockProfiles,
+      default: mockDefault,
+    },
+  }),
+  getConfigReadOnly: () => ({
+    llm: {
+      profiles: mockProfiles,
+      default: mockDefault,
+    },
+  }),
+}));
+
+// Real model catalog — don't mock it, the test exercises real catalog lookups.
+const { doesSupportVision } = await import("./vision-support.js");
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function profile(key: string): ModelProfileInfo {
+  return {
+    key,
+    label: key,
+    description: null,
+    isActive: false,
+    isDisabled: false,
+    isMix: false,
+  };
+}
+
+function setMockConfig(
+  profiles: Record<string, MockProfileEntry>,
+  def: { provider?: string; model?: string } = {},
+) {
+  mockProfiles = profiles;
+  mockDefault = { provider: "anthropic", model: "claude-opus-4-6", ...def };
+}
+
+beforeEach(() => {
+  mockProfiles = {};
+  mockDefault = { provider: "anthropic", model: "claude-opus-4-6" };
+});
+
+// ─── Tests ──────────────────────────────────────────────────────────────────
+
+describe("doesSupportVision", () => {
+  test("returns true for a known vision-capable model", () => {
+    setMockConfig({
+      "vision-profile": { provider: "anthropic", model: "claude-opus-4-6" },
+    });
+    expect(doesSupportVision(profile("vision-profile"))).toBe(true);
+  });
+
+  test("returns false for a known text-only model", () => {
+    setMockConfig({
+      "text-profile": {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/glm-5p2",
+      },
+    });
+    expect(doesSupportVision(profile("text-profile"))).toBe(false);
+  });
+
+  test("returns false for an unknown profile key not in config", () => {
+    setMockConfig({});
+    expect(doesSupportVision(profile("nonexistent"))).toBe(false);
+  });
+
+  test("returns false for an unknown provider/model pair", () => {
+    setMockConfig({
+      "unknown-model": { provider: "unknown-provider", model: "unknown-model" },
+    });
+    expect(doesSupportVision(profile("unknown-model"))).toBe(false);
+  });
+
+  test("inherits provider from llm.default when profile only sets model", () => {
+    setMockConfig(
+      { "model-only": { model: "claude-opus-4-6" } },
+      { provider: "anthropic" },
+    );
+    expect(doesSupportVision(profile("model-only"))).toBe(true);
+  });
+
+  test("inherits model from llm.default when profile only sets provider", () => {
+    // llm.default → anthropic/claude-opus-4-6 (vision-capable)
+    setMockConfig({ "provider-only": { provider: "anthropic" } });
+    expect(doesSupportVision(profile("provider-only"))).toBe(true);
+  });
+
+  test("mix profile returns true when any arm supports vision", () => {
+    setMockConfig({
+      "mix-profile": {
+        mix: [
+          { profile: "text-arm", weight: 0.5 },
+          { profile: "vision-arm", weight: 0.5 },
+        ],
+      },
+      "text-arm": {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/glm-5p2",
+      },
+      "vision-arm": { provider: "anthropic", model: "claude-opus-4-6" },
+    });
+    expect(doesSupportVision(profile("mix-profile"))).toBe(true);
+  });
+
+  test("mix profile returns false when all arms are text-only", () => {
+    setMockConfig({
+      "mix-profile": {
+        mix: [
+          { profile: "text-arm-1", weight: 0.5 },
+          { profile: "text-arm-2", weight: 0.5 },
+        ],
+      },
+      "text-arm-1": {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/glm-5p2",
+      },
+      "text-arm-2": {
+        provider: "fireworks",
+        model: "accounts/fireworks/models/glm-5p2",
+      },
+    });
+    expect(doesSupportVision(profile("mix-profile"))).toBe(false);
+  });
+});
+
+describe("doesSupportVision with a bare string", () => {
+  test("returns true for a known vision-capable model id", () => {
+    expect(doesSupportVision("claude-opus-4-6")).toBe(true);
+  });
+
+  test("returns false for a known text-only model id", () => {
+    expect(doesSupportVision("accounts/fireworks/models/glm-5p2")).toBe(false);
+  });
+
+  test("falls back to resolving the string as a profile key", () => {
+    // "vision-profile" is not a catalog model id, so it resolves as a profile
+    // key → anthropic/claude-opus-4-6 (vision-capable).
+    setMockConfig({
+      "vision-profile": { provider: "anthropic", model: "claude-opus-4-6" },
+    });
+    expect(doesSupportVision("vision-profile")).toBe(true);
+  });
+
+  test("returns false for a string that is neither a model nor a profile", () => {
+    setMockConfig({});
+    expect(doesSupportVision("some-unknown-string")).toBe(false);
+  });
+});

package/src/plugin-api/vision-support.ts

@@ -0,0 +1,113 @@
+/**
+ * Vision-support resolution for plugin consumption.
+ *
+ * A plugin that gates image processing on vision capability (e.g. an
+ * image-to-text fallback for text-only models) calls {@link doesSupportVision}
+ * instead of hardcoding model names. One entry point serves both shapes a
+ * caller might hold:
+ * - a concrete model id (e.g. the provider-reported model that just ran), and
+ * - a profile — either a {@link ModelProfileInfo} or a bare profile key —
+ *   resolved through `llm.profiles` to an effective `(provider, model)`.
+ *
+ * A bare string is tried as a model id first and then as a profile key, so the
+ * two callers share one function. Resolution returns `false` when nothing
+ * resolves (rather than failing open): a consumer gating an image→text
+ * fallback wants an unknown model treated as "can't show images" — caption it —
+ * over silently shipping a raw image to a provider that may reject it.
+ */
+
+import { getConfig } from "../config/loader.js";
+import {
+  getCatalogProviderForModel,
+  PROVIDER_CATALOG,
+} from "../providers/model-catalog.js";
+import type { ModelProfileInfo } from "./types.js";
+
+/**
+ * Whether the given model or profile can process image input.
+ *
+ * `modelOrProfile` may be a concrete model id, a profile key, or a
+ * {@link ModelProfileInfo}. A bare string is resolved as a model id first and,
+ * failing that, as a profile key. Returns `false` when nothing resolves.
+ */
+export function doesSupportVision(
+  modelOrProfile: ModelProfileInfo | string,
+): boolean {
+  if (typeof modelOrProfile === "string") {
+    // Concrete model id first, then fall back to treating it as a profile key.
+    return (
+      modelVision(modelOrProfile) ?? profileVision(modelOrProfile) ?? false
+    );
+  }
+  return profileVision(modelOrProfile.key) ?? false;
+}
+
+/**
+ * Catalog vision flag for a concrete model id, or `undefined` when the catalog
+ * doesn't know the model. The same model id carries the same capability under
+ * every provider that offers it, so the first catalog match wins.
+ */
+function modelVision(model: string): boolean | undefined {
+  for (const provider of PROVIDER_CATALOG) {
+    const catalogModel = provider.models.find((m) => m.id === model);
+    if (catalogModel != null) return catalogModel.supportsVision ?? false;
+  }
+  return undefined;
+}
+
+/**
+ * Resolve a profile key through `llm.profiles` to its vision capability, or
+ * `undefined` when the key is unknown or resolves to a model the catalog
+ * doesn't know. A mix profile resolves to `true` if any arm supports vision
+ * (the mix can route to it) and `false` only once every arm is a known
+ * text-only model.
+ */
+function profileVision(profileKey: string): boolean | undefined {
+  const { llm } = getConfig();
+  const entry = llm.profiles[profileKey];
+  if (entry == null) return undefined;
+
+  if (entry.mix != null) {
+    let sawUnknown = false;
+    for (const arm of entry.mix) {
+      const armEntry = llm.profiles[arm.profile];
+      const armVision =
+        armEntry == null ? undefined : resolveEntryVision(armEntry, llm);
+      if (armVision === true) return true;
+      if (armVision == null) sawUnknown = true;
+    }
+    return sawUnknown ? undefined : false;
+  }
+
+  return resolveEntryVision(entry, llm);
+}
+
+/**
+ * Resolve whether a concrete (non-mix) profile entry supports vision by
+ * merging its fields over `llm.default` and inferring the provider when only
+ * the model is set. Returns `undefined` when the effective `(provider, model)`
+ * can't be determined or isn't in the catalog.
+ */
+function resolveEntryVision(
+  entry: { provider?: string; model?: string },
+  llm: { default?: { provider?: string; model?: string } },
+): boolean | undefined {
+  const provider = entry.provider ?? llm.default?.provider;
+  const model = entry.model ?? llm.default?.model;
+
+  // Infer provider from model when missing (mirrors the resolver's
+  // withImpliedProviderForKnownModel).
+  const effectiveProvider =
+    provider ??
+    (typeof model === "string" ? getCatalogProviderForModel(model) : undefined);
+
+  if (typeof effectiveProvider !== "string" || typeof model !== "string") {
+    return undefined;
+  }
+
+  const catalogProvider = PROVIDER_CATALOG.find(
+    (p) => p.id === effectiveProvider,
+  );
+  const catalogModel = catalogProvider?.models.find((m) => m.id === model);
+  return catalogModel?.supportsVision;
+}

package/src/plugins/defaults/advisor/__tests__/consult.test.ts

@@ -8,12 +8,23 @@ let sendMessageArgs: Record<string, unknown> | null = null;
 let responseText = "Use a channel-based worker pool; drain on shutdown.";
 let sendMessageError: Error | null = null;
 let providerResolves = true;
+let providerSupportsWeb = false;
+let streamDeltas: string[] = [];
 
 const fakeProvider = {
   name: "mock-advisor-provider",
+  get supportsNativeWebSearch() {
+    return providerSupportsWeb;
+  },
   async sendMessage(messages: unknown, options: unknown) {
     sendMessageArgs = { messages, options } as Record<string, unknown>;
     if (sendMessageError) throw sendMessageError;
+    const onEvent = (
+      options as { onEvent?: (e: { type: string; text?: string }) => void }
+    ).onEvent;
+    if (onEvent) {
+      for (const text of streamDeltas) onEvent({ type: "text_delta", text });
+    }
     return {
       content: [{ type: "text", text: responseText }],
       model: "mock-model",
@@ -29,6 +40,14 @@ mock.module("../../../../providers/provider-send-message.js", () => ({
   getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
 }));
 
+// Keep the tool tests focused on the consult wiring: stub the context pack so
+// they don't reach into the registry / workspace / memory sources (those have
+// their own coverage). The consult itself never imports this module.
+mock.module("../context-pack.js", () => ({
+  buildAdvisorContext: async () => null,
+  deriveRecallQuery: () => null,
+}));
+
 const { consultAdvisor } = await import("../consult.js");
 const advisorTool = (await import("../tools/advisor.js")).default;
 const { recordSystemPrompt, recordMessages, resetAdvisorStateForTests } =
@@ -49,6 +68,8 @@ beforeEach(() => {
   responseText = "Use a channel-based worker pool; drain on shutdown.";
   sendMessageError = null;
   providerResolves = true;
+  providerSupportsWeb = false;
+  streamDeltas = [];
   resetAdvisorStateForTests();
 });
 
@@ -100,6 +121,37 @@ describe("consultAdvisor", () => {
     expect(options.systemPrompt).toContain("You are a coding agent.");
   });
 
+  test("stays tool-less when the provider has no native web search", async () => {
+    providerSupportsWeb = false;
+    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
+    const options = sendMessageArgs?.options as { tools?: unknown };
+    expect(options.tools).toBeUndefined();
+    expect(optionConfig().tool_choice).toEqual({ type: "none" });
+  });
+
+  test("enables native web search when the provider supports it", async () => {
+    providerSupportsWeb = true;
+    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
+
+    const options = sendMessageArgs?.options as {
+      tools?: Array<{ name: string }>;
+    };
+    expect(options.tools?.map((t) => t.name)).toEqual(["web_search"]);
+    // tool_choice must not be `none`, or the provider suppresses its server tool.
+    expect(optionConfig().tool_choice).toEqual({ type: "auto" });
+  });
+
+  test("embeds the runtime context in the advisor system prompt", async () => {
+    await consultAdvisor({
+      systemPrompt: "You are a coding agent.",
+      messages: [userMsg("hi")],
+      runtimeContext: "## Available tools\n- bash — run commands",
+    });
+    const options = sendMessageArgs?.options as { systemPrompt: string };
+    expect(options.systemPrompt).toContain("<agent_runtime_context>");
+    expect(options.systemPrompt).toContain("- bash — run commands");
+  });
+
   test("soft-fails when no provider is configured", async () => {
     providerResolves = false;
     const advice = await consultAdvisor({
@@ -123,6 +175,29 @@ describe("consultAdvisor", () => {
     });
     expect(advice).toContain("no guidance");
   });
+
+  test("streams the model's text deltas to `onText` as it generates", async () => {
+    streamDeltas = ["Use a ", "channel-based ", "worker pool."];
+    const chunks: string[] = [];
+
+    const advice = await consultAdvisor({
+      systemPrompt: null,
+      messages: [userMsg("hi")],
+      onText: (c) => chunks.push(c),
+    });
+
+    // Each visible delta is forwarded live...
+    expect(chunks).toEqual(["Use a ", "channel-based ", "worker pool."]);
+    // ...and the complete guidance is still returned.
+    expect(advice).toBe(responseText);
+  });
+
+  test("registers no `onEvent` sink when `onText` is absent", async () => {
+    streamDeltas = ["x"];
+    await consultAdvisor({ systemPrompt: null, messages: [userMsg("hi")] });
+    const options = sendMessageArgs?.options as { onEvent?: unknown };
+    expect(options.onEvent).toBeUndefined();
+  });
 });
 
 describe("advisor tool.execute", () => {
@@ -150,4 +225,19 @@ describe("advisor tool.execute", () => {
     expect(result?.content).toContain("advisor unavailable");
     expect(result?.content).toContain("kaboom");
   });
+
+  test("streams the consult live via `ctx.onOutput`", async () => {
+    recordMessages("c3", [userMsg("hi")]);
+    streamDeltas = ["plan: ", "do X"];
+    const out: string[] = [];
+
+    const result = await advisorTool.execute?.({}, {
+      conversationId: "c3",
+      onOutput: (c: string) => out.push(c),
+    } as never);
+
+    expect(out).toEqual(["plan: ", "do X"]);
+    expect(result?.isError).toBe(false);
+    expect(result?.content).toBe(responseText);
+  });
 });

package/src/plugins/defaults/advisor/__tests__/context-pack-gating.test.ts

@@ -0,0 +1,106 @@
+/**
+ * Personal-memory gating for the advisor context pack: NOW.md and PKB must only
+ * reach the advisor when the turn's trust admits personal memory (and, for
+ * NOW.md, when the scratchpad-injection toggle is on) — the same policy the
+ * runtime memory injectors apply. Without it, a low-risk advisor consult on a
+ * remote/trusted-contact turn could forward private content the main agent
+ * would never receive.
+ *
+ * Mocks are isolated to this file (the test runner runs each file in its own
+ * process), so the broad module stubs here don't leak into other suites.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+let personalAllowed = false;
+let scratchpadEnabled = true;
+let gateArg: unknown = null;
+
+mock.module("../../../../daemon/trust-context.js", () => ({
+  isPersonalMemoryAllowed: (trust: unknown) => {
+    gateArg = trust;
+    return personalAllowed;
+  },
+}));
+mock.module("../../../../daemon/now-scratchpad.js", () => ({
+  readNowScratchpad: () => "NOW-CONTENT",
+}));
+mock.module("../../../../memory/pkb/context.js", () => ({
+  readPkbContext: () => "PKB-CONTENT",
+}));
+mock.module("../../../../config/loader.js", () => ({
+  getConfig: () => ({
+    memory: {
+      retrieval: { scratchpadInjection: { enabled: scratchpadEnabled } },
+    },
+    llm: {},
+  }),
+}));
+// Keep every other section empty so the assertions isolate NOW.md / PKB.
+mock.module("../../../../daemon/conversation-workspace.js", () => ({
+  resolveWorkspaceTopLevelContext: () => null,
+}));
+mock.module("../../../../daemon/conversation-runtime-assembly.js", () => ({
+  buildActiveDocuments: () => null,
+}));
+mock.module("../../../../runtime/capabilities.js", () => ({
+  resolveCapabilities: () => ({ canAccessMemory: false }),
+}));
+mock.module("../../../../config/skills.js", () => ({
+  loadSkillCatalog: () => [],
+}));
+
+const { buildAdvisorContext } = await import("../context-pack.js");
+
+const sources = {
+  conversationId: "c1",
+  workingDir: "/tmp",
+  // A remote, non-guardian per-turn snapshot — the case the live-state read
+  // could have wrongly elevated.
+  trustClass: "unknown" as const,
+  sourceChannel: "telegram",
+  transcript: [],
+  allowedToolNames: new Set<string>(),
+};
+
+beforeEach(() => {
+  personalAllowed = false;
+  scratchpadEnabled = true;
+  gateArg = null;
+});
+
+describe("advisor context pack — personal-memory gating", () => {
+  test("withholds NOW.md and PKB when personal memory is disallowed", async () => {
+    personalAllowed = false;
+    const ctx = (await buildAdvisorContext(sources)) ?? "";
+    expect(ctx).not.toContain("NOW-CONTENT");
+    expect(ctx).not.toContain("PKB-CONTENT");
+  });
+
+  test("includes NOW.md and PKB when allowed and the scratchpad toggle is on", async () => {
+    personalAllowed = true;
+    scratchpadEnabled = true;
+    const ctx = await buildAdvisorContext(sources);
+    expect(ctx).toContain("NOW-CONTENT");
+    expect(ctx).toContain("PKB-CONTENT");
+  });
+
+  test("withholds NOW.md when the scratchpad toggle is off, PKB still allowed", async () => {
+    personalAllowed = true;
+    scratchpadEnabled = false;
+    const ctx = (await buildAdvisorContext(sources)) ?? "";
+    expect(ctx).not.toContain("NOW-CONTENT");
+    expect(ctx).toContain("PKB-CONTENT");
+  });
+
+  test("feeds the gate the per-turn trust snapshot, not live conversation state", async () => {
+    personalAllowed = true;
+    await buildAdvisorContext(sources);
+    // The gate must see exactly the snapshot threaded from ToolContext —
+    // trustClass + executionChannel — so a concurrent live-trust change can't
+    // elevate this invocation.
+    expect(gateArg).toEqual({
+      sourceChannel: "telegram",
+      trustClass: "unknown",
+    });
+  });
+});

package/src/plugins/defaults/advisor/__tests__/context-pack.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, test } from "bun:test";
+
+import type { Message } from "../../../../providers/types.js";
+import { buildAdvisorContext, deriveRecallQuery } from "../context-pack.js";
+
+const userMsg = (t: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text: t }],
+});
+
+describe("deriveRecallQuery", () => {
+  test("returns the most recent user message text", () => {
+    const query = deriveRecallQuery([
+      userMsg("the original task"),
+      { role: "assistant", content: [{ type: "text", text: "ok" }] },
+      userMsg("the latest question"),
+    ]);
+    expect(query).toBe("the latest question");
+  });
+
+  test("returns null when there is no user text", () => {
+    expect(
+      deriveRecallQuery([
+        { role: "assistant", content: [{ type: "text", text: "hi" }] },
+      ]),
+    ).toBeNull();
+    expect(deriveRecallQuery([])).toBeNull();
+  });
+});
+
+describe("buildAdvisorContext", () => {
+  test("lists the agent's available tools, skipping the advisor itself", async () => {
+    const context = await buildAdvisorContext({
+      conversationId: "ctx-1",
+      workingDir: "/tmp/does-not-exist",
+      allowedToolNames: new Set(["bash", "advisor", "read_file"]),
+      trustClass: "unknown",
+      transcript: [userMsg("hi")],
+    });
+
+    expect(context).toContain("## Available tools");
+    expect(context).toContain("- bash");
+    expect(context).toContain("- read_file");
+    // The advisor advises; it never tells the agent to consult itself.
+    expect(context).not.toContain("- advisor");
+  });
+
+  test("omits the tools section when no tools are available", async () => {
+    const context = await buildAdvisorContext({
+      conversationId: "ctx-2",
+      workingDir: "/tmp/does-not-exist",
+      allowedToolNames: new Set(),
+      trustClass: "unknown",
+      transcript: [],
+    });
+    // Other sources (e.g. the skills catalog) may still contribute, but with no
+    // allowed tools the tools section must not appear.
+    if (context !== null) expect(context).not.toContain("## Available tools");
+  });
+});