@vellumai/assistant 0.10.0 → 0.10.1-staging.1

package/src/plugins/defaults/image-fallback/__tests__/image-fallback.test.ts

@@ -0,0 +1,302 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type {
+  ImageContent,
+  Message,
+  ModelProfileInfo,
+  UserPromptSubmitContext,
+} from "@vellumai/plugin-api";
+
+// ─── Mocks ──────────────────────────────────────────────────────────────────
+
+// Control doesSupportVision per-profile from the test.
+let visionProfiles: Set<string>;
+let mockProfiles: ModelProfileInfo[];
+let sendMessageResponse = {
+  content: [{ type: "text", text: "A red chart showing Q3 revenue." }],
+};
+let providerResolves = true;
+
+const fakeProvider = {
+  name: "mock-vision-provider",
+  async sendMessage() {
+    return sendMessageResponse;
+  },
+};
+
+// Mock @vellumai/plugin-api — only the runtime handles the plugin imports.
+// `extractAllText` stays real (imported from the relative path, not plugin-api).
+mock.module("@vellumai/plugin-api", () => ({
+  doesSupportVision: (profile: ModelProfileInfo) => visionProfiles.has(profile.key),
+  getModelProfiles: () => mockProfiles,
+  getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
+}));
+
+// Mock the image-persist module to avoid filesystem side effects in tests.
+let mockPersistPath: string | null = "/workspace/data/attachments/mock-hash.png";
+mock.module("../src/image-persist.js", () => ({
+  persistImage: () => mockPersistPath,
+}));
+
+// ─── Imports (after mocks are registered) ───────────────────────────────────
+
+const userPromptSubmit = (await import("../hooks/user-prompt-submit.js")).default;
+const { findVisionProfile } = await import("../src/vision-caption.js");
+const { resetCaptionCacheForTests } = await import("../src/caption-cache.js");
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+const logger = {
+  info() {},
+  warn() {},
+  error() {},
+  debug() {},
+  warnOnce() {},
+};
+
+function profile(
+  key: string,
+  overrides: Partial<ModelProfileInfo> = {},
+): ModelProfileInfo {
+  return {
+    key,
+    label: key,
+    description: null,
+    isActive: false,
+    isDisabled: false,
+    isMix: false,
+    ...overrides,
+  };
+}
+
+function imageBlock(data = "base64data"): ImageContent {
+  return {
+    type: "image",
+    source: { type: "base64", media_type: "image/png", data },
+  };
+}
+
+function imageMsg(data = "base64data"): Message {
+  return { role: "user", content: [imageBlock(data)] };
+}
+
+function textMsg(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+function makeCtx(
+  overrides: Partial<UserPromptSubmitContext> = {},
+): UserPromptSubmitContext {
+  return {
+    conversationId: "c1",
+    userMessageId: "m1",
+    requestId: "r1",
+    modelProfileKey: "text-only",
+    isNonInteractive: false,
+    prompt: "What is in this image?",
+    originalMessages: [],
+    latestMessages: [],
+    logger,
+    ...overrides,
+  } as unknown as UserPromptSubmitContext;
+}
+
+// ─── Setup ──────────────────────────────────────────────────────────────────
+
+beforeEach(() => {
+  visionProfiles = new Set<string>(["vision-profile"]);
+  mockProfiles = [
+    profile("text-only", { label: "Text Only", isActive: true }),
+    profile("vision-profile", { label: "Vision" }),
+  ];
+  sendMessageResponse = {
+    content: [{ type: "text", text: "A red chart showing Q3 revenue." }],
+  };
+  providerResolves = true;
+  mockPersistPath = "/workspace/data/attachments/mock-hash.png";
+  resetCaptionCacheForTests();
+});
+
+// ─── Tests ──────────────────────────────────────────────────────────────────
+
+describe("image-fallback user-prompt-submit hook", () => {
+  test("is a no-op when the active model supports vision", async () => {
+    visionProfiles = new Set(["text-only"]); // active profile supports vision
+    const messages = [imageMsg()];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("image");
+  });
+
+  test("does not gate on isNonInteractive — captions even for background runs", async () => {
+    const messages = [imageMsg()];
+    const ctx = makeCtx({ latestMessages: messages, isNonInteractive: true });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
+      "[Image:",
+    );
+  });
+
+  test("replaces image blocks with captions when active model is text-only", async () => {
+    const messages = [imageMsg("img1")];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
+      "[Image: A red chart showing Q3 revenue.]",
+    );
+  });
+
+  test("references the saved image path in the caption text", async () => {
+    const messages = [imageMsg("img1")];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    const text = (ctx.latestMessages[0].content[0] as { text: string }).text;
+    expect(text).toContain("(saved to /workspace/data/attachments/");
+  });
+
+  test("works without a saved path when persist fails", async () => {
+    mockPersistPath = null;
+    const messages = [imageMsg("img1")];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    const text = (ctx.latestMessages[0].content[0] as { text: string }).text;
+    expect(text).toContain("[Image: A red chart showing Q3 revenue.]");
+    expect(text).not.toContain("(saved to");
+  });
+
+  test("preserves non-image blocks and captions only images", async () => {
+    const messages: Message[] = [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "Look at this:" },
+          imageBlock("img1"),
+          { type: "text", text: "What do you see?" },
+        ],
+      },
+    ];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toBe(
+      "Look at this:",
+    );
+    expect(ctx.latestMessages[0].content[1].type).toBe("text");
+    expect((ctx.latestMessages[0].content[1] as { text: string }).text).toContain(
+      "[Image:",
+    );
+    expect((ctx.latestMessages[0].content[2] as { text: string }).text).toBe(
+      "What do you see?",
+    );
+  });
+
+  test("uses fail-open placeholder when no vision profile is configured", async () => {
+    visionProfiles = new Set<string>(); // no vision profiles
+    const messages = [imageMsg()];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
+      "no vision-capable model",
+    );
+  });
+
+  test("uses fail-open placeholder when provider resolution returns null", async () => {
+    providerResolves = false;
+    const messages = [imageMsg()];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
+      "captioning failed",
+    );
+  });
+
+  test("caches captions — second call with same image does not invoke provider", async () => {
+    let callCount = 0;
+    const trackingProvider = {
+      name: "mock-vision-provider",
+      async sendMessage() {
+        callCount++;
+        return sendMessageResponse;
+      },
+    };
+    // Override the mock to track calls.
+    mock.module("@vellumai/plugin-api", () => ({
+      doesSupportVision: (p: ModelProfileInfo) => visionProfiles.has(p.key),
+      getModelProfiles: () => mockProfiles,
+      getConfiguredProvider: async () => trackingProvider,
+    }));
+
+    const messages1 = [imageMsg("same-data")];
+    const ctx1 = makeCtx({ latestMessages: messages1 });
+    await userPromptSubmit(ctx1);
+    expect(callCount).toBe(1);
+
+    // Second turn with the same image — should hit cache, no new provider call.
+    const messages2 = [imageMsg("same-data")];
+    const ctx2 = makeCtx({ latestMessages: messages2 });
+    await userPromptSubmit(ctx2);
+    expect(callCount).toBe(1); // still 1 — cache hit
+
+    // Restore the original mock for other tests.
+    mock.module("@vellumai/plugin-api", () => ({
+      doesSupportVision: (p: ModelProfileInfo) => visionProfiles.has(p.key),
+      getModelProfiles: () => mockProfiles,
+      getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
+    }));
+  });
+
+  test("handles multiple images across multiple messages", async () => {
+    const messages: Message[] = [
+      imageMsg("img-a"),
+      textMsg("and another:"),
+      {
+        role: "user",
+        content: [imageBlock("img-b"), { type: "text", text: "both?" }],
+      },
+    ];
+    const ctx = makeCtx({ latestMessages: messages });
+    await userPromptSubmit(ctx);
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
+      "[Image:",
+    );
+    expect(ctx.latestMessages[2].content[0].type).toBe("text");
+    expect((ctx.latestMessages[2].content[0] as { text: string }).text).toContain(
+      "[Image:",
+    );
+    expect((ctx.latestMessages[2].content[1] as { text: string }).text).toBe(
+      "both?",
+    );
+  });
+
+  test("resolves active profile via isActive when modelProfileKey is null", async () => {
+    const messages = [imageMsg()];
+    const ctx = makeCtx({ latestMessages: messages, modelProfileKey: null });
+    await userPromptSubmit(ctx);
+    // The active profile is "text-only" (isActive: true), which doesn't support
+    // vision, so images should be captioned.
+    expect(ctx.latestMessages[0].content[0].type).toBe("text");
+  });
+});
+
+describe("findVisionProfile", () => {
+  test("returns the first enabled vision-capable profile", () => {
+    expect(findVisionProfile()).toBe("vision-profile");
+  });
+
+  test("skips disabled vision profiles", () => {
+    mockProfiles = [
+      profile("text-only", { label: "Text", isActive: true }),
+      profile("vision-profile", { label: "Vision", isDisabled: true }),
+    ];
+    expect(findVisionProfile()).toBeNull();
+  });
+
+  test("returns null when no profiles support vision", () => {
+    visionProfiles = new Set<string>();
+    expect(findVisionProfile()).toBeNull();
+  });
+});

package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts

@@ -0,0 +1,103 @@
+/**
+ * Default `user-prompt-submit` hook: when the active model is text-only,
+ * captions image blocks via a vision-capable profile and substitutes the
+ * caption as a text block so the model can still reason about the image's
+ * content.
+ *
+ * The hook runs once per user turn, after the assistant assembles
+ * `latestMessages` and before they flow into `agentLoop.run()`. It:
+ *
+ * 1. Resolves the active profile from `modelProfileKey` (or the workspace's
+ *    active profile when the key is `null`) and checks `doesSupportVision`.
+ *    If the model already handles images, the hook is a no-op.
+ * 2. Finds a vision-capable profile for captioning via `findVisionProfile`.
+ *    If none exists, images are replaced with a fail-open placeholder so the
+ *    model at least knows an image was present.
+ * 3. Persists each image to the workspace attachments directory (content-hash
+ *    deduped) so the original image is accessible to future vision-capable
+ *    turns or subagents.
+ * 4. Captions each `ImageContent` block through the `vision` call site (with
+ *    an in-memory content-hash cache to avoid re-captioning across turns), and
+ *    replaces the block with `[Image: <caption>] (saved to <path>)`.
+ *
+ * Fail-open is the dominant error mode: a captioning failure leaves a
+ * placeholder text block (with the saved image path) rather than the raw
+ * image (which would cause a provider rejection on a text-only model) or
+ * dropping the image entirely (which would lose information).
+ */
+
+import {
+  doesSupportVision,
+  getModelProfiles,
+  type ImageContent,
+  type PluginHookFn,
+  type UserPromptSubmitContext,
+} from "@vellumai/plugin-api";
+
+import { persistImage } from "../src/image-persist.js";
+import { captionImage, findVisionProfile } from "../src/vision-caption.js";
+
+const userPromptSubmit: PluginHookFn<UserPromptSubmitContext> = async (ctx) => {
+  // Resolve the active profile from modelProfileKey, falling back to the
+  // workspace's active profile when the key is null (profile unchanged since
+  // the last notified turn).
+  const profiles = getModelProfiles();
+  const activeProfile =
+    ctx.modelProfileKey != null
+      ? profiles.find((p) => p.key === ctx.modelProfileKey)
+      : profiles.find((p) => p.isActive);
+  if (activeProfile == null) return;
+
+  // If the active model already supports vision, nothing to do.
+  if (doesSupportVision(activeProfile)) return;
+
+  // Find a vision-capable profile for captioning.
+  const visionProfileKey = findVisionProfile();
+
+  // Scan all messages for image blocks and replace them with captions.
+  let imageCount = 0;
+  for (const message of ctx.latestMessages) {
+    for (let i = 0; i < message.content.length; i++) {
+      const block = message.content[i];
+      if (block.type !== "image") continue;
+
+      imageCount++;
+      const image = block as ImageContent;
+
+      // Persist the image to the workspace so it's accessible to future
+      // vision-capable turns or subagents.
+      const savedPath = persistImage(
+        image.source.data,
+        image.source.media_type,
+      );
+
+      if (visionProfileKey != null) {
+        const caption = await captionImage(image, visionProfileKey, ctx.logger);
+        const pathSuffix = savedPath != null ? ` (saved to ${savedPath})` : "";
+        message.content[i] = {
+          type: "text",
+          text:
+            caption != null
+              ? `[Image: ${caption}]${pathSuffix}`
+              : `[Image: captioning failed — unable to describe]${pathSuffix}`,
+        };
+      } else {
+        // No vision profile configured at all — fail-open placeholder.
+        const pathSuffix = savedPath != null ? ` (saved to ${savedPath})` : "";
+        message.content[i] = {
+          type: "text",
+          text: `[Image: no vision-capable model configured to describe this image]${pathSuffix}`,
+        };
+      }
+    }
+  }
+
+  if (imageCount > 0) {
+    ctx.logger.info(
+      { plugin: "image-fallback", imageCount },
+      "Replaced image blocks with text captions for text-only model",
+    );
+  }
+};
+
+export default userPromptSubmit;

package/src/plugins/defaults/image-fallback/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "default-image-fallback",
+  "version": "1.0.0",
+  "description": "First-party default plugin that captions image attachments via a vision-capable profile when the active model is text-only, substituting the caption as a text block so the model can still reason about the image's content.",
+  "private": true,
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "node": ">=20.12.0"
+  },
+  "peerDependencies": {
+    "@vellumai/plugin-api": "^0.8.0"
+  }
+}

package/src/plugins/defaults/image-fallback/src/caption-cache.ts

@@ -0,0 +1,49 @@
+/**
+ * In-memory LRU cache for image captions, keyed by the sha-256 hash of the
+ * image's base64 data.
+ *
+ * The cache survives across turns within a session, which is the primary
+ * correctness concern: without it, the same image would be re-captioned
+ * (and re-billed) on every turn because `user-prompt-submit` rebuilds
+ * `latestMessages` from the stored conversation each time. Re-captioning
+ * on a daemon restart is acceptable — captions are cheap and the image
+ * is still in the history.
+ */
+
+import { createHash } from "node:crypto";
+
+const MAX_ENTRIES = 500;
+
+const cache = new Map<string, string>();
+
+/** sha-256 hex digest of an image's base64 payload. */
+export function imageHash(data: string): string {
+  return createHash("sha256").update(data).digest("hex");
+}
+
+/** Look up a cached caption. `undefined` = miss; a string (even empty) = hit. */
+export function getCachedCaption(hash: string): string | undefined {
+  const value = cache.get(hash);
+  if (value !== undefined) {
+    // Move to end (most-recently-used) for LRU eviction.
+    cache.delete(hash);
+    cache.set(hash, value);
+  }
+  return value;
+}
+
+/** Store a caption, evicting the least-recently-used entry if at capacity. */
+export function setCachedCaption(hash: string, caption: string): void {
+  if (cache.size >= MAX_ENTRIES) {
+    const oldest = cache.keys().next();
+    if (!oldest.done) {
+      cache.delete(oldest.value as string);
+    }
+  }
+  cache.set(hash, caption);
+}
+
+/** Test-only: clear the cache. */
+export function resetCaptionCacheForTests(): void {
+  cache.clear();
+}

package/src/plugins/defaults/image-fallback/src/image-persist.ts

@@ -0,0 +1,59 @@
+/**
+ * Persist image data to the workspace attachments directory.
+ *
+ * When the active model is text-only, the image-fallback plugin captions the
+ * image and substitutes a text block. Saving the raw image to disk and
+ * referencing the path in the caption text means a future turn with a
+ * vision-capable model (or a subagent) could still access the original image
+ * via file_read, and the user can find the image at a known location.
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { imageHash } from "./caption-cache.js";
+
+/** The workspace attachments directory. */
+const ATTACHMENTS_DIR = "/workspace/data/attachments";
+
+/** File extension for a given media type, falling back to `.bin`. */
+function extensionForMediaType(mediaType: string): string {
+  switch (mediaType) {
+    case "image/png":
+      return ".png";
+    case "image/jpeg":
+      return ".jpg";
+    case "image/gif":
+      return ".gif";
+    case "image/webp":
+      return ".webp";
+    default:
+      return ".bin";
+  }
+}
+
+/**
+ * Save an image's base64 data to the attachments dir if not already present.
+ * Returns the absolute file path, or `null` when the write fails.
+ */
+export function persistImage(
+  data: string,
+  mediaType: string,
+): string | null {
+  try {
+    mkdirSync(ATTACHMENTS_DIR, { recursive: true });
+
+    const hash = imageHash(data);
+    const ext = extensionForMediaType(mediaType);
+    const filename = `${hash}${ext}`;
+    const filepath = join(ATTACHMENTS_DIR, filename);
+
+    // Skip if already saved (content-hash dedup).
+    if (existsSync(filepath)) return filepath;
+
+    writeFileSync(filepath, Buffer.from(data, "base64"));
+    return filepath;
+  } catch {
+    return null;
+  }
+}

package/src/plugins/defaults/image-fallback/src/vision-caption.ts

@@ -0,0 +1,120 @@
+/**
+ * Vision-based image captioning for the image-fallback plugin.
+ *
+ * When the active model cannot process images, this module finds a
+ * vision-capable profile in the workspace's configured profiles and runs a
+ * one-shot captioning call through the assistant's own inference (no
+ * plugin-supplied API key). The caption replaces the image block in the
+ * outgoing message history.
+ */
+
+import {
+  doesSupportVision,
+  getConfiguredProvider,
+  getModelProfiles,
+  type ImageContent,
+  type PluginLogger,
+} from "@vellumai/plugin-api";
+
+import { extractAllText } from "../../../../providers/provider-send-message.js";
+import {
+  getCachedCaption,
+  imageHash,
+  setCachedCaption,
+} from "./caption-cache.js";
+
+const CAPTION_TIMEOUT_MS = 30_000;
+
+const CAPTION_SYSTEM_PROMPT =
+  "You are a vision assistant. Describe the image concisely in 1-2 sentences. " +
+  "Focus on the key visual content, text, charts, or UI elements that would be " +
+  "relevant for a text-based assistant to understand and reason about.";
+
+const CAPTION_USER_PROMPT = "Describe this image concisely for a text-only assistant.";
+
+/**
+ * Find a vision-capable, enabled profile key for captioning.
+ *
+ * Scans the workspace's profiles in `getModelProfiles()` order (the same order
+ * the `/model` picker shows them) and returns the first enabled profile whose
+ * resolved model supports vision. Returns `null` when no vision profile exists
+ * — the hook fails-open in that case, leaving a placeholder text block.
+ */
+export function findVisionProfile(): string | null {
+  for (const profile of getModelProfiles()) {
+    if (profile.isDisabled) continue;
+    if (doesSupportVision(profile)) {
+      return profile.key;
+    }
+  }
+  return null;
+}
+
+/**
+ * Caption a single image block via a vision-capable profile.
+ *
+ * @param image     The image content block to caption.
+ * @param profileKey  Key of a vision-capable profile (from {@link findVisionProfile}).
+ * @param logger    Turn-scoped logger for attribution.
+ * @returns The caption text, or `null` when captioning failed (caller should
+ *          use a fail-open placeholder).
+ */
+export async function captionImage(
+  image: ImageContent,
+  profileKey: string,
+  logger: PluginLogger,
+): Promise<string | null> {
+  const hash = imageHash(image.source.data);
+  const cached = getCachedCaption(hash);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  try {
+    const provider = await getConfiguredProvider("vision", {
+      overrideProfile: profileKey,
+      forceOverrideProfile: true,
+    });
+    if (!provider) {
+      logger.warn(
+        { plugin: "image-fallback" },
+        "No provider resolved for vision captioning profile",
+      );
+      return null;
+    }
+
+    const response = await provider.sendMessage(
+      [
+        {
+          role: "user",
+          content: [image, { type: "text", text: CAPTION_USER_PROMPT }],
+        },
+      ],
+      {
+        systemPrompt: CAPTION_SYSTEM_PROMPT,
+        config: {
+          callSite: "vision",
+          overrideProfile: profileKey,
+          forceOverrideProfile: true,
+          tool_choice: { type: "none" },
+        },
+        signal: AbortSignal.timeout(CAPTION_TIMEOUT_MS),
+      },
+    );
+
+    const caption = extractAllText(response).trim();
+    if (caption.length > 0) {
+      setCachedCaption(hash, caption);
+      return caption;
+    }
+
+    logger.warn({ plugin: "image-fallback" }, "Vision captioning returned empty text");
+    return null;
+  } catch (err) {
+    logger.warn(
+      { plugin: "image-fallback", err },
+      "Vision captioning call failed",
+    );
+    return null;
+  }
+}