@vellumai/assistant 0.10.1-staging.1 → 0.10.1-staging.2

package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts

@@ -13,29 +13,23 @@
  * 2. Finds a vision-capable profile for captioning via `findVisionProfile`.
  *    If none exists, images are replaced with a fail-open placeholder so the
  *    model at least knows an image was present.
- * 3. Persists each image to the workspace attachments directory (content-hash
- *    deduped) so the original image is accessible to future vision-capable
- *    turns or subagents.
- * 4. Captions each `ImageContent` block through the `vision` call site (with
- *    an in-memory content-hash cache to avoid re-captioning across turns), and
- *    replaces the block with `[Image: <caption>] (saved to <path>)`.
+ * 3. Replaces each `ImageContent` block with a `[Image …]` text caption via
+ *    {@link captionImageBlocks} (which also persists the original and caches
+ *    captions across turns).
  *
- * Fail-open is the dominant error mode: a captioning failure leaves a
- * placeholder text block (with the saved image path) rather than the raw
- * image (which would cause a provider rejection on a text-only model) or
- * dropping the image entirely (which would lose information).
+ * The companion `post-tool-use` hook applies the same substitution to images a
+ * tool returns (e.g. a browser screenshot).
  */
 
 import {
   doesSupportVision,
   getModelProfiles,
-  type ImageContent,
   type PluginHookFn,
   type UserPromptSubmitContext,
 } from "@vellumai/plugin-api";
 
-import { persistImage } from "../src/image-persist.js";
-import { captionImage, findVisionProfile } from "../src/vision-caption.js";
+import { captionImageBlocks } from "../src/caption-blocks.js";
+import { findVisionProfile } from "../src/vision-caption.js";
 
 const userPromptSubmit: PluginHookFn<UserPromptSubmitContext> = async (ctx) => {
   // Resolve the active profile from modelProfileKey, falling back to the
@@ -57,39 +51,11 @@ const userPromptSubmit: PluginHookFn<UserPromptSubmitContext> = async (ctx) => {
   // Scan all messages for image blocks and replace them with captions.
   let imageCount = 0;
   for (const message of ctx.latestMessages) {
-    for (let i = 0; i < message.content.length; i++) {
-      const block = message.content[i];
-      if (block.type !== "image") continue;
-
-      imageCount++;
-      const image = block as ImageContent;
-
-      // Persist the image to the workspace so it's accessible to future
-      // vision-capable turns or subagents.
-      const savedPath = persistImage(
-        image.source.data,
-        image.source.media_type,
-      );
-
-      if (visionProfileKey != null) {
-        const caption = await captionImage(image, visionProfileKey, ctx.logger);
-        const pathSuffix = savedPath != null ? ` (saved to ${savedPath})` : "";
-        message.content[i] = {
-          type: "text",
-          text:
-            caption != null
-              ? `[Image: ${caption}]${pathSuffix}`
-              : `[Image: captioning failed — unable to describe]${pathSuffix}`,
-        };
-      } else {
-        // No vision profile configured at all — fail-open placeholder.
-        const pathSuffix = savedPath != null ? ` (saved to ${savedPath})` : "";
-        message.content[i] = {
-          type: "text",
-          text: `[Image: no vision-capable model configured to describe this image]${pathSuffix}`,
-        };
-      }
-    }
+    imageCount += await captionImageBlocks(
+      message.content,
+      visionProfileKey,
+      ctx.logger,
+    );
   }
 
   if (imageCount > 0) {

package/src/plugins/defaults/image-fallback/src/caption-blocks.ts

@@ -0,0 +1,77 @@
+/**
+ * Shared image→text substitution for the image-fallback plugin's hooks.
+ *
+ * Two hooks replace `image` content blocks with a text caption when the active
+ * model can't process images: `user-prompt-submit` handles user-attached
+ * images, and `post-tool-use` handles images a tool returns (e.g. a browser
+ * screenshot). This module holds the per-block substitution they share —
+ * persist the original image to a known location, caption it via a
+ * vision-capable profile, and swap in a `[Image …]` text block.
+ *
+ * The caption text states up front that the active model can't view images and
+ * the image was auto-described to text, so the model treats the block as a
+ * derived description rather than a verbatim transcript.
+ *
+ * Fail-open is the dominant error mode: a captioning failure leaves a
+ * placeholder text block rather than the raw image (which a text-only provider
+ * would reject) or nothing (which would lose information).
+ */
+
+import type {
+  ContentBlock,
+  ImageContent,
+  PluginLogger,
+} from "@vellumai/plugin-api";
+
+import { persistImage } from "./image-persist.js";
+import { captionImage } from "./vision-caption.js";
+
+/**
+ * Replace every `image` block in `blocks` (in place) with a text caption so a
+ * text-only model can still reason about the image's content. Returns the
+ * number of image blocks replaced.
+ *
+ * @param blocks            Content-block array to scan and mutate in place.
+ * @param visionProfileKey  Key of a vision-capable profile for captioning, or
+ *                          `null` when none is configured (fail-open
+ *                          placeholder).
+ * @param logger            Turn-scoped logger for attribution.
+ */
+export async function captionImageBlocks(
+  blocks: ContentBlock[],
+  visionProfileKey: string | null,
+  logger: PluginLogger,
+): Promise<number> {
+  let imageCount = 0;
+
+  for (let i = 0; i < blocks.length; i++) {
+    const block = blocks[i];
+    if (block.type !== "image") continue;
+
+    imageCount++;
+    const image = block as ImageContent;
+
+    // Persist the original to a known, content-hash-deduped location so it
+    // survives the text substitution and stays findable on disk.
+    persistImage(image.source.data, image.source.media_type);
+
+    if (visionProfileKey != null) {
+      const caption = await captionImage(image, visionProfileKey, logger);
+      blocks[i] = {
+        type: "text",
+        text:
+          caption != null
+            ? `[Image auto-described for text-only model: ${caption}]`
+            : `[Image: auto-description failed (text-only model)]`,
+      };
+    } else {
+      // No vision profile configured at all — fail-open placeholder.
+      blocks[i] = {
+        type: "text",
+        text: `[Image: no vision-capable model configured to describe it]`,
+      };
+    }
+  }
+
+  return imageCount;
+}

package/src/plugins/defaults/image-fallback/src/image-persist.ts

@@ -2,10 +2,10 @@
  * Persist image data to the workspace attachments directory.
  *
  * When the active model is text-only, the image-fallback plugin captions the
- * image and substitutes a text block. Saving the raw image to disk and
- * referencing the path in the caption text means a future turn with a
- * vision-capable model (or a subagent) could still access the original image
- * via file_read, and the user can find the image at a known location.
+ * image and substitutes a text block. Saving the raw image to a known,
+ * content-hash-deduped location means the original survives the text
+ * substitution and stays findable on disk for the user (or a subagent with a
+ * vision-capable model that reads it via file_read).
  */
 
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
@@ -36,10 +36,7 @@ function extensionForMediaType(mediaType: string): string {
  * Save an image's base64 data to the attachments dir if not already present.
  * Returns the absolute file path, or `null` when the write fails.
  */
-export function persistImage(
-  data: string,
-  mediaType: string,
-): string | null {
+export function persistImage(data: string, mediaType: string): string | null {
   try {
     mkdirSync(ATTACHMENTS_DIR, { recursive: true });
 

package/src/plugins/defaults/index.ts

@@ -47,6 +47,7 @@ import historyRepairStop from "./history-repair/hooks/stop.js";
 import historyRepairUserPromptSubmit from "./history-repair/hooks/user-prompt-submit.js";
 import historyRepairPkg from "./history-repair/package.json" with { type: "json" };
 import { resetRepairStateStoreForTests } from "./history-repair/repair-state-store.js";
+import imageFallbackPostToolUse from "./image-fallback/hooks/post-tool-use.js";
 import imageFallbackUserPromptSubmit from "./image-fallback/hooks/user-prompt-submit.js";
 import imageFallbackPkg from "./image-fallback/package.json" with { type: "json" };
 import { resetCaptionCacheForTests } from "./image-fallback/src/caption-cache.js";
@@ -81,12 +82,14 @@ import toolResultTruncatePostToolUse from "./tool-result-truncate/hooks/post-too
 import toolResultTruncatePkg from "./tool-result-truncate/package.json" with { type: "json" };
 
 /**
- * `image-fallback` — a `user-prompt-submit` hook that captions image blocks via
- * a vision-capable profile when the active model is text-only, substituting the
- * caption as a `[Image: <caption>]` text block so the model can still reason
- * about the image's content. Self-gates on `isNonInteractive`; fail-open with a
- * placeholder when no vision profile is configured or captioning fails. An
- * in-memory content-hash cache avoids re-captioning the same image across turns.
+ * `image-fallback` — captions image blocks via a vision-capable profile when
+ * the active model is text-only, substituting the caption as an `[Image …]`
+ * text block so the model can still reason about the image's content. The
+ * `user-prompt-submit` hook handles user-attached images; the `post-tool-use`
+ * hook handles images a tool returns (e.g. a browser screenshot) nested in the
+ * tool result's `contentBlocks`. Fail-open with a placeholder when no vision
+ * profile is configured or captioning fails. An in-memory content-hash cache
+ * avoids re-captioning the same image across turns.
  */
 export const defaultImageFallbackPlugin: Plugin = {
   manifest: {
@@ -95,6 +98,7 @@ export const defaultImageFallbackPlugin: Plugin = {
   },
   hooks: {
     "user-prompt-submit": imageFallbackUserPromptSubmit,
+    "post-tool-use": imageFallbackPostToolUse,
   },
 };
 

package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts

@@ -49,6 +49,7 @@ interface ProviderCall {
   options: SendMessageOptions | undefined;
 }
 const providerCalls: ProviderCall[] = [];
+const warnCalls: Array<{ args: unknown[] }> = [];
 
 mock.module("../../../../providers/provider-send-message.js", () => ({
   getConfiguredProvider: async () => providerStub,
@@ -57,10 +58,12 @@ mock.module("../../../../providers/provider-send-message.js", () => ({
 }));
 
 mock.module("../../../../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: (_t, prop) => (prop === "child" ? () => ({}) : () => {}),
+  getLogger: () => ({
+    warn: (...args: unknown[]) => warnCalls.push({ args }),
+    child: () => ({
+      warn: (...args: unknown[]) => warnCalls.push({ args }),
     }),
+  }),
 }));
 
 const { selectPool, MemoryV3RetrievalUnavailableError } =
@@ -97,10 +100,21 @@ function noToolResponse(): ProviderResponse {
     model: "stub-model",
     stopReason: "end_turn",
     usage: { inputTokens: 0, outputTokens: 0 },
+    rawRequest: { model: "MiniMaxAI/MiniMax-M3" },
+    rawResponse: { model: "accounts/fireworks/models/minimax-m3" },
     content: [{ type: "text", text: "no tool call" }],
   };
 }
 
+function wrongToolResponse(): ProviderResponse {
+  return {
+    model: "stub-model",
+    stopReason: "tool_use",
+    usage: { inputTokens: 0, outputTokens: 0 },
+    content: [{ type: "tool_use", id: "tu-1", name: "wrong_tool", input: {} }],
+  };
+}
+
 /** Provider returning a different response per call (the i-th call returns
  * responses[i], or the last entry once exhausted). */
 function makeSequenceProvider(responses: ProviderResponse[]): Provider {
@@ -118,12 +132,12 @@ function makeSequenceProvider(responses: ProviderResponse[]): Provider {
 
 /** Provider that records each call and then throws — the throw-after-retries
  * path (the provider's own RetryProvider has already exhausted its backoff). */
-function makeThrowingProvider(): Provider {
+function makeThrowingProvider(message = "boom"): Provider {
   return {
     name: "throwing",
     sendMessage: async (messages, options) => {
       providerCalls.push({ messages, options });
-      throw new Error("boom");
+      throw new Error(message);
     },
   };
 }
@@ -167,9 +181,19 @@ function sentBlocks(callIndex = 0): RenderedBlock[] {
     .content as unknown as RenderedBlock[];
 }
 
+function warnPayloads(): Array<Record<string, unknown>> {
+  return warnCalls
+    .map((call) => call.args[0])
+    .filter(
+      (payload): payload is Record<string, unknown> =>
+        payload !== null && typeof payload === "object",
+    );
+}
+
 beforeEach(() => {
   providerStub = null;
   providerCalls.length = 0;
+  warnCalls.length = 0;
 });
 
 // ---------------------------------------------------------------------------
@@ -250,6 +274,62 @@ describe("selectPool — infrastructure failures throw", () => {
       MemoryV3RetrievalUnavailableError,
     );
     expect(providerCalls).toHaveLength(3);
+    const payloads = warnPayloads();
+    const attemptPayloads = payloads.filter(
+      (payload) => payload.reason === "missing_tool_use",
+    );
+    expect(attemptPayloads).toHaveLength(3);
+    expect(attemptPayloads[0]).toMatchObject({
+      attempt: 1,
+      reason: "missing_tool_use",
+      providerName: "stub",
+      candidateCount: 4,
+      stableCount: 2,
+      finderCount: 2,
+      response: {
+        model: "stub-model",
+        stopReason: "end_turn",
+        requestModel: "MiniMaxAI/MiniMax-M3",
+        responseModel: "accounts/fireworks/models/minimax-m3",
+        contentBlockTypes: ["text"],
+        toolUseNames: [],
+      },
+    });
+    const aggregatePayload = payloads.find((payload) =>
+      Array.isArray(payload.failures),
+    );
+    expect(aggregatePayload?.providerName).toBe("stub");
+    const failures = aggregatePayload?.failures as
+      | Array<Record<string, unknown>>
+      | undefined;
+    expect(failures?.[0]).toMatchObject({ reason: "missing_tool_use" });
+  });
+
+  test("wrong tool_use name logs the unexpected name before throwing", async () => {
+    providerStub = makeProvider(wrongToolResponse());
+    await expect(selectPool(makePool(), makeTurn("x"))).rejects.toThrow(
+      MemoryV3RetrievalUnavailableError,
+    );
+    expect(providerCalls).toHaveLength(3);
+    expect(
+      warnPayloads().filter(
+        (payload) => payload.reason === "unexpected_tool_name",
+      ),
+    ).toEqual([
+      expect.objectContaining({
+        attempt: 1,
+        reason: "unexpected_tool_name",
+        providerName: "stub",
+        toolName: "wrong_tool",
+        response: expect.objectContaining({
+          stopReason: "tool_use",
+          contentBlockTypes: ["tool_use"],
+          toolUseNames: ["wrong_tool"],
+        }),
+      }),
+      expect.objectContaining({ attempt: 2 }),
+      expect.objectContaining({ attempt: 3 }),
+    ]);
   });
 
   test("schema mismatch → throws after retrying", async () => {
@@ -258,6 +338,17 @@ describe("selectPool — infrastructure failures throw", () => {
       MemoryV3RetrievalUnavailableError,
     );
     expect(providerCalls).toHaveLength(3);
+    expect(
+      warnPayloads().filter((payload) => payload.reason === "schema_mismatch"),
+    ).toEqual([
+      expect.objectContaining({
+        attempt: 1,
+        reason: "schema_mismatch",
+        schemaIssues: [expect.objectContaining({ path: "ids" })],
+      }),
+      expect.objectContaining({ attempt: 2 }),
+      expect.objectContaining({ attempt: 3 }),
+    ]);
   });
 
   test("provider throw → throws after retrying", async () => {
@@ -266,6 +357,44 @@ describe("selectPool — infrastructure failures throw", () => {
       MemoryV3RetrievalUnavailableError,
     );
     expect(providerCalls).toHaveLength(3);
+    expect(
+      warnPayloads().filter((payload) => payload.reason === "provider_error"),
+    ).toEqual([
+      expect.objectContaining({
+        attempt: 1,
+        reason: "provider_error",
+        providerName: "throwing",
+        error: { name: "Error", message: "boom" },
+      }),
+      expect.objectContaining({ attempt: 2 }),
+      expect.objectContaining({ attempt: 3 }),
+    ]);
+  });
+
+  test("provider throw redacts sensitive message details in diagnostics", async () => {
+    const providerSecret = ["sk-proj-", "a".repeat(40)].join("");
+    const message = `provider rejected Authorization: Bearer ${providerSecret}`;
+    providerStub = makeThrowingProvider(message);
+
+    let thrown: unknown;
+    try {
+      await selectPool(makePool(), makeTurn("x"));
+    } catch (error) {
+      thrown = error;
+    }
+
+    expect(thrown).toBeInstanceOf(MemoryV3RetrievalUnavailableError);
+    expect((thrown as Error).message).not.toContain(providerSecret);
+    expect((thrown as Error).message).toContain("[REDACTED]");
+
+    const providerErrors = warnPayloads().filter(
+      (payload) => payload.reason === "provider_error",
+    );
+    const error = providerErrors[0]?.error as
+      | Record<string, unknown>
+      | undefined;
+    expect(error?.message).not.toContain(providerSecret);
+    expect(error?.message).toContain("[REDACTED]");
   });
 
   test("a malformed response that recovers on retry returns its pages", async () => {