@vellumai/assistant 0.10.7 → 0.10.8-dev.202607102228.5945895

package/src/plugins/defaults/image-fallback/hooks/post-compact.ts

@@ -0,0 +1,52 @@
+/**
+ * Default `post-compact` hook: when the compacted turn's model is text-only,
+ * re-sweeps the rebuilt history for image blocks and substitutes text
+ * captions, so the provider call the loop resumes with never carries a raw
+ * image to a model that rejects them.
+ *
+ * Compaction runs mid-turn, after the turn-start `user-prompt-submit` sweep,
+ * and rebuilds history from persistence — which keeps raw images (the hook
+ * pipeline hands hooks deep-cloned contexts, so earlier caption substitutions
+ * never reach persisted rows). Two paths re-surface images here: the
+ * compactor re-attaches retained images as top-level image blocks, and the
+ * verbatim tail restores tool results whose `contentBlocks` still carry their
+ * original images. Both are swept via {@link captionImagesInMessages};
+ * previously captioned images resolve from the caption cache without a vision
+ * call, and a captioning failure leaves a fail-open placeholder rather than
+ * the raw image.
+ */
+
+import {
+  type HookFunction,
+  type PostCompactContext,
+} from "@vellumai/plugin-api";
+
+import {
+  captionImagesInMessages,
+  needsImageFallback,
+} from "../src/caption-blocks.js";
+import { findVisionProfile } from "../src/vision-caption.js";
+
+const postCompact: HookFunction<PostCompactContext> = async (ctx) => {
+  // If the turn's model already supports vision, leave images in place.
+  if (!needsImageFallback(ctx.modelProfileKey)) return;
+
+  // Find a vision-capable profile for captioning.
+  const visionProfileKey = findVisionProfile();
+
+  const imageCount = await captionImagesInMessages(
+    ctx.history,
+    ctx.conversationId,
+    visionProfileKey,
+    ctx.logger,
+  );
+
+  if (imageCount > 0) {
+    ctx.logger.info(
+      { plugin: "image-fallback", imageCount },
+      "Replaced compacted-history image blocks with text captions for text-only model",
+    );
+  }
+};
+
+export default postCompact;

package/src/plugins/defaults/image-fallback/hooks/post-model-call.ts

@@ -0,0 +1,80 @@
+/**
+ * Default `post-model-call` hook: recovers from a provider vision-not-supported
+ * rejection.
+ *
+ * The turn-start sweep captions history images when the turn's profile is
+ * text-only, but raw images can still reach a non-vision model through paths
+ * the sweep does not cover — messages that join an in-flight turn (rapid-fire
+ * batches, subagent completions), catalog entries that call a model
+ * vision-capable when the serving endpoint rejects images, or provider
+ * fallback to a text-only model after the sweep decided. The provider's
+ * rejection is ground truth that image input cannot be sent, regardless of
+ * what the catalog claims. This hook recognizes the rejection class, captions
+ * every image block in the working history via the plugin's shared
+ * substitution (fail-open placeholders when captioning fails), and asks the
+ * loop to retry the call.
+ *
+ * Unlike the image-recovery plugin's image-too-large flow, nothing is
+ * persisted: the stored rows keep the raw image, which clients render and
+ * vision-capable profiles can still consume on later turns. The caption cache
+ * makes the re-sweeps on those turns lookup-only.
+ *
+ * Bounded to one pass per turn via the per-conversation recovery state: a
+ * second consecutive vision rejection means captioning could not clear the
+ * request of image input, so the hook leaves the error to surface rather than
+ * looping. The loop's per-run backstop caps these retries globally; this
+ * one-shot mark keeps a single recovery attempt per turn. This hook only ever
+ * marks the conversation; the sibling `stop` hook (see `./stop.ts`) clears the
+ * mark when the turn terminates, so the next turn recovers afresh.
+ *
+ * A finalized reply (the model returned content) is left untouched; only a
+ * provider rejection is this hook's to act on.
+ */
+
+import type { HookFunction, PostModelCallContext } from "@vellumai/plugin-api";
+
+import { captionImagesInMessages } from "../src/caption-blocks.js";
+import {
+  isVisionRecoveryAttempted,
+  markVisionRecoveryAttempted,
+} from "../src/recovery-state.js";
+import { findVisionProfile } from "../src/vision-caption.js";
+import { isVisionNotSupportedError } from "../src/vision-error.js";
+
+const postModelCall: HookFunction<PostModelCallContext> = async (ctx) => {
+  if (
+    !ctx.error ||
+    !isVisionNotSupportedError(ctx.error.message) ||
+    isVisionRecoveryAttempted(ctx.conversationId)
+  ) {
+    return;
+  }
+  markVisionRecoveryAttempted(ctx.conversationId);
+
+  const visionProfileKey = findVisionProfile();
+  const captioned = await captionImagesInMessages(
+    ctx.messages,
+    ctx.conversationId,
+    visionProfileKey,
+    ctx.logger,
+  );
+
+  if (captioned === 0) {
+    // The rejection names image input but the working history holds no image
+    // blocks to caption — retrying the identical request would just re-reject,
+    // so leave the error to surface.
+    ctx.logger.warn(
+      { plugin: "image-fallback" },
+      "Provider vision-not-supported error but no image blocks found in history; not retrying",
+    );
+    return;
+  }
+
+  ctx.decision = "continue";
+  ctx.logger.warn(
+    { plugin: "image-fallback", captioned },
+    "Provider vision-not-supported error — captioned image blocks and retrying",
+  );
+};
+
+export default postModelCall;

package/src/plugins/defaults/image-fallback/hooks/post-tool-use.ts

@@ -6,13 +6,15 @@
  *
  * Tool images arrive nested in `toolResponse.contentBlocks` (the rich-content
  * companion to the tool result's text `content`), so the hook scans there
- * rather than the top-level message content the `user-prompt-submit` hook
- * handles. Both share {@link captionImageBlocks}.
+ * rather than the message-level sweep the `user-prompt-submit` and
+ * `post-compact` hooks run. All three share {@link captionImageBlocks}.
  *
  * Capability is read straight off `ctx.model` — the provider-reported model id
  * for the turn that issued this tool call — so the decision tracks the model
- * that actually ran, including a text-only override. The substitution is in
- * place, so the persisted/displayed tool result carries the caption too.
+ * that actually ran, including a text-only override. The hook receives a deep
+ * clone of the tool result, so the caption reaches the provider-bound history
+ * only — the persisted/displayed tool result keeps the original image, and
+ * later rebuild-from-persistence sweeps re-caption it from the cache.
  */
 
 import {
@@ -38,6 +40,7 @@ const postToolUse: HookFunction<PostToolUseContext> = async (ctx) => {
 
   const imageCount = await captionImageBlocks(
     blocks,
+    ctx.conversationId,
     visionProfileKey,
     ctx.logger,
   );

package/src/plugins/defaults/image-fallback/hooks/shutdown.ts

@@ -0,0 +1,14 @@
+/**
+ * Default `shutdown` hook: closes the plugin-owned caption database so a
+ * daemon shutdown or an in-place plugin redeploy never leaks the handle.
+ */
+
+import { type HookFunction, type ShutdownContext } from "@vellumai/plugin-api";
+
+import { closeCaptionStore } from "../src/caption-cache.js";
+
+const shutdown: HookFunction<ShutdownContext> = async () => {
+  closeCaptionStore();
+};
+
+export default shutdown;

package/src/plugins/defaults/image-fallback/hooks/stop.ts

@@ -0,0 +1,23 @@
+/**
+ * Default `stop` hook: clears the per-conversation vision-recovery bound when
+ * a turn terminates.
+ *
+ * The `post-model-call` hook (see `./post-model-call.ts`) marks the bound when
+ * it captions raw image blocks after a vision-not-supported rejection and asks
+ * the loop to retry. `stop` is the definitive terminal hook — it fires exactly
+ * once when the turn is truly ending, after every retry decision has been made
+ * — so clearing the bound here unconditionally guarantees the next turn always
+ * recovers afresh, no matter how the turn ended (a finalized reply, an
+ * unrecovered rejection, an abort, or a retry the loop's per-run backstop
+ * refused).
+ */
+
+import type { HookFunction, StopContext } from "@vellumai/plugin-api";
+
+import { clearVisionRecoveryAttempted } from "../src/recovery-state.js";
+
+const stop: HookFunction<StopContext> = async (ctx) => {
+  clearVisionRecoveryAttempted(ctx.conversationId);
+};
+
+export default stop;

package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts

@@ -14,11 +14,13 @@
  *    If none exists, images are replaced with a fail-open placeholder so the
  *    model at least knows an image was present.
  * 3. Replaces each image block with a `[Image …]` text caption via
- *    {@link captionImageBlocks} (which also persists the original and caches
- *    captions across turns).
+ *    {@link captionImagesInMessages} (which also persists the original and
+ *    caches captions across turns), sweeping top-level blocks and images
+ *    nested in `tool_result` blocks alike.
  *
  * The companion `post-tool-use` hook applies the same substitution to images a
- * tool returns (e.g. a browser screenshot).
+ * tool returns (e.g. a browser screenshot), and `post-compact` re-sweeps the
+ * rebuilt history after a mid-turn compaction.
  */
 
 import {
@@ -27,7 +29,7 @@ import {
 } from "@vellumai/plugin-api";
 
 import {
-  captionImageBlocks,
+  captionImagesInMessages,
   needsImageFallback,
 } from "../src/caption-blocks.js";
 import { findVisionProfile } from "../src/vision-caption.js";
@@ -40,14 +42,12 @@ const userPromptSubmit: HookFunction<UserPromptSubmitContext> = async (ctx) => {
   const visionProfileKey = findVisionProfile();
 
   // Scan all messages for image blocks and replace them with captions.
-  let imageCount = 0;
-  for (const message of ctx.latestMessages) {
-    imageCount += await captionImageBlocks(
-      message.content,
-      visionProfileKey,
-      ctx.logger,
-    );
-  }
+  const imageCount = await captionImagesInMessages(
+    ctx.latestMessages,
+    ctx.conversationId,
+    visionProfileKey,
+    ctx.logger,
+  );
 
   if (imageCount > 0) {
     ctx.logger.info(

package/src/plugins/defaults/image-fallback/src/caption-blocks.ts

@@ -1,19 +1,24 @@
 /**
  * Shared image→text substitution for the image-fallback plugin's hooks.
  *
- * Two hooks replace `image` content blocks with a text caption when the turn's
- * model can't process images: `user-prompt-submit` handles user-attached
- * images, and `post-tool-use` handles images a tool returns (e.g. a browser
- * screenshot). This module holds what they share — deciding whether a profile
- * needs the fallback ({@link needsImageFallback}) and the per-block
- * substitution ({@link captionImageBlocks}): persist the original image to a
- * known location, caption it via a vision-capable profile, and swap in a
- * `[Image …]` text block.
+ * Three hooks replace `image` content blocks with a text caption when the
+ * turn's model can't process images: `user-prompt-submit` sweeps the turn's
+ * history at turn start, `post-tool-use` handles images a tool returns (e.g. a
+ * browser screenshot) as they arrive, and `post-compact` re-sweeps the rebuilt
+ * history after a mid-turn compaction. This module holds what they share —
+ * deciding whether a profile needs the fallback ({@link needsImageFallback}),
+ * the per-block substitution ({@link captionImageBlocks}): persist the
+ * original image to a known location, caption it via a vision-capable
+ * profile, and swap in a `[Image …]` text block — and the message-level deep
+ * sweep ({@link captionImagesInMessages}) that reaches images nested inside
+ * `tool_result` blocks as well as top-level ones.
  *
- * The substitution mutates the blocks in place, so the caption replaces the
- * image everywhere the block is referenced (the provider-bound history and the
- * persisted/displayed copy alike) — a text-only turn does not keep the raw
- * image around.
+ * The substitution mutates the blocks in place, but the hook pipeline hands
+ * each hook a deep clone of its context, so the caption reaches only the
+ * provider-bound history — persisted rows keep the raw image (clients render
+ * it). Rebuild-from-persistence paths therefore re-surface raw images, which
+ * is why the sweeps re-run per turn and per compaction; the caption cache
+ * makes re-encounters lookup-only.
  *
  * The caption text states up front that the model can't view images and the
  * image was auto-described to text, so the model treats the block as a derived
@@ -29,7 +34,9 @@ import {
   doesSupportVision,
   getModelProfiles,
   type ImageContent,
+  type Message,
   type PluginLogger,
+  resolveMediaSourceData,
 } from "@vellumai/plugin-api";
 
 import { persistImage } from "./image-persist.js";
@@ -56,6 +63,9 @@ export function needsImageFallback(modelProfileKey: string): boolean {
  * number of image blocks replaced.
  *
  * @param blocks            Content-block array to scan and mutate in place.
+ * @param conversationId    Conversation the blocks belong to, recorded on the
+ *                          caption-cache rows so `conversation-deleted`
+ *                          cleanup stays accurate.
  * @param visionProfileKey  Key of a vision-capable profile for captioning, or
  *                          `null` when none is configured (fail-open
  *                          placeholder).
@@ -63,6 +73,7 @@ export function needsImageFallback(modelProfileKey: string): boolean {
  */
 export async function captionImageBlocks(
   blocks: ContentBlock[],
+  conversationId: string,
   visionProfileKey: string | null,
   logger: PluginLogger,
 ): Promise<number> {
@@ -76,11 +87,20 @@ export async function captionImageBlocks(
     const image = block as ImageContent;
 
     // Persist the original to a known, content-hash-deduped location so it
-    // survives the text substitution and stays findable on disk.
-    persistImage(image.source.data, image.source.media_type);
+    // survives the text substitution and stays findable on disk. Resolve a
+    // reference source to its bytes first (a no-op for inline base64).
+    const resolvedForPersist = resolveMediaSourceData(image.source);
+    if (resolvedForPersist) {
+      persistImage(resolvedForPersist.data, resolvedForPersist.media_type);
+    }
 
     if (visionProfileKey != null) {
-      const caption = await captionImage(image, visionProfileKey, logger);
+      const caption = await captionImage(
+        image,
+        conversationId,
+        visionProfileKey,
+        logger,
+      );
       blocks[i] = {
         type: "text",
         text:
@@ -99,3 +119,41 @@ export async function captionImageBlocks(
 
   return imageCount;
 }
+
+/**
+ * Deep-sweep a message list (in place) for image blocks and replace each with
+ * a text caption via {@link captionImageBlocks}. Covers both top-level image
+ * blocks (user-attached images, the compactor's retained-image message) and
+ * images nested in a `tool_result` block's rich `contentBlocks` (tool results
+ * restored from persistence carry their raw images there). Returns the number
+ * of image blocks replaced.
+ */
+export async function captionImagesInMessages(
+  messages: Message[],
+  conversationId: string,
+  visionProfileKey: string | null,
+  logger: PluginLogger,
+): Promise<number> {
+  let imageCount = 0;
+
+  for (const message of messages) {
+    imageCount += await captionImageBlocks(
+      message.content,
+      conversationId,
+      visionProfileKey,
+      logger,
+    );
+    for (const block of message.content) {
+      if (block.type === "tool_result" && block.contentBlocks != null) {
+        imageCount += await captionImageBlocks(
+          block.contentBlocks,
+          conversationId,
+          visionProfileKey,
+          logger,
+        );
+      }
+    }
+  }
+
+  return imageCount;
+}

package/src/plugins/defaults/image-fallback/src/caption-cache.ts

@@ -1,49 +1,235 @@
 /**
- * In-memory LRU cache for image captions, keyed by the sha-256 hash of the
- * image's base64 data.
+ * Read-through cache for image captions, keyed by the sha-256 hash of the
+ * image's base64 data: an in-memory LRU in front of a plugin-owned SQLite
+ * file (`caption-cache.sqlite` in the plugin's storage dir, opened by the
+ * `init` hook via {@link initCaptionStore}).
  *
- * The cache survives across turns within a session, which is the primary
- * correctness concern: without it, the same image would be re-captioned
- * (and re-billed) on every turn because `user-prompt-submit` rebuilds
- * `latestMessages` from the stored conversation each time. Re-captioning
- * on a daemon restart is acceptable — captions are cheap and the image
- * is still in the history.
+ * Raw images stay in persisted history (clients render them), so every path
+ * that rebuilds provider-bound context from persistence — each turn's
+ * `user-prompt-submit` sweep, the mid-turn `post-compact` sweep, a daemon
+ * restart — re-encounters the same images. The in-memory layer keeps
+ * same-session sweeps free of even a DB hit; the durable layer makes sweeps
+ * lookup-only across restarts, so a previously captioned image is never
+ * re-billed as a vision call.
+ *
+ * Rows are keyed `(image_hash, conversation_id)` while lookups match on the
+ * hash alone: the same image pasted in two conversations shares one caption
+ * (and one vision call), but each conversation that touched a hash holds its
+ * own row, so `conversation-deleted` cleanup ({@link
+ * deleteConversationCaptions}) removes exactly that conversation's derived
+ * data — a caption disappears entirely only when no surviving conversation
+ * references its image.
+ *
+ * The durable layer fails open: before `init` runs, or when SQLite errors,
+ * reads behave as misses and writes are dropped, degrading to
+ * in-memory-only caching rather than surfacing an error into the sweep.
  */
 
 import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
 
-const MAX_ENTRIES = 500;
+const MAX_MEMORY_ENTRIES = 500;
+const MAX_DB_ROWS = 2000;
 
+/** In-memory LRU, keyed by image hash (conversation-agnostic). */
 const cache = new Map<string, string>();
 
+/** Plugin-owned SQLite handle; `null` until `init` opens it (fail-open). */
+let db: Database | null = null;
+
 /** sha-256 hex digest of an image's base64 payload. */
 export function imageHash(data: string): string {
   return createHash("sha256").update(data).digest("hex");
 }
 
-/** Look up a cached caption. `undefined` = miss; a string (even empty) = hit. */
-export function getCachedCaption(hash: string): string | undefined {
+/**
+ * Open (or create) the plugin's caption database inside its storage dir and
+ * ensure the schema exists. Called by the plugin's `init` hook. Idempotent —
+ * re-running against an existing file is a no-op — and fail-open: on error
+ * the store stays memory-only.
+ */
+export function initCaptionStore(storageDir: string): void {
+  try {
+    closeCaptionStore();
+    const handle = new Database(join(storageDir, "caption-cache.sqlite"));
+    handle.exec("PRAGMA journal_mode=WAL");
+    handle.exec(/*sql*/ `
+      CREATE TABLE IF NOT EXISTS image_captions (
+        image_hash      TEXT NOT NULL,
+        conversation_id TEXT NOT NULL,
+        caption         TEXT NOT NULL,
+        created_at      INTEGER NOT NULL,
+        last_used_at    INTEGER NOT NULL,
+        PRIMARY KEY (image_hash, conversation_id)
+      )
+    `);
+    handle.exec(
+      /*sql*/ `CREATE INDEX IF NOT EXISTS idx_image_captions_last_used ON image_captions(last_used_at)`,
+    );
+    handle.exec(
+      /*sql*/ `CREATE INDEX IF NOT EXISTS idx_image_captions_conversation ON image_captions(conversation_id)`,
+    );
+    db = handle;
+  } catch {
+    db = null;
+  }
+}
+
+/** Close the caption database. Called by the plugin's `shutdown` hook. */
+export function closeCaptionStore(): void {
+  try {
+    db?.close();
+  } catch {
+    // Already closed or unusable — either way the handle is discarded.
+  }
+  db = null;
+}
+
+/**
+ * Record that `conversationId` uses `hash`, refreshing recency and the
+ * caption text, and evict the least-recently-used rows beyond the cap.
+ */
+function dbRecordUse(
+  hash: string,
+  conversationId: string,
+  caption: string,
+): void {
+  if (db == null) {
+    return;
+  }
+  try {
+    const now = Date.now();
+    db.query(
+      /*sql*/ `
+      INSERT INTO image_captions (image_hash, conversation_id, caption, created_at, last_used_at)
+      VALUES (?, ?, ?, ?, ?)
+      ON CONFLICT(image_hash, conversation_id) DO UPDATE SET
+        caption = excluded.caption,
+        last_used_at = excluded.last_used_at
+    `,
+    ).run(hash, conversationId, caption, now, now);
+    db.query(
+      /*sql*/ `
+      DELETE FROM image_captions WHERE (image_hash, conversation_id) IN (
+        SELECT image_hash, conversation_id FROM image_captions
+        ORDER BY last_used_at DESC LIMIT -1 OFFSET ?
+      )
+    `,
+    ).run(MAX_DB_ROWS);
+  } catch {
+    // Fail open: the caption still lives in the in-memory layer.
+  }
+}
+
+function dbLookupCaption(hash: string): string | undefined {
+  if (db == null) {
+    return undefined;
+  }
+  try {
+    const row = db
+      .query(
+        /*sql*/ `SELECT caption FROM image_captions WHERE image_hash = ? LIMIT 1`,
+      )
+      .get(hash) as { caption: string } | null;
+    return row?.caption;
+  } catch {
+    // Fail open: an unavailable DB behaves as a miss.
+    return undefined;
+  }
+}
+
+function setMemoryCaption(hash: string, caption: string): void {
+  if (cache.size >= MAX_MEMORY_ENTRIES && !cache.has(hash)) {
+    const oldest = cache.keys().next();
+    if (!oldest.done) {
+      cache.delete(oldest.value as string);
+    }
+  }
+  cache.set(hash, caption);
+}
+
+/**
+ * Look up a cached caption for an image used by `conversationId`.
+ * `undefined` = miss; a string (even empty) = hit. A hit from either layer
+ * also records the (hash, conversation) association so `conversation-deleted`
+ * cleanup stays accurate.
+ */
+export function getCachedCaption(
+  hash: string,
+  conversationId: string,
+): string | undefined {
   const value = cache.get(hash);
   if (value !== undefined) {
     // Move to end (most-recently-used) for LRU eviction.
     cache.delete(hash);
     cache.set(hash, value);
+    dbRecordUse(hash, conversationId, value);
+    return value;
   }
-  return value;
+  const persisted = dbLookupCaption(hash);
+  if (persisted !== undefined) {
+    setMemoryCaption(hash, persisted);
+    dbRecordUse(hash, conversationId, persisted);
+  }
+  return persisted;
 }
 
-/** Store a caption, evicting the least-recently-used entry if at capacity. */
-export function setCachedCaption(hash: string, caption: string): void {
-  if (cache.size >= MAX_ENTRIES) {
-    const oldest = cache.keys().next();
-    if (!oldest.done) {
-      cache.delete(oldest.value as string);
+/** Store a caption in both the in-memory layer and the durable table. */
+export function setCachedCaption(
+  hash: string,
+  conversationId: string,
+  caption: string,
+): void {
+  setMemoryCaption(hash, caption);
+  dbRecordUse(hash, conversationId, caption);
+}
+
+/**
+ * Remove a deleted conversation's caption rows. Hashes no other conversation
+ * references are also dropped from the in-memory layer, so the derived
+ * caption text does not outlive the last conversation that contained the
+ * image.
+ */
+export function deleteConversationCaptions(conversationId: string): number {
+  if (db == null) {
+    return 0;
+  }
+  try {
+    const rows = db
+      .query(
+        /*sql*/ `SELECT image_hash FROM image_captions WHERE conversation_id = ?`,
+      )
+      .all(conversationId) as Array<{ image_hash: string }>;
+    if (rows.length === 0) {
+      return 0;
+    }
+    db.query(
+      /*sql*/ `DELETE FROM image_captions WHERE conversation_id = ?`,
+    ).run(conversationId);
+    for (const { image_hash } of rows) {
+      const remaining = db
+        .query(
+          /*sql*/ `SELECT 1 FROM image_captions WHERE image_hash = ? LIMIT 1`,
+        )
+        .get(image_hash);
+      if (remaining == null) {
+        cache.delete(image_hash);
+      }
     }
+    return rows.length;
+  } catch {
+    // Fail open: cleanup is best-effort; rows age out via LRU eviction.
+    return 0;
   }
-  cache.set(hash, caption);
 }
 
-/** Test-only: clear the cache. */
+/** Test-only: clear the in-memory layer and best-effort clear the table. */
 export function resetCaptionCacheForTests(): void {
   cache.clear();
+  try {
+    db?.exec(/*sql*/ `DELETE FROM image_captions`);
+  } catch {
+    // Store not initialized in DB-less test environments — nothing to clear.
+  }
 }