@vellumai/assistant 0.10.10-dev.202607171828.fc2f5d5 → 0.10.10-dev.202607172024.ca75f7f

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.10.10-dev.202607171828.fc2f5d5",
+  "version": "0.10.10-dev.202607172024.ca75f7f",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/attachment-upload-trusted-source.test.ts

@@ -23,6 +23,13 @@ import type { RouteHandlerArgs } from "../runtime/routes/types.js";
 const SMALL_PNG_BASE64 =
   "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
 
+// Non-image bytes: storage normalization sniffs image magic bytes and
+// overrides a disagreeing declared MIME, so a declared-MIME assertion needs a
+// payload that doesn't sniff as an image.
+const FAKE_VIDEO_BASE64 = Buffer.from("fake matroska payload").toString(
+  "base64",
+);
+
 const uploadRoute = ROUTES.find((r) => r.operationId === "attachment_upload")!;
 
 function makeUploadArgs(
@@ -59,7 +66,7 @@ describe("attachment upload — trustedSource flag", () => {
         {
           filename: "clip.mkv",
           mimeType: "video/x-matroska",
-          data: SMALL_PNG_BASE64,
+          data: FAKE_VIDEO_BASE64,
           trustedSource: true,
         },
         "svc_gateway",

package/src/__tests__/attachments-store-heic-normalize.test.ts

@@ -31,7 +31,10 @@ mock.module("../util/image-conversion.js", () => ({
   normalizeImageBase64: (mimeType: string, dataBase64: string) =>
     mimeType === "image/heic"
       ? { mimeType: "image/jpeg", dataBase64: FAKE_JPEG_B64, converted: true }
-      : { mimeType, dataBase64, converted: false },
+      : mimeType === "image/mislabeled"
+        ? // Sniff-corrected declared MIME: payload untouched, converted false.
+          { mimeType: "image/png", dataBase64, converted: false }
+        : { mimeType, dataBase64, converted: false },
 }));
 
 import {
@@ -131,6 +134,16 @@ describe("HEIC upload normalization wiring", () => {
     expect(stored.sizeBytes).toBe(HEIC_BYTES.length);
   });
 
+  test("uploadAttachment (base64) propagates a MIME-only sniff correction", () => {
+    const stored = uploadAttachment("photo.png", "image/mislabeled", HEIC_B64);
+
+    // Corrected MIME is stored; filename and payload stay verbatim.
+    expect(stored.originalFilename).toBe("photo.png");
+    expect(stored.mimeType).toBe("image/png");
+    const row = getAttachmentById(stored.id);
+    expect(row?.dataBase64).toBe(HEIC_B64);
+  });
+
   test("attachInlineAttachmentToMessage normalizes only when opted in", async () => {
     const conv = createConversation();
     const msg = await addMessage(

package/src/__tests__/conversation-error.test.ts

@@ -379,6 +379,24 @@ describe("classifyConversationError", () => {
       expect(result.retryable).toBe(false);
     });
 
+    it("classifies Anthropic 400 media-type mismatch as image_media_type_mismatch (non-retryable)", () => {
+      // The rejection for an image whose declared media type disagrees with
+      // its bytes (e.g. a JPEG renamed to .png before upload). It must route
+      // to the image-recovery classification so the relabel path fires, not
+      // loop through the generic PROVIDER_API branch.
+      const err = new ProviderError(
+        'Anthropic API error (400): 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.50.content.0.image.source.base64.data: Image does not match the provided media type image/png"},"request_id":"req_011Ccs00000000000000000"}',
+        "anthropic",
+        400,
+      );
+      const result = classifyConversationError(err, baseCtx);
+      expect(result.code).toBe("IMAGE_TOO_LARGE");
+      expect(result.errorCategory).toBe("image_media_type_mismatch");
+      expect(result.retryable).toBe(false);
+      expect(result.userMessage).not.toContain("invalid_request_error");
+      expect(result.userMessage.toLowerCase()).toContain("image");
+    });
+
     it("classifies Anthropic 400 'Could not process image' as image_unprocessable (non-retryable)", () => {
       // The rejection Anthropic returns for images below its minimum size
       // floor (e.g. a 16×14 px upload). It must route to the image-recovery

package/src/__tests__/image-conversion.test.ts

@@ -14,6 +14,8 @@ import {
   jpegFilenameFor,
   normalizeImageBase64,
   normalizeImageBytes,
+  sniffBase64ImageMimeType,
+  sniffImageMimeType,
 } from "../util/image-conversion.js";
 import {
   fakeHeifHeaderBytes,
@@ -69,6 +71,48 @@ describe("jpegFilenameFor", () => {
   });
 });
 
+const JPEG_HEADER_BYTES = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
+const GIF_HEADER_BYTES = Buffer.from("GIF89a\x01\x00\x01\x00", "latin1");
+const WEBP_HEADER_BYTES = Buffer.concat([
+  Buffer.from("RIFF"),
+  Buffer.from([0x24, 0x00, 0x00, 0x00]),
+  Buffer.from("WEBPVP8 "),
+]);
+
+describe("sniffImageMimeType", () => {
+  test("identifies PNG, JPEG, GIF, and WebP signatures", () => {
+    expect(sniffImageMimeType(PNG_1PX_BYTES)).toBe("image/png");
+    expect(sniffImageMimeType(JPEG_HEADER_BYTES)).toBe("image/jpeg");
+    expect(sniffImageMimeType(GIF_HEADER_BYTES)).toBe("image/gif");
+    expect(sniffImageMimeType(WEBP_HEADER_BYTES)).toBe("image/webp");
+  });
+
+  test("returns null for unrecognized or truncated content", () => {
+    expect(sniffImageMimeType(Buffer.from("plain text content"))).toBeNull();
+    expect(sniffImageMimeType(Buffer.alloc(0))).toBeNull();
+    expect(sniffImageMimeType(fakeHeifHeaderBytes())).toBeNull();
+    // RIFF container that is not WebP (e.g. WAV audio).
+    expect(
+      sniffImageMimeType(
+        Buffer.concat([
+          Buffer.from("RIFF"),
+          Buffer.from([0x24, 0x00, 0x00, 0x00]),
+          Buffer.from("WAVEfmt "),
+        ]),
+      ),
+    ).toBeNull();
+  });
+
+  test("base64 variant sniffs from the encoded head", () => {
+    expect(sniffBase64ImageMimeType(PNG_1PX_BYTES.toString("base64"))).toBe(
+      "image/png",
+    );
+    expect(
+      sniffBase64ImageMimeType(Buffer.from("not an image").toString("base64")),
+    ).toBeNull();
+  });
+});
+
 describe("normalizeImageBytes passthrough", () => {
   test("non-HEIF bytes pass through untouched", () => {
     const result = normalizeImageBytes("image/png", PNG_1PX_BYTES);
@@ -96,6 +140,32 @@ describe("normalizeImageBase64 passthrough", () => {
   });
 });
 
+describe("declared-MIME correction from sniffed bytes", () => {
+  test("normalizeImageBytes relabels a mislabeled image, bytes untouched", () => {
+    // A JPEG renamed to .png arrives declared as image/png; providers reject
+    // the mismatch, so the sniffed format wins.
+    const result = normalizeImageBytes("image/png", JPEG_HEADER_BYTES);
+    expect(result.converted).toBe(false);
+    expect(result.mimeType).toBe("image/jpeg");
+    expect(result.bytes).toBe(JPEG_HEADER_BYTES);
+  });
+
+  test("normalizeImageBase64 relabels a mislabeled image, payload untouched", () => {
+    const b64 = PNG_1PX_BYTES.toString("base64");
+    const result = normalizeImageBase64("image/jpeg", b64);
+    expect(result.converted).toBe(false);
+    expect(result.mimeType).toBe("image/png");
+    expect(result.dataBase64).toBe(b64);
+  });
+
+  test("unrecognized bytes keep the declared MIME", () => {
+    const bytes = Buffer.from("plain text content");
+    const result = normalizeImageBytes("text/plain", bytes);
+    expect(result.mimeType).toBe("text/plain");
+    expect(result.bytes).toBe(bytes);
+  });
+});
+
 describe.skipIf(process.platform !== "darwin")(
   "real HEIC conversion (sips)",
   () => {

package/src/__tests__/image-recovery-hook.test.ts

@@ -241,6 +241,89 @@ describe("image-recovery post-model-call hook — direct", () => {
     });
   });
 
+  test("media-type-mismatch rejection → relabels the mislabeled image and continues", async () => {
+    // GIVEN a stored history holding a normally-sized PNG declared as
+    // image/jpeg (a renamed file — clients derive the MIME from the
+    // extension), and the provider rejection that mismatch produces.
+    const pngData = makePngBase64(1024, 768);
+    const mislabeled: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/jpeg", data: pngData },
+    };
+    const conv = createConversation();
+    await addMessage(conv.id, "user", JSON.stringify([mislabeled]), {
+      skipIndexing: true,
+    });
+    const ctx = makePostModelCallCtx({
+      conversationId: conv.id,
+      messages: [{ role: "user", content: [structuredClone(mislabeled)] }],
+      error: new Error(
+        'Anthropic API error (400): 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.0.content.0.image.source.base64.data: Image does not match the provided media type image/jpeg"}}',
+      ),
+    });
+
+    // WHEN the hook runs.
+    await postModelCall(ctx);
+
+    // THEN it retries with the image relabeled to its sniffed type — bytes
+    // kept — both in the working history and in the stored row, so the
+    // mismatch cannot re-reject on this or any later turn.
+    expect(ctx.decision).toBe("continue");
+    const workingImage = ctx.messages[0].content[0] as Extract<
+      ContentBlock,
+      { type: "image" }
+    >;
+    expect(workingImage.source).toMatchObject({
+      media_type: "image/png",
+      data: pngData,
+    });
+    const storedImage = getMessages(conv.id)[0].content[0] as Extract<
+      ContentBlock,
+      { type: "image" }
+    >;
+    expect(storedImage.source).toMatchObject({
+      media_type: "image/png",
+      data: pngData,
+    });
+  });
+
+  test("media-type-mismatch on an unrecognized format → no retry, error surfaces", async () => {
+    // GIVEN a mismatch rejection for an image whose actual format the sniffer
+    // cannot identify (a renamed BMP: "BM" magic, not in the sniff set), so no
+    // relabel is possible and the image is within size limits.
+    const bmpData = Buffer.concat([
+      Buffer.from("BM"),
+      Buffer.alloc(64, 0),
+    ]).toString("base64");
+    const unrecognized: ContentBlock = {
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: bmpData },
+    };
+    const ctx = makePostModelCallCtx({
+      messages: [{ role: "user", content: [structuredClone(unrecognized)] }],
+      error: new Error(
+        'Anthropic API error (400): 400 {"type":"error","error":{"type":"invalid_request_error","message":"messages.0.content.0.image.source.base64.data: Image does not match the provided media type image/png"}}',
+      ),
+    });
+
+    // WHEN the hook runs.
+    await postModelCall(ctx);
+
+    // THEN it does not retry (nothing could be corrected) and leaves the
+    // history untouched, so the original error surfaces instead of a false
+    // "corrected — resend" loop.
+    expect(ctx.decision).toBe("stop");
+    expect(isImageRecoveryAttempted(ctx.conversationId)).toBe(false);
+    const image = ctx.messages[0].content[0] as Extract<
+      ContentBlock,
+      { type: "image" }
+    >;
+    expect(image.source).toMatchObject({
+      media_type: "image/png",
+      data: bmpData,
+    });
+  });
+
   test("durably persists the downgrade so the image cannot resurface", async () => {
     // GIVEN a conversation whose stored history holds an oversized image.
     const conv = createConversation();

package/src/__tests__/list-messages-attachments.test.ts

@@ -172,7 +172,7 @@ describe("handleListMessages attachments", () => {
       "user",
       JSON.stringify([{ type: "text", text: "here are files" }]),
     );
-    const imgStored = uploadAttachment("photo.jpg", "image/jpeg", IMAGE_BASE64);
+    const imgStored = uploadAttachment("photo.png", "image/png", IMAGE_BASE64);
     const docStored = uploadAttachment(
       "doc.pdf",
       "application/pdf",
@@ -187,7 +187,7 @@ describe("handleListMessages attachments", () => {
     const attachments = body.messages[0].attachments!;
     expect(attachments).toHaveLength(2);
 
-    const imgAtt = attachments.find((a) => a.mimeType === "image/jpeg");
+    const imgAtt = attachments.find((a) => a.mimeType === "image/png");
     const docAtt = attachments.find((a) => a.mimeType === "application/pdf");
     expect(imgAtt!.data).toBe(IMAGE_BASE64);
     expect(docAtt!.data).toBeUndefined();

package/src/__tests__/persist-unsendable-image.test.ts

@@ -14,6 +14,7 @@
 
 import { beforeEach, describe, expect, test } from "bun:test";
 
+import { uploadAttachment } from "../persistence/attachments-store.js";
 import {
   addMessage,
   createConversation,
@@ -83,10 +84,10 @@ function makePngBase64(width: number, height: number, padBytes = 0): string {
   return padBytes > 0 ? header + "A".repeat(padBytes) : header;
 }
 
-function imageBlock(data: string): ContentBlock {
+function imageBlock(data: string, mediaType = "image/png"): ContentBlock {
   return {
     type: "image",
-    source: { type: "base64", media_type: "image/png", data },
+    source: { type: "base64", media_type: mediaType, data },
   };
 }
 
@@ -286,6 +287,41 @@ describe("persistUnsendableImageDowngrades", () => {
     );
   });
 
+  /** A stored image whose declared media type disagrees with its bytes (e.g.
+   *  a JPEG renamed to .png before upload) is relabeled in place — bytes kept,
+   *  media_type corrected — so it stops re-rejecting on every later turn. */
+  test("relabels a mislabeled image instead of removing it", async () => {
+    // GIVEN a normally-sized PNG stored with a declared type of image/jpeg
+    const conv = createConversation();
+    const pngData = makePngBase64(1024, 768);
+    await addMessage(
+      conv.id,
+      "user",
+      JSON.stringify([imageBlock(pngData, "image/jpeg")]),
+      { skipIndexing: true },
+    );
+
+    // WHEN the downgrade is persisted
+    const rewritten = persistUnsendableImageDowngrades(conv.id);
+
+    // THEN the image survives with its media type corrected to the sniffed one
+    expect(rewritten).toBe(1);
+    const [content] = storedContent(conv.id);
+    const image = content.find((b) => b.type === "image") as Extract<
+      ContentBlock,
+      { type: "image" }
+    >;
+    expect(image).toBeDefined();
+    expect(image.source).toMatchObject({
+      type: "base64",
+      media_type: "image/png",
+      data: pngData,
+    });
+
+    // AND a second run is a no-op (the corrected block no longer mismatches)
+    expect(persistUnsendableImageDowngrades(conv.id)).toBe(0);
+  });
+
   /** Re-running after a rewrite is a safe no-op (no image blocks remain). */
   test("is idempotent — a second run rewrites nothing", async () => {
     // GIVEN a conversation whose oversized image has already been downgraded
@@ -340,4 +376,52 @@ describe("unsendableImageReplacement", () => {
     const replacement = unsendableImageReplacement(undersized);
     expect(replacement?.type).toBe("text");
   });
+
+  /** A within-limits image whose declared media type disagrees with its bytes
+   *  is relabeled with the sniffed type instead of being noted out. */
+  test("relabels a mislabeled image with its sniffed media type", () => {
+    const pngData = makePngBase64(1024, 768);
+    const mislabeled = imageBlock(pngData, "image/jpeg") as Extract<
+      ContentBlock,
+      { type: "image" }
+    >;
+    const replacement = unsendableImageReplacement(mislabeled);
+    expect(replacement).toMatchObject({
+      type: "image",
+      source: { type: "base64", media_type: "image/png", data: pngData },
+    });
+  });
+
+  /** Relabeling a persisted (workspace_ref) image keeps the reference shape —
+   *  inlining it would bake the full payload into the stored message row. */
+  test("relabels a mislabeled workspace_ref without inlining the payload", () => {
+    const pngData = makePngBase64(1024, 768);
+    const stored = uploadAttachment("photo.png", "image/png", pngData);
+    const mislabeledRef = {
+      type: "image",
+      source: {
+        type: "workspace_ref",
+        media_type: "image/jpeg",
+        attachmentId: stored.id,
+        sizeBytes: Buffer.from(pngData, "base64").length,
+      },
+    } as Extract<ContentBlock, { type: "image" }>;
+
+    const replacement = unsendableImageReplacement(mislabeledRef);
+
+    expect(replacement).toMatchObject({
+      type: "image",
+      source: {
+        type: "workspace_ref",
+        media_type: "image/png",
+        attachmentId: stored.id,
+      },
+    });
+    // AND the corrected reference no longer matches on a second pass
+    expect(
+      unsendableImageReplacement(
+        replacement as Extract<ContentBlock, { type: "image" }>,
+      ),
+    ).toBeNull();
+  });
 });

package/src/__tests__/plugin-import-boundary-guard.test.ts

@@ -75,6 +75,7 @@ const BASELINE: Record<string, readonly string[]> = {
     "../../../agent/image-optimize.js",
     "../../../context/image-dimensions.js",
     "../../../persistence/conversation-crud.js",
+    "../../../util/image-conversion.js",
     "../../../util/logger.js",
   ],
   memory: [

package/src/__tests__/subagent-spawn-and-await.test.ts

@@ -52,6 +52,8 @@ let nextConversationConfig: FakeConversationConfig = {};
 let runLoopInvoked = false;
 /** The first user message persisted by the most recent FakeConversation. */
 let lastPersistedUserMessage: string | undefined;
+/** Records `setSubagentDenySideEffects` on the most recent FakeConversation. */
+let lastDenySideEffects: boolean | undefined;
 
 class FakeConversation {
   messages: Message[];
@@ -92,6 +94,9 @@ class FakeConversation {
   setAssistantId() {}
   setEnabledPlugins() {}
   setSubagentAllowedTools() {}
+  setSubagentDenySideEffects(deny: boolean) {
+    lastDenySideEffects = deny;
+  }
   setPreactivatedSkillIds() {}
   getCurrentSystemPrompt() {
     return "system";
@@ -214,6 +219,33 @@ function registerFakeParent(parentConversationId: string): {
 }
 
 describe("SubagentManager.spawnAndAwait", () => {
+  // Reset outside the test body so TypeScript does not narrow the module var to
+  // `undefined` across the opaque spawnAndAwait call.
+  beforeEach(() => {
+    lastDenySideEffects = undefined;
+  });
+
+  test("wires denySideEffectTools onto the subagent conversation (read-only)", async () => {
+    nextConversationConfig = {};
+
+    const manager = new SubagentManager();
+    await manager.spawnAndAwait(
+      makeConfig({ denySideEffectTools: true }),
+      () => {},
+    );
+
+    expect(lastDenySideEffects).toBe(true);
+  });
+
+  test("leaves denySideEffectTools off by default", async () => {
+    nextConversationConfig = {};
+
+    const manager = new SubagentManager();
+    await manager.spawnAndAwait(makeConfig(), () => {});
+
+    expect(lastDenySideEffects).toBeUndefined();
+  });
+
   test("resolves to the child's final assistant text", async () => {
     nextConversationConfig = {
       messages: [