ai-sdk-provider-claude-code 1.1.3 → 1.1.4

package/README.md

@@ -121,10 +121,19 @@ Key changes:
 - 🔧 Tool management (MCP servers, permissions)
  - 🧩 Callbacks (hooks, canUseTool)
 
+## Image Inputs (Streaming Only)
+
+- Enable streaming input (`streamingInput: 'always'` or provide `canUseTool`) before sending images.
+- Supported payloads: data URLs (`data:image/png;base64,...`), strings prefixed with `base64:<mediaType>,<data>`, or objects `{ data: '<base64>', mimeType: 'image/png' }`.
+- Remote HTTP(S) image URLs are ignored with the warning "Image URLs are not supported by this provider; supply base64/data URLs." (`supportsImageUrls` remains `false`).
+- When streaming input is disabled, image parts trigger the streaming prerequisite warning and are omitted from the request.
+- Use realistic image payloads—very small placeholders may result in the model asking for a different image.
+- `examples/images.ts` accepts a local image path and converts it to a data URL on the fly: `npx tsx examples/images.ts /absolute/path/to/image.png`.
+
 ## Limitations
 
 - Requires Node.js ≥ 18
-- No image support
+- Image inputs require streaming mode with base64/data URLs (remote fetch is not supported)
 - Some AI SDK parameters unsupported (temperature, maxTokens, etc.)
 - `canUseTool` requires streaming input at the SDK level (AsyncIterable prompt). This provider supports it via `streamingInput`: use `'auto'` (default when `canUseTool` is set) or `'always'`. See GUIDE for details.
 

package/dist/index.cjs

@@ -43,26 +43,176 @@ var import_provider2 = require("@ai-sdk/provider");
 var import_provider_utils = require("@ai-sdk/provider-utils");
 
 // src/convert-to-claude-code-messages.ts
+var IMAGE_URL_WARNING = "Image URLs are not supported by this provider; supply base64/data URLs.";
+var IMAGE_CONVERSION_WARNING = "Unable to convert image content; supply base64/data URLs.";
+function normalizeBase64(base64) {
+  return base64.replace(/\s+/g, "");
+}
+function isImageMimeType(mimeType) {
+  return typeof mimeType === "string" && mimeType.trim().toLowerCase().startsWith("image/");
+}
+function createImageContent(mediaType, data) {
+  const trimmedType = mediaType.trim();
+  const trimmedData = normalizeBase64(data.trim());
+  if (!trimmedType || !trimmedData) {
+    return void 0;
+  }
+  return {
+    type: "image",
+    source: {
+      type: "base64",
+      media_type: trimmedType,
+      data: trimmedData
+    }
+  };
+}
+function extractMimeType(candidate) {
+  if (typeof candidate === "string" && candidate.trim()) {
+    return candidate.trim();
+  }
+  return void 0;
+}
+function parseObjectImage(imageObj, fallbackMimeType) {
+  const data = typeof imageObj.data === "string" ? imageObj.data : void 0;
+  const mimeType = extractMimeType(
+    imageObj.mimeType ?? imageObj.mediaType ?? imageObj.media_type ?? fallbackMimeType
+  );
+  if (!data || !mimeType) {
+    return void 0;
+  }
+  return createImageContent(mimeType, data);
+}
+function parseStringImage(value, fallbackMimeType) {
+  const trimmed = value.trim();
+  if (/^https?:\/\//i.test(trimmed)) {
+    return { warning: IMAGE_URL_WARNING };
+  }
+  const dataUrlMatch = trimmed.match(/^data:([^;]+);base64,(.+)$/i);
+  if (dataUrlMatch) {
+    const [, mediaType, data] = dataUrlMatch;
+    const content = createImageContent(mediaType, data);
+    return content ? { content } : { warning: IMAGE_CONVERSION_WARNING };
+  }
+  const base64Match = trimmed.match(/^base64:([^,]+),(.+)$/i);
+  if (base64Match) {
+    const [, explicitMimeType, data] = base64Match;
+    const content = createImageContent(explicitMimeType, data);
+    return content ? { content } : { warning: IMAGE_CONVERSION_WARNING };
+  }
+  if (fallbackMimeType) {
+    const content = createImageContent(fallbackMimeType, trimmed);
+    if (content) {
+      return { content };
+    }
+  }
+  return { warning: IMAGE_CONVERSION_WARNING };
+}
+function parseImagePart(part) {
+  if (!part || typeof part !== "object") {
+    return { warning: IMAGE_CONVERSION_WARNING };
+  }
+  const imageValue = part.image;
+  const mimeType = extractMimeType(part.mimeType);
+  if (typeof imageValue === "string") {
+    return parseStringImage(imageValue, mimeType);
+  }
+  if (imageValue && typeof imageValue === "object") {
+    const content = parseObjectImage(imageValue, mimeType);
+    return content ? { content } : { warning: IMAGE_CONVERSION_WARNING };
+  }
+  return { warning: IMAGE_CONVERSION_WARNING };
+}
+function convertBinaryToBase64(data) {
+  if (typeof Buffer !== "undefined") {
+    const buffer = data instanceof Uint8Array ? Buffer.from(data) : Buffer.from(new Uint8Array(data));
+    return buffer.toString("base64");
+  }
+  if (typeof btoa === "function") {
+    const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+    let binary = "";
+    const chunkSize = 32768;
+    for (let i = 0; i < bytes.length; i += chunkSize) {
+      const chunk = bytes.subarray(i, i + chunkSize);
+      binary += String.fromCharCode(...chunk);
+    }
+    return btoa(binary);
+  }
+  return void 0;
+}
+function parseFilePart(part) {
+  const mimeType = extractMimeType(part.mediaType ?? part.mimeType);
+  if (!mimeType || !isImageMimeType(mimeType)) {
+    return {};
+  }
+  const data = part.data;
+  if (typeof data === "string") {
+    const content = createImageContent(mimeType, data);
+    return content ? { content } : { warning: IMAGE_CONVERSION_WARNING };
+  }
+  if (data instanceof Uint8Array || typeof ArrayBuffer !== "undefined" && data instanceof ArrayBuffer) {
+    const base64 = convertBinaryToBase64(data);
+    if (!base64) {
+      return { warning: IMAGE_CONVERSION_WARNING };
+    }
+    const content = createImageContent(mimeType, base64);
+    return content ? { content } : { warning: IMAGE_CONVERSION_WARNING };
+  }
+  return { warning: IMAGE_CONVERSION_WARNING };
+}
 function convertToClaudeCodeMessages(prompt, mode = { type: "regular" }, jsonSchema) {
   const messages = [];
   const warnings = [];
   let systemPrompt;
+  const streamingSegments = [];
+  const imageMap = /* @__PURE__ */ new Map();
+  let hasImageParts = false;
+  const addSegment = (formatted) => {
+    streamingSegments.push({ formatted });
+    return streamingSegments.length - 1;
+  };
+  const addImageForSegment = (segmentIndex, content) => {
+    hasImageParts = true;
+    if (!imageMap.has(segmentIndex)) {
+      imageMap.set(segmentIndex, []);
+    }
+    imageMap.get(segmentIndex)?.push(content);
+  };
   for (const message of prompt) {
     switch (message.role) {
       case "system":
         systemPrompt = message.content;
+        if (typeof message.content === "string" && message.content.trim().length > 0) {
+          addSegment(message.content);
+        } else {
+          addSegment("");
+        }
         break;
       case "user":
         if (typeof message.content === "string") {
           messages.push(message.content);
+          addSegment(`Human: ${message.content}`);
         } else {
           const textParts = message.content.filter((part) => part.type === "text").map((part) => part.text).join("\n");
+          const segmentIndex = addSegment(textParts ? `Human: ${textParts}` : "");
           if (textParts) {
             messages.push(textParts);
           }
-          const imageParts = message.content.filter((part) => part.type === "image");
-          if (imageParts.length > 0) {
-            warnings.push("Claude Code SDK does not support image inputs. Images will be ignored.");
+          for (const part of message.content) {
+            if (part.type === "image") {
+              const { content, warning } = parseImagePart(part);
+              if (content) {
+                addImageForSegment(segmentIndex, content);
+              } else if (warning) {
+                warnings.push(warning);
+              }
+            } else if (part.type === "file") {
+              const { content, warning } = parseFilePart(part);
+              if (content) {
+                addImageForSegment(segmentIndex, content);
+              } else if (warning) {
+                warnings.push(warning);
+              }
+            }
           }
         }
         break;
@@ -81,13 +231,17 @@ function convertToClaudeCodeMessages(prompt, mode = { type: "regular" }, jsonSch
 [Tool calls made]`;
           }
         }
-        messages.push(`Assistant: ${assistantContent}`);
+        const formattedAssistant = `Assistant: ${assistantContent}`;
+        messages.push(formattedAssistant);
+        addSegment(formattedAssistant);
         break;
       }
       case "tool":
         for (const tool3 of message.content) {
           const resultText = tool3.output.type === "text" ? tool3.output.value : JSON.stringify(tool3.output.value);
-          messages.push(`Tool Result (${tool3.toolName}): ${resultText}`);
+          const formattedToolResult = `Tool Result (${tool3.toolName}): ${resultText}`;
+          messages.push(formattedToolResult);
+          addSegment(formattedToolResult);
         }
         break;
     }
@@ -96,22 +250,67 @@ function convertToClaudeCodeMessages(prompt, mode = { type: "regular" }, jsonSch
   if (systemPrompt) {
     finalPrompt = systemPrompt;
   }
-  if (messages.length === 0) {
-    return { messagesPrompt: finalPrompt, systemPrompt };
-  }
-  const formattedMessages = [];
-  for (let i = 0; i < messages.length; i++) {
-    const msg = messages[i];
-    if (msg.startsWith("Assistant:") || msg.startsWith("Tool Result")) {
-      formattedMessages.push(msg);
+  if (messages.length > 0) {
+    const formattedMessages = [];
+    for (let i = 0; i < messages.length; i++) {
+      const msg = messages[i];
+      if (msg.startsWith("Assistant:") || msg.startsWith("Tool Result")) {
+        formattedMessages.push(msg);
+      } else {
+        formattedMessages.push(`Human: ${msg}`);
+      }
+    }
+    if (finalPrompt) {
+      const joinedMessages = formattedMessages.join("\n\n");
+      finalPrompt = joinedMessages ? `${finalPrompt}
+
+${joinedMessages}` : finalPrompt;
     } else {
-      formattedMessages.push(`Human: ${msg}`);
+      finalPrompt = formattedMessages.join("\n\n");
     }
   }
-  if (finalPrompt) {
-    finalPrompt = finalPrompt + "\n\n" + formattedMessages.join("\n\n");
-  } else {
-    finalPrompt = formattedMessages.join("\n\n");
+  let streamingParts = [];
+  const imagePartsInOrder = [];
+  const appendImagesForIndex = (index) => {
+    const images = imageMap.get(index);
+    if (!images) {
+      return;
+    }
+    images.forEach((image) => {
+      streamingParts.push(image);
+      imagePartsInOrder.push(image);
+    });
+  };
+  if (streamingSegments.length > 0) {
+    let accumulatedText = "";
+    let emittedText = false;
+    const flushText = () => {
+      if (!accumulatedText) {
+        return;
+      }
+      streamingParts.push({ type: "text", text: accumulatedText });
+      accumulatedText = "";
+      emittedText = true;
+    };
+    streamingSegments.forEach((segment, index) => {
+      const segmentText = segment.formatted;
+      if (segmentText) {
+        if (!accumulatedText) {
+          accumulatedText = emittedText ? `
+
+${segmentText}` : segmentText;
+        } else {
+          accumulatedText += `
+
+${segmentText}`;
+        }
+      }
+      if (imageMap.has(index)) {
+        flushText();
+        appendImagesForIndex(index);
+      }
+    });
+    flushText();
   }
   if (mode?.type === "object-json" && jsonSchema) {
     const schemaStr = JSON.stringify(jsonSchema, null, 2);
@@ -127,11 +326,20 @@ Now, based on the following conversation, generate ONLY the JSON object with the
 ${finalPrompt}
 
 Remember: Your ENTIRE response must be ONLY the JSON object, starting with { and ending with }`;
+    streamingParts = [
+      { type: "text", text: finalPrompt },
+      ...imagePartsInOrder
+    ];
   }
   return {
     messagesPrompt: finalPrompt,
     systemPrompt,
-    ...warnings.length > 0 && { warnings }
+    ...warnings.length > 0 && { warnings },
+    streamingContentParts: streamingParts.length > 0 ? streamingParts : [
+      { type: "text", text: finalPrompt },
+      ...imagePartsInOrder
+    ],
+    hasImageParts
   };
 }
 
@@ -474,12 +682,14 @@ function isAbortError(err) {
   }
   return false;
 }
-function toAsyncIterablePrompt(messagesPrompt, outputStreamEnded, sessionId) {
+var STREAMING_FEATURE_WARNING = "Claude Code SDK features (hooks/MCP/images) require streaming input. Set `streamingInput: 'always'` or provide `canUseTool` (auto streams only when canUseTool is set).";
+function toAsyncIterablePrompt(messagesPrompt, outputStreamEnded, sessionId, contentParts) {
+  const content = contentParts && contentParts.length > 0 ? contentParts : [{ type: "text", text: messagesPrompt }];
   const msg = {
     type: "user",
     message: {
       role: "user",
-      content: [{ type: "text", text: messagesPrompt }]
+      content
     },
     parent_tool_use_id: null,
     session_id: sessionId ?? ""
@@ -673,7 +883,12 @@ var ClaudeCodeLanguageModel = class {
   }
   async doGenerate(options) {
     const mode = options.responseFormat?.type === "json" ? { type: "object-json" } : { type: "regular" };
-    const { messagesPrompt, warnings: messageWarnings } = convertToClaudeCodeMessages(
+    const {
+      messagesPrompt,
+      warnings: messageWarnings,
+      streamingContentParts,
+      hasImageParts
+    } = convertToClaudeCodeMessages(
       options.prompt,
       mode,
       options.responseFormat?.type === "json" ? options.responseFormat.schema : void 0
@@ -702,18 +917,29 @@ var ClaudeCodeLanguageModel = class {
         });
       });
     }
+    const modeSetting = this.settings.streamingInput ?? "auto";
+    const wantsStreamInput = modeSetting === "always" || modeSetting === "auto" && !!this.settings.canUseTool;
+    if (!wantsStreamInput && hasImageParts) {
+      warnings.push({
+        type: "other",
+        message: STREAMING_FEATURE_WARNING
+      });
+    }
     let done = () => {
     };
     const outputStreamEnded = new Promise((resolve) => {
       done = () => resolve(void 0);
     });
     try {
-      const modeSetting = this.settings.streamingInput ?? "auto";
-      const wantsStream = modeSetting === "always" || modeSetting === "auto" && !!this.settings.canUseTool;
       if (this.settings.canUseTool && this.settings.permissionPromptToolName) {
         throw new Error("canUseTool requires streamingInput mode ('auto' or 'always') and cannot be used with permissionPromptToolName (SDK constraint). Set streamingInput: 'auto' (or 'always') and remove permissionPromptToolName, or remove canUseTool.");
       }
-      const sdkPrompt = wantsStream ? toAsyncIterablePrompt(messagesPrompt, outputStreamEnded, this.settings.resume ?? this.sessionId) : messagesPrompt;
+      const sdkPrompt = wantsStreamInput ? toAsyncIterablePrompt(
+        messagesPrompt,
+        outputStreamEnded,
+        this.settings.resume ?? this.sessionId,
+        streamingContentParts
+      ) : messagesPrompt;
       const response = (0, import_claude_code.query)({
         prompt: sdkPrompt,
         options: queryOptions
@@ -785,7 +1011,12 @@ var ClaudeCodeLanguageModel = class {
   }
   async doStream(options) {
     const mode = options.responseFormat?.type === "json" ? { type: "object-json" } : { type: "regular" };
-    const { messagesPrompt, warnings: messageWarnings } = convertToClaudeCodeMessages(
+    const {
+      messagesPrompt,
+      warnings: messageWarnings,
+      streamingContentParts,
+      hasImageParts
+    } = convertToClaudeCodeMessages(
       options.prompt,
       mode,
       options.responseFormat?.type === "json" ? options.responseFormat.schema : void 0
@@ -808,6 +1039,14 @@ var ClaudeCodeLanguageModel = class {
         });
       });
     }
+    const modeSetting = this.settings.streamingInput ?? "auto";
+    const wantsStreamInput = modeSetting === "always" || modeSetting === "auto" && !!this.settings.canUseTool;
+    if (!wantsStreamInput && hasImageParts) {
+      warnings.push({
+        type: "other",
+        message: STREAMING_FEATURE_WARNING
+      });
+    }
     const stream = new ReadableStream({
       start: async (controller) => {
         let done = () => {
@@ -817,12 +1056,15 @@ var ClaudeCodeLanguageModel = class {
         });
         try {
           controller.enqueue({ type: "stream-start", warnings });
-          const modeSetting = this.settings.streamingInput ?? "auto";
-          const wantsStream = modeSetting === "always" || modeSetting === "auto" && !!this.settings.canUseTool;
           if (this.settings.canUseTool && this.settings.permissionPromptToolName) {
             throw new Error("canUseTool requires streamingInput mode ('auto' or 'always') and cannot be used with permissionPromptToolName (SDK constraint). Set streamingInput: 'auto' (or 'always') and remove permissionPromptToolName, or remove canUseTool.");
           }
-          const sdkPrompt = wantsStream ? toAsyncIterablePrompt(messagesPrompt, outputStreamEnded, this.settings.resume ?? this.sessionId) : messagesPrompt;
+          const sdkPrompt = wantsStreamInput ? toAsyncIterablePrompt(
+            messagesPrompt,
+            outputStreamEnded,
+            this.settings.resume ?? this.sessionId,
+            streamingContentParts
+          ) : messagesPrompt;
           const response = (0, import_claude_code.query)({
             prompt: sdkPrompt,
             options: queryOptions