@ai-sdk/langchain 2.0.21 → 2.0.23

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @ai-sdk/langchain
 
+## 2.0.23
+
+### Patch Changes
+
+- 78683c1: fix(langchain): add multimodal support to convertUserContent
+
+## 2.0.22
+
+### Patch Changes
+
+- Updated dependencies [f748c46]
+  - ai@6.0.20
+
 ## 2.0.21
 
 ### Patch Changes

package/dist/index.js

@@ -78,22 +78,171 @@ function convertAssistantContent(content) {
     tool_calls: toolCalls.length > 0 ? toolCalls : void 0
   });
 }
+function getDefaultFilename(mediaType, prefix = "file") {
+  const ext = mediaType.split("/")[1] || "bin";
+  return `${prefix}.${ext}`;
+}
 function convertUserContent(content) {
+  var _a;
   if (typeof content === "string") {
     return new import_messages.HumanMessage({ content });
   }
-  const textParts = content.filter(
-    (part) => part.type === "text"
-  ).map((part) => part.text);
-  return new import_messages.HumanMessage({ content: textParts.join("") });
+  const contentBlocks = [];
+  for (const part of content) {
+    if (part.type === "text") {
+      contentBlocks.push({ type: "text", text: part.text });
+    } else if (part.type === "image") {
+      const imagePart = part;
+      if (imagePart.image instanceof URL) {
+        contentBlocks.push({
+          type: "image_url",
+          image_url: { url: imagePart.image.toString() }
+        });
+      } else if (typeof imagePart.image === "string") {
+        if (imagePart.image.startsWith("http://") || imagePart.image.startsWith("https://") || imagePart.image.startsWith("data:")) {
+          contentBlocks.push({
+            type: "image_url",
+            image_url: { url: imagePart.image }
+          });
+        } else {
+          const mimeType = imagePart.mediaType || "image/png";
+          contentBlocks.push({
+            type: "image_url",
+            image_url: { url: `data:${mimeType};base64,${imagePart.image}` }
+          });
+        }
+      } else if (
+        /**
+         * Handle Uint8Array or ArrayBuffer (binary data)
+         */
+        imagePart.image instanceof Uint8Array || imagePart.image instanceof ArrayBuffer
+      ) {
+        const bytes = imagePart.image instanceof ArrayBuffer ? new Uint8Array(imagePart.image) : imagePart.image;
+        const base64 = btoa(String.fromCharCode(...bytes));
+        const mimeType = imagePart.mediaType || "image/png";
+        contentBlocks.push({
+          type: "image_url",
+          image_url: { url: `data:${mimeType};base64,${base64}` }
+        });
+      }
+    } else if (part.type === "file") {
+      const filePart = part;
+      const isImage = (_a = filePart.mediaType) == null ? void 0 : _a.startsWith("image/");
+      if (isImage) {
+        if (filePart.data instanceof URL) {
+          contentBlocks.push({
+            type: "image_url",
+            image_url: { url: filePart.data.toString() }
+          });
+        } else if (typeof filePart.data === "string") {
+          if (filePart.data.startsWith("http://") || filePart.data.startsWith("https://") || filePart.data.startsWith("data:")) {
+            contentBlocks.push({
+              type: "image_url",
+              image_url: { url: filePart.data }
+            });
+          } else {
+            contentBlocks.push({
+              type: "image_url",
+              image_url: {
+                url: `data:${filePart.mediaType};base64,${filePart.data}`
+              }
+            });
+          }
+        } else if (filePart.data instanceof Uint8Array || filePart.data instanceof ArrayBuffer) {
+          const bytes = filePart.data instanceof ArrayBuffer ? new Uint8Array(filePart.data) : filePart.data;
+          const base64 = btoa(String.fromCharCode(...bytes));
+          contentBlocks.push({
+            type: "image_url",
+            image_url: { url: `data:${filePart.mediaType};base64,${base64}` }
+          });
+        }
+      } else {
+        const filename = filePart.filename || getDefaultFilename(filePart.mediaType, "file");
+        if (filePart.data instanceof URL) {
+          contentBlocks.push({
+            type: "file",
+            url: filePart.data.toString(),
+            mimeType: filePart.mediaType,
+            filename
+          });
+        } else if (typeof filePart.data === "string") {
+          if (filePart.data.startsWith("http://") || filePart.data.startsWith("https://")) {
+            contentBlocks.push({
+              type: "file",
+              url: filePart.data,
+              mimeType: filePart.mediaType,
+              filename
+            });
+          } else if (filePart.data.startsWith("data:")) {
+            const matches = filePart.data.match(/^data:([^;]+);base64,(.+)$/);
+            if (matches) {
+              contentBlocks.push({
+                type: "file",
+                data: matches[2],
+                mimeType: matches[1],
+                filename
+              });
+            } else {
+              contentBlocks.push({
+                type: "file",
+                url: filePart.data,
+                mimeType: filePart.mediaType,
+                filename
+              });
+            }
+          } else {
+            contentBlocks.push({
+              type: "file",
+              data: filePart.data,
+              mimeType: filePart.mediaType,
+              filename
+            });
+          }
+        } else if (filePart.data instanceof Uint8Array || filePart.data instanceof ArrayBuffer) {
+          const bytes = filePart.data instanceof ArrayBuffer ? new Uint8Array(filePart.data) : filePart.data;
+          const base64 = btoa(String.fromCharCode(...bytes));
+          contentBlocks.push({
+            type: "file",
+            data: base64,
+            mimeType: filePart.mediaType,
+            filename
+          });
+        }
+      }
+    }
+  }
+  if (contentBlocks.every((block) => block.type === "text")) {
+    return new import_messages.HumanMessage({
+      content: contentBlocks.map((block) => block.text).join("")
+    });
+  }
+  return new import_messages.HumanMessage({ content: contentBlocks });
 }
 function isToolResultPart(item) {
   return item != null && typeof item === "object" && "type" in item && item.type === "tool-result";
 }
 function processModelChunk(chunk, state, controller) {
+  if (!state.emittedImages) {
+    state.emittedImages = /* @__PURE__ */ new Set();
+  }
   if (chunk.id) {
     state.messageId = chunk.id;
   }
+  const chunkObj = chunk;
+  const additionalKwargs = chunkObj.additional_kwargs;
+  const imageOutputs = extractImageOutputs(additionalKwargs);
+  for (const imageOutput of imageOutputs) {
+    if (imageOutput.result && !state.emittedImages.has(imageOutput.id)) {
+      state.emittedImages.add(imageOutput.id);
+      const mediaType = `image/${imageOutput.output_format || "png"}`;
+      controller.enqueue({
+        type: "file",
+        mediaType,
+        url: `data:${mediaType};base64,${imageOutput.result}`
+      });
+      state.started = true;
+    }
+  }
   const reasoning = extractReasoningFromContentBlocks(chunk) || extractReasoningFromValuesMessage(chunk);
   if (reasoning) {
     if (!state.reasoningStarted) {