ai 3.4.6 → 3.4.7

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # ai
 
+## 3.4.7
+
+### Patch Changes
+
+- db04700: feat (core): support converting attachments to file parts
+- 988707c: feat (ai/core): automatically download files from urls
+
 ## 3.4.6
 
 ### Patch Changes

package/dist/index.js

@@ -900,6 +900,22 @@ var InvalidMessageRoleError = class extends import_provider5.AISDKError {
 };
 _a4 = symbol4;
 
+// core/prompt/split-data-url.ts
+function splitDataUrl(dataUrl) {
+  try {
+    const [header, base64Content] = dataUrl.split(",");
+    return {
+      mimeType: header.split(";")[0].split(":")[1],
+      base64Content
+    };
+  } catch (error) {
+    return {
+      mimeType: void 0,
+      base64Content: void 0
+    };
+  }
+}
+
 // core/prompt/convert-to-language-model-prompt.ts
 async function convertToLanguageModelPrompt({
   prompt,
@@ -910,7 +926,7 @@ async function convertToLanguageModelPrompt({
   if (prompt.system != null) {
     languageModelMessages.push({ role: "system", content: prompt.system });
   }
-  const downloadedImages = modelSupportsImageUrls || prompt.messages == null ? null : await downloadImages(prompt.messages, downloadImplementation);
+  const downloadedAssets = modelSupportsImageUrls || prompt.messages == null ? null : await downloadAssets(prompt.messages, downloadImplementation);
   const promptType = prompt.type;
   switch (promptType) {
     case "prompt": {
@@ -923,7 +939,7 @@ async function convertToLanguageModelPrompt({
     case "messages": {
       languageModelMessages.push(
         ...prompt.messages.map(
-          (message) => convertToLanguageModelMessage(message, downloadedImages)
+          (message) => convertToLanguageModelMessage(message, downloadedAssets)
         )
       );
       break;
@@ -935,7 +951,7 @@ async function convertToLanguageModelPrompt({
   }
   return languageModelMessages;
 }
-function convertToLanguageModelMessage(message, downloadedImages) {
+function convertToLanguageModelMessage(message, downloadedAssets) {
   const role = message.role;
   switch (role) {
     case "system": {
@@ -957,7 +973,7 @@ function convertToLanguageModelMessage(message, downloadedImages) {
         role: "user",
         content: message.content.map(
           (part) => {
-            var _a11, _b, _c;
+            var _a11, _b, _c, _d, _e;
             switch (part.type) {
               case "text": {
                 return {
@@ -968,7 +984,7 @@ function convertToLanguageModelMessage(message, downloadedImages) {
               }
               case "image": {
                 if (part.image instanceof URL) {
-                  if (downloadedImages == null) {
+                  if (downloadedAssets == null) {
                     return {
                       type: "image",
                       image: part.image,
@@ -976,7 +992,7 @@ function convertToLanguageModelMessage(message, downloadedImages) {
                       providerMetadata: part.experimental_providerMetadata
                     };
                   } else {
-                    const downloadedImage = downloadedImages[part.image.toString()];
+                    const downloadedImage = downloadedAssets[part.image.toString()];
                     return {
                       type: "image",
                       image: downloadedImage.data,
@@ -991,7 +1007,7 @@ function convertToLanguageModelMessage(message, downloadedImages) {
                     switch (url.protocol) {
                       case "http:":
                       case "https:": {
-                        if (downloadedImages == null) {
+                        if (downloadedAssets == null) {
                           return {
                             type: "image",
                             image: url,
@@ -999,7 +1015,7 @@ function convertToLanguageModelMessage(message, downloadedImages) {
                             providerMetadata: part.experimental_providerMetadata
                           };
                         } else {
-                          const downloadedImage = downloadedImages[part.image];
+                          const downloadedImage = downloadedAssets[url.toString()];
                           return {
                             type: "image",
                             image: downloadedImage.data,
@@ -1010,8 +1026,9 @@ function convertToLanguageModelMessage(message, downloadedImages) {
                       }
                       case "data:": {
                         try {
-                          const [header, base64Content] = part.image.split(",");
-                          const mimeType = header.split(";")[0].split(":")[1];
+                          const { mimeType, base64Content } = splitDataUrl(
+                            part.image
+                          );
                           if (mimeType == null || base64Content == null) {
                             throw new Error("Invalid data URL format");
                           }
@@ -1029,11 +1046,6 @@ function convertToLanguageModelMessage(message, downloadedImages) {
                           );
                         }
                       }
-                      default: {
-                        throw new Error(
-                          `Unsupported URL protocol: ${url.protocol}`
-                        );
-                      }
                     }
                   } catch (_ignored) {
                   }
@@ -1048,12 +1060,75 @@ function convertToLanguageModelMessage(message, downloadedImages) {
               }
               case "file": {
                 if (part.data instanceof URL) {
-                  return {
-                    type: "file",
-                    data: part.data,
-                    mimeType: part.mimeType,
-                    providerMetadata: part.experimental_providerMetadata
-                  };
+                  if (downloadedAssets == null) {
+                    return {
+                      type: "file",
+                      data: part.data,
+                      mimeType: part.mimeType,
+                      providerMetadata: part.experimental_providerMetadata
+                    };
+                  } else {
+                    const downloadedImage = downloadedAssets[part.data.toString()];
+                    return {
+                      type: "file",
+                      data: (0, import_provider_utils3.convertUint8ArrayToBase64)(downloadedImage.data),
+                      mimeType: (_d = part.mimeType) != null ? _d : downloadedImage.mimeType,
+                      providerMetadata: part.experimental_providerMetadata
+                    };
+                  }
+                }
+                if (typeof part.data === "string") {
+                  try {
+                    const url = new URL(part.data);
+                    switch (url.protocol) {
+                      case "http:":
+                      case "https:": {
+                        if (downloadedAssets == null) {
+                          return {
+                            type: "file",
+                            data: url,
+                            mimeType: part.mimeType,
+                            providerMetadata: part.experimental_providerMetadata
+                          };
+                        } else {
+                          const downloadedImage = downloadedAssets[url.toString()];
+                          return {
+                            type: "file",
+                            data: (0, import_provider_utils3.convertUint8ArrayToBase64)(
+                              downloadedImage.data
+                            ),
+                            mimeType: (_e = part.mimeType) != null ? _e : downloadedImage.mimeType,
+                            providerMetadata: part.experimental_providerMetadata
+                          };
+                        }
+                      }
+                      case "data:": {
+                        try {
+                          const { mimeType, base64Content } = splitDataUrl(
+                            part.data
+                          );
+                          if (mimeType == null || base64Content == null) {
+                            throw new Error("Invalid data URL format");
+                          }
+                          return {
+                            type: "file",
+                            data: convertDataContentToBase64String(
+                              base64Content
+                            ),
+                            mimeType,
+                            providerMetadata: part.experimental_providerMetadata
+                          };
+                        } catch (error) {
+                          throw new Error(
+                            `Error processing data URL: ${(0, import_provider_utils3.getErrorMessage)(
+                              message
+                            )}`
+                          );
+                        }
+                      }
+                    }
+                  } catch (_ignored) {
+                  }
                 }
                 const imageBase64 = convertDataContentToBase64String(
                   part.data
@@ -1113,12 +1188,14 @@ function convertToLanguageModelMessage(message, downloadedImages) {
     }
   }
 }
-async function downloadImages(messages, downloadImplementation) {
+async function downloadAssets(messages, downloadImplementation) {
   const urls = messages.filter((message) => message.role === "user").map((message) => message.content).filter(
     (content) => Array.isArray(content)
-  ).flat().filter((part) => part.type === "image").map((part) => part.image).map(
+  ).flat().filter(
+    (part) => part.type === "image" || part.type === "file"
+  ).map((part) => part.type === "image" ? part.image : part.data).map(
     (part) => (
-      // support string urls in image parts:
+      // support string urls:
       typeof part === "string" && (part.startsWith("http:") || part.startsWith("https:")) ? new URL(part) : part
     )
   ).filter((image) => image instanceof URL);
@@ -4687,6 +4764,17 @@ function attachmentsToParts(attachments) {
       case "https:": {
         if ((_a11 = attachment.contentType) == null ? void 0 : _a11.startsWith("image/")) {
           parts.push({ type: "image", image: url });
+        } else {
+          if (!attachment.contentType) {
+            throw new Error(
+              "If the attachment is not an image, it must specify a content type"
+            );
+          }
+          parts.push({
+            type: "file",
+            data: url,
+            mimeType: attachment.contentType
+          });
         }
         break;
       }
@@ -4715,6 +4803,17 @@ function attachmentsToParts(attachments) {
               convertDataContentToUint8Array(base64Content)
             )
           });
+        } else {
+          if (!attachment.contentType) {
+            throw new Error(
+              "If the attachment is not an image or text, it must specify a content type"
+            );
+          }
+          parts.push({
+            type: "file",
+            data: base64Content,
+            mimeType: attachment.contentType
+          });
         }
         break;
       }