ai 3.4.17 → 3.4.20

package/rsc/dist/rsc-server.mjs

@@ -213,12 +213,6 @@ function createAI({
 // rsc/stream-ui/stream-ui.tsx
 import { safeParseJSON } from "@ai-sdk/provider-utils";
 
-// core/prompt/convert-to-language-model-prompt.ts
-import {
-  convertUint8ArrayToBase64 as convertUint8ArrayToBase642,
-  getErrorMessage
-} from "@ai-sdk/provider-utils";
-
 // util/download-error.ts
 import { AISDKError } from "@ai-sdk/provider";
 var name = "AI_DownloadError";
@@ -457,36 +451,21 @@ function splitDataUrl(dataUrl) {
 async function convertToLanguageModelPrompt({
   prompt,
   modelSupportsImageUrls = true,
+  modelSupportsUrl = () => false,
   downloadImplementation = download
 }) {
-  const languageModelMessages = [];
-  if (prompt.system != null) {
-    languageModelMessages.push({ role: "system", content: prompt.system });
-  }
-  const downloadedAssets = modelSupportsImageUrls || prompt.messages == null ? null : await downloadAssets(prompt.messages, downloadImplementation);
-  const promptType = prompt.type;
-  switch (promptType) {
-    case "prompt": {
-      languageModelMessages.push({
-        role: "user",
-        content: [{ type: "text", text: prompt.prompt }]
-      });
-      break;
-    }
-    case "messages": {
-      languageModelMessages.push(
-        ...prompt.messages.map(
-          (message) => convertToLanguageModelMessage(message, downloadedAssets)
-        )
-      );
-      break;
-    }
-    default: {
-      const _exhaustiveCheck = promptType;
-      throw new Error(`Unsupported prompt type: ${_exhaustiveCheck}`);
-    }
-  }
-  return languageModelMessages;
+  const downloadedAssets = await downloadAssets(
+    prompt.messages,
+    downloadImplementation,
+    modelSupportsImageUrls,
+    modelSupportsUrl
+  );
+  return [
+    ...prompt.system != null ? [{ role: "system", content: prompt.system }] : [],
+    ...prompt.messages.map(
+      (message) => convertToLanguageModelMessage(message, downloadedAssets)
+    )
+  ];
 }
 function convertToLanguageModelMessage(message, downloadedAssets) {
   const role = message.role;
@@ -508,178 +487,7 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
       }
       return {
         role: "user",
-        content: message.content.map(
-          (part) => {
-            var _a8, _b, _c, _d, _e;
-            switch (part.type) {
-              case "text": {
-                return {
-                  type: "text",
-                  text: part.text,
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-              case "image": {
-                if (part.image instanceof URL) {
-                  if (downloadedAssets == null) {
-                    return {
-                      type: "image",
-                      image: part.image,
-                      mimeType: part.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  } else {
-                    const downloadedImage = downloadedAssets[part.image.toString()];
-                    return {
-                      type: "image",
-                      image: downloadedImage.data,
-                      mimeType: (_a8 = part.mimeType) != null ? _a8 : downloadedImage.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  }
-                }
-                if (typeof part.image === "string") {
-                  try {
-                    const url = new URL(part.image);
-                    switch (url.protocol) {
-                      case "http:":
-                      case "https:": {
-                        if (downloadedAssets == null) {
-                          return {
-                            type: "image",
-                            image: url,
-                            mimeType: part.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } else {
-                          const downloadedImage = downloadedAssets[url.toString()];
-                          return {
-                            type: "image",
-                            image: downloadedImage.data,
-                            mimeType: (_b = part.mimeType) != null ? _b : downloadedImage.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        }
-                      }
-                      case "data:": {
-                        try {
-                          const { mimeType, base64Content } = splitDataUrl(
-                            part.image
-                          );
-                          if (mimeType == null || base64Content == null) {
-                            throw new Error("Invalid data URL format");
-                          }
-                          return {
-                            type: "image",
-                            image: convertDataContentToUint8Array(base64Content),
-                            mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } catch (error) {
-                          throw new Error(
-                            `Error processing data URL: ${getErrorMessage(
-                              message
-                            )}`
-                          );
-                        }
-                      }
-                    }
-                  } catch (_ignored) {
-                  }
-                }
-                const imageUint8 = convertDataContentToUint8Array(part.image);
-                return {
-                  type: "image",
-                  image: imageUint8,
-                  mimeType: (_c = part.mimeType) != null ? _c : detectImageMimeType(imageUint8),
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-              case "file": {
-                if (part.data instanceof URL) {
-                  if (downloadedAssets == null) {
-                    return {
-                      type: "file",
-                      data: part.data,
-                      mimeType: part.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  } else {
-                    const downloadedImage = downloadedAssets[part.data.toString()];
-                    return {
-                      type: "file",
-                      data: convertUint8ArrayToBase642(downloadedImage.data),
-                      mimeType: (_d = part.mimeType) != null ? _d : downloadedImage.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  }
-                }
-                if (typeof part.data === "string") {
-                  try {
-                    const url = new URL(part.data);
-                    switch (url.protocol) {
-                      case "http:":
-                      case "https:": {
-                        if (downloadedAssets == null) {
-                          return {
-                            type: "file",
-                            data: url,
-                            mimeType: part.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } else {
-                          const downloadedImage = downloadedAssets[url.toString()];
-                          return {
-                            type: "file",
-                            data: convertUint8ArrayToBase642(
-                              downloadedImage.data
-                            ),
-                            mimeType: (_e = part.mimeType) != null ? _e : downloadedImage.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        }
-                      }
-                      case "data:": {
-                        try {
-                          const { mimeType, base64Content } = splitDataUrl(
-                            part.data
-                          );
-                          if (mimeType == null || base64Content == null) {
-                            throw new Error("Invalid data URL format");
-                          }
-                          return {
-                            type: "file",
-                            data: convertDataContentToBase64String(
-                              base64Content
-                            ),
-                            mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } catch (error) {
-                          throw new Error(
-                            `Error processing data URL: ${getErrorMessage(
-                              message
-                            )}`
-                          );
-                        }
-                      }
-                    }
-                  } catch (_ignored) {
-                  }
-                }
-                const imageBase64 = convertDataContentToBase64String(
-                  part.data
-                );
-                return {
-                  type: "file",
-                  data: imageBase64,
-                  mimeType: part.mimeType,
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-            }
-          }
-        ).filter((part) => part.type !== "text" || part.text !== ""),
+        content: message.content.map((part) => convertPartToLanguageModelPart(part, downloadedAssets)).filter((part) => part.type !== "text" || part.text !== ""),
         providerMetadata: message.experimental_providerMetadata
       };
     }
@@ -714,6 +522,8 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
           toolCallId: part.toolCallId,
           toolName: part.toolName,
           result: part.result,
+          content: part.experimental_content,
+          isError: part.isError,
           providerMetadata: part.experimental_providerMetadata
         })),
         providerMetadata: message.experimental_providerMetadata
@@ -725,17 +535,19 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
     }
   }
 }
-async function downloadAssets(messages, downloadImplementation) {
+async function downloadAssets(messages, downloadImplementation, modelSupportsImageUrls, modelSupportsUrl) {
   const urls = messages.filter((message) => message.role === "user").map((message) => message.content).filter(
     (content) => Array.isArray(content)
   ).flat().filter(
     (part) => part.type === "image" || part.type === "file"
+  ).filter(
+    (part) => !(part.type === "image" && modelSupportsImageUrls === true)
   ).map((part) => part.type === "image" ? part.image : part.data).map(
     (part) => (
       // support string urls:
       typeof part === "string" && (part.startsWith("http:") || part.startsWith("https:")) ? new URL(part) : part
     )
-  ).filter((image) => image instanceof URL);
+  ).filter((image) => image instanceof URL).filter((url) => !modelSupportsUrl(url));
   const downloadedImages = await Promise.all(
     urls.map(async (url) => ({
       url,
@@ -746,6 +558,79 @@ async function downloadAssets(messages, downloadImplementation) {
     downloadedImages.map(({ url, data }) => [url.toString(), data])
   );
 }
+function convertPartToLanguageModelPart(part, downloadedAssets) {
+  if (part.type === "text") {
+    return {
+      type: "text",
+      text: part.text,
+      providerMetadata: part.experimental_providerMetadata
+    };
+  }
+  let mimeType = part.mimeType;
+  let data;
+  let content;
+  let normalizedData;
+  const type = part.type;
+  switch (type) {
+    case "image":
+      data = part.image;
+      break;
+    case "file":
+      data = part.data;
+      break;
+    default:
+      throw new Error(`Unsupported part type: ${type}`);
+  }
+  try {
+    content = typeof data === "string" ? new URL(data) : data;
+  } catch (error) {
+    content = data;
+  }
+  if (content instanceof URL) {
+    if (content.protocol === "data:") {
+      const { mimeType: dataUrlMimeType, base64Content } = splitDataUrl(
+        content.toString()
+      );
+      if (dataUrlMimeType == null || base64Content == null) {
+        throw new Error(`Invalid data URL format in part ${type}`);
+      }
+      mimeType = dataUrlMimeType;
+      normalizedData = convertDataContentToUint8Array(base64Content);
+    } else {
+      const downloadedFile = downloadedAssets[content.toString()];
+      if (downloadedFile) {
+        normalizedData = downloadedFile.data;
+        mimeType != null ? mimeType : mimeType = downloadedFile.mimeType;
+      } else {
+        normalizedData = content;
+      }
+    }
+  } else {
+    normalizedData = convertDataContentToUint8Array(content);
+  }
+  switch (type) {
+    case "image":
+      if (mimeType == null && normalizedData instanceof Uint8Array) {
+        mimeType = detectImageMimeType(normalizedData);
+      }
+      return {
+        type: "image",
+        image: normalizedData,
+        mimeType,
+        providerMetadata: part.experimental_providerMetadata
+      };
+    case "file":
+      if (mimeType == null) {
+        throw new Error(`Mime type is missing for file part`);
+      }
+      return {
+        type: "file",
+        data: normalizedData instanceof Uint8Array ? convertDataContentToBase64String(normalizedData) : normalizedData,
+        mimeType,
+        providerMetadata: part.experimental_providerMetadata
+      };
+  }
+}
 
 // errors/invalid-argument-error.ts
 import { AISDKError as AISDKError4 } from "@ai-sdk/provider";
@@ -923,23 +808,41 @@ function prepareToolsAndToolChoice({
     ([name8]) => activeTools.includes(name8)
   ) : Object.entries(tools);
   return {
-    tools: filteredTools.map(([name8, tool]) => ({
-      type: "function",
-      name: name8,
-      description: tool.description,
-      parameters: asSchema(tool.parameters).jsonSchema
-    })),
+    tools: filteredTools.map(([name8, tool]) => {
+      const toolType = tool.type;
+      switch (toolType) {
+        case void 0:
+        case "function":
+          return {
+            type: "function",
+            name: name8,
+            description: tool.description,
+            parameters: asSchema(tool.parameters).jsonSchema
+          };
+        case "provider-defined":
+          return {
+            type: "provider-defined",
+            name: name8,
+            id: tool.id,
+            args: tool.args
+          };
+        default: {
+          const exhaustiveCheck = toolType;
+          throw new Error(`Unsupported tool type: ${exhaustiveCheck}`);
+        }
+      }
+    }),
     toolChoice: toolChoice == null ? { type: "auto" } : typeof toolChoice === "string" ? { type: toolChoice } : { type: "tool", toolName: toolChoice.toolName }
   };
 }
 
-// core/prompt/validate-prompt.ts
+// core/prompt/standardize-prompt.ts
 import { InvalidPromptError } from "@ai-sdk/provider";
 import { safeValidateTypes } from "@ai-sdk/provider-utils";
-import { z as z6 } from "zod";
+import { z as z7 } from "zod";
 
 // core/prompt/message.ts
-import { z as z5 } from "zod";
+import { z as z6 } from "zod";
 
 // core/types/provider-metadata.ts
 import { z as z3 } from "zod";
@@ -964,75 +867,91 @@ var providerMetadataSchema = z3.record(
 );
 
 // core/prompt/content-part.ts
+import { z as z5 } from "zod";
+
+// core/prompt/tool-result-content.ts
 import { z as z4 } from "zod";
-var textPartSchema = z4.object({
-  type: z4.literal("text"),
-  text: z4.string(),
+var toolResultContentSchema = z4.array(
+  z4.union([
+    z4.object({ type: z4.literal("text"), text: z4.string() }),
+    z4.object({
+      type: z4.literal("image"),
+      data: z4.string(),
+      mimeType: z4.string().optional()
+    })
+  ])
+);
+
+// core/prompt/content-part.ts
+var textPartSchema = z5.object({
+  type: z5.literal("text"),
+  text: z5.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var imagePartSchema = z4.object({
-  type: z4.literal("image"),
-  image: z4.union([dataContentSchema, z4.instanceof(URL)]),
-  mimeType: z4.string().optional(),
+var imagePartSchema = z5.object({
+  type: z5.literal("image"),
+  image: z5.union([dataContentSchema, z5.instanceof(URL)]),
+  mimeType: z5.string().optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var filePartSchema = z4.object({
-  type: z4.literal("file"),
-  data: z4.union([dataContentSchema, z4.instanceof(URL)]),
-  mimeType: z4.string(),
+var filePartSchema = z5.object({
+  type: z5.literal("file"),
+  data: z5.union([dataContentSchema, z5.instanceof(URL)]),
+  mimeType: z5.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var toolCallPartSchema = z4.object({
-  type: z4.literal("tool-call"),
-  toolCallId: z4.string(),
-  toolName: z4.string(),
-  args: z4.unknown()
+var toolCallPartSchema = z5.object({
+  type: z5.literal("tool-call"),
+  toolCallId: z5.string(),
+  toolName: z5.string(),
+  args: z5.unknown()
 });
-var toolResultPartSchema = z4.object({
-  type: z4.literal("tool-result"),
-  toolCallId: z4.string(),
-  toolName: z4.string(),
-  result: z4.unknown(),
-  isError: z4.boolean().optional(),
+var toolResultPartSchema = z5.object({
+  type: z5.literal("tool-result"),
+  toolCallId: z5.string(),
+  toolName: z5.string(),
+  result: z5.unknown(),
+  content: toolResultContentSchema.optional(),
+  isError: z5.boolean().optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
 
 // core/prompt/message.ts
-var coreSystemMessageSchema = z5.object({
-  role: z5.literal("system"),
-  content: z5.string(),
+var coreSystemMessageSchema = z6.object({
+  role: z6.literal("system"),
+  content: z6.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreUserMessageSchema = z5.object({
-  role: z5.literal("user"),
-  content: z5.union([
-    z5.string(),
-    z5.array(z5.union([textPartSchema, imagePartSchema, filePartSchema]))
+var coreUserMessageSchema = z6.object({
+  role: z6.literal("user"),
+  content: z6.union([
+    z6.string(),
+    z6.array(z6.union([textPartSchema, imagePartSchema, filePartSchema]))
   ]),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreAssistantMessageSchema = z5.object({
-  role: z5.literal("assistant"),
-  content: z5.union([
-    z5.string(),
-    z5.array(z5.union([textPartSchema, toolCallPartSchema]))
+var coreAssistantMessageSchema = z6.object({
+  role: z6.literal("assistant"),
+  content: z6.union([
+    z6.string(),
+    z6.array(z6.union([textPartSchema, toolCallPartSchema]))
   ]),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreToolMessageSchema = z5.object({
-  role: z5.literal("tool"),
-  content: z5.array(toolResultPartSchema),
+var coreToolMessageSchema = z6.object({
+  role: z6.literal("tool"),
+  content: z6.array(toolResultPartSchema),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreMessageSchema = z5.union([
+var coreMessageSchema = z6.union([
   coreSystemMessageSchema,
   coreUserMessageSchema,
   coreAssistantMessageSchema,
   coreToolMessageSchema
 ]);
 
-// core/prompt/validate-prompt.ts
-function validatePrompt(prompt) {
+// core/prompt/standardize-prompt.ts
+function standardizePrompt(prompt) {
   if (prompt.prompt == null && prompt.messages == null) {
     throw new InvalidPromptError({
       prompt,
@@ -1060,15 +979,19 @@ function validatePrompt(prompt) {
     }
     return {
       type: "prompt",
-      prompt: prompt.prompt,
-      messages: void 0,
-      system: prompt.system
+      system: prompt.system,
+      messages: [
+        {
+          role: "user",
+          content: prompt.prompt
+        }
+      ]
     };
   }
   if (prompt.messages != null) {
     const validationResult = safeValidateTypes({
       value: prompt.messages,
-      schema: z6.array(coreMessageSchema)
+      schema: z7.array(coreMessageSchema)
     });
     if (!validationResult.success) {
       throw new InvalidPromptError({
@@ -1079,7 +1002,6 @@ function validatePrompt(prompt) {
     }
     return {
       type: "messages",
-      prompt: void 0,
       messages: prompt.messages,
       // only possible case bc of checks above
       system: prompt.system
@@ -1098,7 +1020,7 @@ function calculateLanguageModelUsage(usage) {
 }
 
 // errors/invalid-tool-arguments-error.ts
-import { AISDKError as AISDKError5, getErrorMessage as getErrorMessage2 } from "@ai-sdk/provider";
+import { AISDKError as AISDKError5, getErrorMessage } from "@ai-sdk/provider";
 var name5 = "AI_InvalidToolArgumentsError";
 var marker5 = `vercel.ai.error.${name5}`;
 var symbol5 = Symbol.for(marker5);
@@ -1108,7 +1030,7 @@ var InvalidToolArgumentsError = class extends AISDKError5 {
     toolArgs,
     toolName,
     cause,
-    message = `Invalid arguments for tool ${toolName}: ${getErrorMessage2(
+    message = `Invalid arguments for tool ${toolName}: ${getErrorMessage(
       cause
     )}`
   }) {
@@ -1195,7 +1117,7 @@ function isGenerator(value) {
 
 // util/retry-with-exponential-backoff.ts
 import { APICallError } from "@ai-sdk/provider";
-import { getErrorMessage as getErrorMessage3, isAbortError } from "@ai-sdk/provider-utils";
+import { getErrorMessage as getErrorMessage2, isAbortError } from "@ai-sdk/provider-utils";
 
 // util/delay.ts
 async function delay(delayInMs) {
@@ -1268,7 +1190,7 @@ async function _retryWithExponentialBackoff(f, {
     if (maxRetries === 0) {
       throw error;
     }
-    const errorMessage = getErrorMessage3(error);
+    const errorMessage = getErrorMessage2(error);
     const newErrors = [...errors, error];
     const tryNumber = newErrors.length;
     if (tryNumber > maxRetries) {
@@ -1486,7 +1408,7 @@ async function streamUI({
     renderFinished.resolve(void 0);
   }
   const retry = retryWithExponentialBackoff({ maxRetries });
-  const validatedPrompt = validatePrompt({ system, prompt, messages });
+  const validatedPrompt = standardizePrompt({ system, prompt, messages });
   const result = await retry(
     async () => model.doStream({
       mode: {
@@ -1501,7 +1423,8 @@ async function streamUI({
       inputFormat: validatedPrompt.type,
       prompt: await convertToLanguageModelPrompt({
         prompt: validatedPrompt,
-        modelSupportsImageUrls: model.supportsImageUrls
+        modelSupportsImageUrls: model.supportsImageUrls,
+        modelSupportsUrl: model.supportsUrl
       }),
       providerMetadata,
       abortSignal,