ai 3.4.18 → 3.4.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "3.4.18",
+  "version": "3.4.20",
   "description": "AI SDK by Vercel - The AI Toolkit for TypeScript and JavaScript",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -64,20 +64,19 @@
     }
   },
   "dependencies": {
-    "@ai-sdk/provider": "0.0.24",
-    "@ai-sdk/provider-utils": "1.0.20",
-    "@ai-sdk/react": "0.0.64",
-    "@ai-sdk/solid": "0.0.50",
-    "@ai-sdk/svelte": "0.0.52",
-    "@ai-sdk/ui-utils": "0.0.46",
-    "@ai-sdk/vue": "0.0.55",
+    "@ai-sdk/provider": "0.0.26",
+    "@ai-sdk/provider-utils": "1.0.22",
+    "@ai-sdk/react": "0.0.66",
+    "@ai-sdk/solid": "0.0.52",
+    "@ai-sdk/svelte": "0.0.54",
+    "@ai-sdk/ui-utils": "0.0.48",
+    "@ai-sdk/vue": "0.0.57",
     "@opentelemetry/api": "1.9.0",
     "eventsource-parser": "1.1.2",
     "jsondiffpatch": "0.6.0",
-    "json-schema": "0.4.0",
-    "nanoid": "3.3.6",
-    "secure-json-parse": "2.7.0",
-    "zod-to-json-schema": "3.23.2"
+    "json-schema": "^0.4.0",
+    "secure-json-parse": "^2.7.0",
+    "zod-to-json-schema": "^3.23.3"
   },
   "devDependencies": {
     "@anthropic-ai/sdk": "0.18.0",

package/rsc/dist/index.d.ts

@@ -234,6 +234,15 @@ Data content. Can either be a base64-encoded string, a Uint8Array, an ArrayBuffe
  */
 type DataContent = string | Uint8Array | ArrayBuffer | Buffer;
 
+type ToolResultContent = Array<{
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType?: string;
+}>;
+
 /**
 Text content part of a prompt. It contains a string of text.
  */
@@ -338,6 +347,10 @@ interface ToolResultPart {
      */
     result: unknown;
     /**
+  Multi-part content of the tool result. Only for tools that support multipart results.
+     */
+    experimental_content?: ToolResultContent;
+    /**
   Optional flag if the result is an error or an error message.
      */
     isError?: boolean;

package/rsc/dist/rsc-server.d.mts

@@ -232,6 +232,15 @@ Data content. Can either be a base64-encoded string, a Uint8Array, an ArrayBuffe
  */
 type DataContent = string | Uint8Array | ArrayBuffer | Buffer;
 
+type ToolResultContent = Array<{
+    type: 'text';
+    text: string;
+} | {
+    type: 'image';
+    data: string;
+    mimeType?: string;
+}>;
+
 /**
 Text content part of a prompt. It contains a string of text.
  */
@@ -336,6 +345,10 @@ interface ToolResultPart {
      */
     result: unknown;
     /**
+  Multi-part content of the tool result. Only for tools that support multipart results.
+     */
+    experimental_content?: ToolResultContent;
+    /**
   Optional flag if the result is an error or an error message.
      */
     isError?: boolean;

package/rsc/dist/rsc-server.mjs

@@ -213,12 +213,6 @@ function createAI({
 // rsc/stream-ui/stream-ui.tsx
 import { safeParseJSON } from "@ai-sdk/provider-utils";
 
-// core/prompt/convert-to-language-model-prompt.ts
-import {
-  convertUint8ArrayToBase64 as convertUint8ArrayToBase642,
-  getErrorMessage
-} from "@ai-sdk/provider-utils";
-
 // util/download-error.ts
 import { AISDKError } from "@ai-sdk/provider";
 var name = "AI_DownloadError";
@@ -457,19 +451,21 @@ function splitDataUrl(dataUrl) {
 async function convertToLanguageModelPrompt({
   prompt,
   modelSupportsImageUrls = true,
+  modelSupportsUrl = () => false,
   downloadImplementation = download
 }) {
-  const downloadedAssets = modelSupportsImageUrls || prompt.messages == null ? null : await downloadAssets(prompt.messages, downloadImplementation);
-  const languageModelMessages = [];
-  if (prompt.system != null) {
-    languageModelMessages.push({ role: "system", content: prompt.system });
-  }
-  languageModelMessages.push(
+  const downloadedAssets = await downloadAssets(
+    prompt.messages,
+    downloadImplementation,
+    modelSupportsImageUrls,
+    modelSupportsUrl
+  );
+  return [
+    ...prompt.system != null ? [{ role: "system", content: prompt.system }] : [],
     ...prompt.messages.map(
       (message) => convertToLanguageModelMessage(message, downloadedAssets)
     )
-  );
-  return languageModelMessages;
+  ];
 }
 function convertToLanguageModelMessage(message, downloadedAssets) {
   const role = message.role;
@@ -491,178 +487,7 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
       }
       return {
         role: "user",
-        content: message.content.map(
-          (part) => {
-            var _a8, _b, _c, _d, _e;
-            switch (part.type) {
-              case "text": {
-                return {
-                  type: "text",
-                  text: part.text,
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-              case "image": {
-                if (part.image instanceof URL) {
-                  if (downloadedAssets == null) {
-                    return {
-                      type: "image",
-                      image: part.image,
-                      mimeType: part.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  } else {
-                    const downloadedImage = downloadedAssets[part.image.toString()];
-                    return {
-                      type: "image",
-                      image: downloadedImage.data,
-                      mimeType: (_a8 = part.mimeType) != null ? _a8 : downloadedImage.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  }
-                }
-                if (typeof part.image === "string") {
-                  try {
-                    const url = new URL(part.image);
-                    switch (url.protocol) {
-                      case "http:":
-                      case "https:": {
-                        if (downloadedAssets == null) {
-                          return {
-                            type: "image",
-                            image: url,
-                            mimeType: part.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } else {
-                          const downloadedImage = downloadedAssets[url.toString()];
-                          return {
-                            type: "image",
-                            image: downloadedImage.data,
-                            mimeType: (_b = part.mimeType) != null ? _b : downloadedImage.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        }
-                      }
-                      case "data:": {
-                        try {
-                          const { mimeType, base64Content } = splitDataUrl(
-                            part.image
-                          );
-                          if (mimeType == null || base64Content == null) {
-                            throw new Error("Invalid data URL format");
-                          }
-                          return {
-                            type: "image",
-                            image: convertDataContentToUint8Array(base64Content),
-                            mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } catch (error) {
-                          throw new Error(
-                            `Error processing data URL: ${getErrorMessage(
-                              message
-                            )}`
-                          );
-                        }
-                      }
-                    }
-                  } catch (_ignored) {
-                  }
-                }
-                const imageUint8 = convertDataContentToUint8Array(part.image);
-                return {
-                  type: "image",
-                  image: imageUint8,
-                  mimeType: (_c = part.mimeType) != null ? _c : detectImageMimeType(imageUint8),
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-              case "file": {
-                if (part.data instanceof URL) {
-                  if (downloadedAssets == null) {
-                    return {
-                      type: "file",
-                      data: part.data,
-                      mimeType: part.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  } else {
-                    const downloadedImage = downloadedAssets[part.data.toString()];
-                    return {
-                      type: "file",
-                      data: convertUint8ArrayToBase642(downloadedImage.data),
-                      mimeType: (_d = part.mimeType) != null ? _d : downloadedImage.mimeType,
-                      providerMetadata: part.experimental_providerMetadata
-                    };
-                  }
-                }
-                if (typeof part.data === "string") {
-                  try {
-                    const url = new URL(part.data);
-                    switch (url.protocol) {
-                      case "http:":
-                      case "https:": {
-                        if (downloadedAssets == null) {
-                          return {
-                            type: "file",
-                            data: url,
-                            mimeType: part.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } else {
-                          const downloadedImage = downloadedAssets[url.toString()];
-                          return {
-                            type: "file",
-                            data: convertUint8ArrayToBase642(
-                              downloadedImage.data
-                            ),
-                            mimeType: (_e = part.mimeType) != null ? _e : downloadedImage.mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        }
-                      }
-                      case "data:": {
-                        try {
-                          const { mimeType, base64Content } = splitDataUrl(
-                            part.data
-                          );
-                          if (mimeType == null || base64Content == null) {
-                            throw new Error("Invalid data URL format");
-                          }
-                          return {
-                            type: "file",
-                            data: convertDataContentToBase64String(
-                              base64Content
-                            ),
-                            mimeType,
-                            providerMetadata: part.experimental_providerMetadata
-                          };
-                        } catch (error) {
-                          throw new Error(
-                            `Error processing data URL: ${getErrorMessage(
-                              message
-                            )}`
-                          );
-                        }
-                      }
-                    }
-                  } catch (_ignored) {
-                  }
-                }
-                const imageBase64 = convertDataContentToBase64String(
-                  part.data
-                );
-                return {
-                  type: "file",
-                  data: imageBase64,
-                  mimeType: part.mimeType,
-                  providerMetadata: part.experimental_providerMetadata
-                };
-              }
-            }
-          }
-        ).filter((part) => part.type !== "text" || part.text !== ""),
+        content: message.content.map((part) => convertPartToLanguageModelPart(part, downloadedAssets)).filter((part) => part.type !== "text" || part.text !== ""),
         providerMetadata: message.experimental_providerMetadata
       };
     }
@@ -697,6 +522,8 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
           toolCallId: part.toolCallId,
           toolName: part.toolName,
           result: part.result,
+          content: part.experimental_content,
+          isError: part.isError,
           providerMetadata: part.experimental_providerMetadata
         })),
         providerMetadata: message.experimental_providerMetadata
@@ -708,17 +535,19 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
     }
   }
 }
-async function downloadAssets(messages, downloadImplementation) {
+async function downloadAssets(messages, downloadImplementation, modelSupportsImageUrls, modelSupportsUrl) {
   const urls = messages.filter((message) => message.role === "user").map((message) => message.content).filter(
     (content) => Array.isArray(content)
   ).flat().filter(
     (part) => part.type === "image" || part.type === "file"
+  ).filter(
+    (part) => !(part.type === "image" && modelSupportsImageUrls === true)
   ).map((part) => part.type === "image" ? part.image : part.data).map(
     (part) => (
       // support string urls:
       typeof part === "string" && (part.startsWith("http:") || part.startsWith("https:")) ? new URL(part) : part
     )
-  ).filter((image) => image instanceof URL);
+  ).filter((image) => image instanceof URL).filter((url) => !modelSupportsUrl(url));
   const downloadedImages = await Promise.all(
     urls.map(async (url) => ({
       url,
@@ -729,6 +558,79 @@ async function downloadAssets(messages, downloadImplementation) {
     downloadedImages.map(({ url, data }) => [url.toString(), data])
   );
 }
+function convertPartToLanguageModelPart(part, downloadedAssets) {
+  if (part.type === "text") {
+    return {
+      type: "text",
+      text: part.text,
+      providerMetadata: part.experimental_providerMetadata
+    };
+  }
+  let mimeType = part.mimeType;
+  let data;
+  let content;
+  let normalizedData;
+  const type = part.type;
+  switch (type) {
+    case "image":
+      data = part.image;
+      break;
+    case "file":
+      data = part.data;
+      break;
+    default:
+      throw new Error(`Unsupported part type: ${type}`);
+  }
+  try {
+    content = typeof data === "string" ? new URL(data) : data;
+  } catch (error) {
+    content = data;
+  }
+  if (content instanceof URL) {
+    if (content.protocol === "data:") {
+      const { mimeType: dataUrlMimeType, base64Content } = splitDataUrl(
+        content.toString()
+      );
+      if (dataUrlMimeType == null || base64Content == null) {
+        throw new Error(`Invalid data URL format in part ${type}`);
+      }
+      mimeType = dataUrlMimeType;
+      normalizedData = convertDataContentToUint8Array(base64Content);
+    } else {
+      const downloadedFile = downloadedAssets[content.toString()];
+      if (downloadedFile) {
+        normalizedData = downloadedFile.data;
+        mimeType != null ? mimeType : mimeType = downloadedFile.mimeType;
+      } else {
+        normalizedData = content;
+      }
+    }
+  } else {
+    normalizedData = convertDataContentToUint8Array(content);
+  }
+  switch (type) {
+    case "image":
+      if (mimeType == null && normalizedData instanceof Uint8Array) {
+        mimeType = detectImageMimeType(normalizedData);
+      }
+      return {
+        type: "image",
+        image: normalizedData,
+        mimeType,
+        providerMetadata: part.experimental_providerMetadata
+      };
+    case "file":
+      if (mimeType == null) {
+        throw new Error(`Mime type is missing for file part`);
+      }
+      return {
+        type: "file",
+        data: normalizedData instanceof Uint8Array ? convertDataContentToBase64String(normalizedData) : normalizedData,
+        mimeType,
+        providerMetadata: part.experimental_providerMetadata
+      };
+  }
+}
 
 // errors/invalid-argument-error.ts
 import { AISDKError as AISDKError4 } from "@ai-sdk/provider";
@@ -906,12 +808,30 @@ function prepareToolsAndToolChoice({
     ([name8]) => activeTools.includes(name8)
   ) : Object.entries(tools);
   return {
-    tools: filteredTools.map(([name8, tool]) => ({
-      type: "function",
-      name: name8,
-      description: tool.description,
-      parameters: asSchema(tool.parameters).jsonSchema
-    })),
+    tools: filteredTools.map(([name8, tool]) => {
+      const toolType = tool.type;
+      switch (toolType) {
+        case void 0:
+        case "function":
+          return {
+            type: "function",
+            name: name8,
+            description: tool.description,
+            parameters: asSchema(tool.parameters).jsonSchema
+          };
+        case "provider-defined":
+          return {
+            type: "provider-defined",
+            name: name8,
+            id: tool.id,
+            args: tool.args
+          };
+        default: {
+          const exhaustiveCheck = toolType;
+          throw new Error(`Unsupported tool type: ${exhaustiveCheck}`);
+        }
+      }
+    }),
     toolChoice: toolChoice == null ? { type: "auto" } : typeof toolChoice === "string" ? { type: toolChoice } : { type: "tool", toolName: toolChoice.toolName }
   };
 }
@@ -919,10 +839,10 @@ function prepareToolsAndToolChoice({
 // core/prompt/standardize-prompt.ts
 import { InvalidPromptError } from "@ai-sdk/provider";
 import { safeValidateTypes } from "@ai-sdk/provider-utils";
-import { z as z6 } from "zod";
+import { z as z7 } from "zod";
 
 // core/prompt/message.ts
-import { z as z5 } from "zod";
+import { z as z6 } from "zod";
 
 // core/types/provider-metadata.ts
 import { z as z3 } from "zod";
@@ -947,67 +867,83 @@ var providerMetadataSchema = z3.record(
 );
 
 // core/prompt/content-part.ts
+import { z as z5 } from "zod";
+
+// core/prompt/tool-result-content.ts
 import { z as z4 } from "zod";
-var textPartSchema = z4.object({
-  type: z4.literal("text"),
-  text: z4.string(),
+var toolResultContentSchema = z4.array(
+  z4.union([
+    z4.object({ type: z4.literal("text"), text: z4.string() }),
+    z4.object({
+      type: z4.literal("image"),
+      data: z4.string(),
+      mimeType: z4.string().optional()
+    })
+  ])
+);
+
+// core/prompt/content-part.ts
+var textPartSchema = z5.object({
+  type: z5.literal("text"),
+  text: z5.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var imagePartSchema = z4.object({
-  type: z4.literal("image"),
-  image: z4.union([dataContentSchema, z4.instanceof(URL)]),
-  mimeType: z4.string().optional(),
+var imagePartSchema = z5.object({
+  type: z5.literal("image"),
+  image: z5.union([dataContentSchema, z5.instanceof(URL)]),
+  mimeType: z5.string().optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var filePartSchema = z4.object({
-  type: z4.literal("file"),
-  data: z4.union([dataContentSchema, z4.instanceof(URL)]),
-  mimeType: z4.string(),
+var filePartSchema = z5.object({
+  type: z5.literal("file"),
+  data: z5.union([dataContentSchema, z5.instanceof(URL)]),
+  mimeType: z5.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var toolCallPartSchema = z4.object({
-  type: z4.literal("tool-call"),
-  toolCallId: z4.string(),
-  toolName: z4.string(),
-  args: z4.unknown()
+var toolCallPartSchema = z5.object({
+  type: z5.literal("tool-call"),
+  toolCallId: z5.string(),
+  toolName: z5.string(),
+  args: z5.unknown()
 });
-var toolResultPartSchema = z4.object({
-  type: z4.literal("tool-result"),
-  toolCallId: z4.string(),
-  toolName: z4.string(),
-  result: z4.unknown(),
-  isError: z4.boolean().optional(),
+var toolResultPartSchema = z5.object({
+  type: z5.literal("tool-result"),
+  toolCallId: z5.string(),
+  toolName: z5.string(),
+  result: z5.unknown(),
+  content: toolResultContentSchema.optional(),
+  isError: z5.boolean().optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
 
 // core/prompt/message.ts
-var coreSystemMessageSchema = z5.object({
-  role: z5.literal("system"),
-  content: z5.string(),
+var coreSystemMessageSchema = z6.object({
+  role: z6.literal("system"),
+  content: z6.string(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreUserMessageSchema = z5.object({
-  role: z5.literal("user"),
-  content: z5.union([
-    z5.string(),
-    z5.array(z5.union([textPartSchema, imagePartSchema, filePartSchema]))
+var coreUserMessageSchema = z6.object({
+  role: z6.literal("user"),
+  content: z6.union([
+    z6.string(),
+    z6.array(z6.union([textPartSchema, imagePartSchema, filePartSchema]))
   ]),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreAssistantMessageSchema = z5.object({
-  role: z5.literal("assistant"),
-  content: z5.union([
-    z5.string(),
-    z5.array(z5.union([textPartSchema, toolCallPartSchema]))
+var coreAssistantMessageSchema = z6.object({
+  role: z6.literal("assistant"),
+  content: z6.union([
+    z6.string(),
+    z6.array(z6.union([textPartSchema, toolCallPartSchema]))
   ]),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreToolMessageSchema = z5.object({
-  role: z5.literal("tool"),
-  content: z5.array(toolResultPartSchema),
+var coreToolMessageSchema = z6.object({
+  role: z6.literal("tool"),
+  content: z6.array(toolResultPartSchema),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
-var coreMessageSchema = z5.union([
+var coreMessageSchema = z6.union([
   coreSystemMessageSchema,
   coreUserMessageSchema,
   coreAssistantMessageSchema,
@@ -1055,7 +991,7 @@ function standardizePrompt(prompt) {
   if (prompt.messages != null) {
     const validationResult = safeValidateTypes({
       value: prompt.messages,
-      schema: z6.array(coreMessageSchema)
+      schema: z7.array(coreMessageSchema)
     });
     if (!validationResult.success) {
       throw new InvalidPromptError({
@@ -1084,7 +1020,7 @@ function calculateLanguageModelUsage(usage) {
 }
 
 // errors/invalid-tool-arguments-error.ts
-import { AISDKError as AISDKError5, getErrorMessage as getErrorMessage2 } from "@ai-sdk/provider";
+import { AISDKError as AISDKError5, getErrorMessage } from "@ai-sdk/provider";
 var name5 = "AI_InvalidToolArgumentsError";
 var marker5 = `vercel.ai.error.${name5}`;
 var symbol5 = Symbol.for(marker5);
@@ -1094,7 +1030,7 @@ var InvalidToolArgumentsError = class extends AISDKError5 {
     toolArgs,
     toolName,
     cause,
-    message = `Invalid arguments for tool ${toolName}: ${getErrorMessage2(
+    message = `Invalid arguments for tool ${toolName}: ${getErrorMessage(
       cause
     )}`
   }) {
@@ -1181,7 +1117,7 @@ function isGenerator(value) {
 
 // util/retry-with-exponential-backoff.ts
 import { APICallError } from "@ai-sdk/provider";
-import { getErrorMessage as getErrorMessage3, isAbortError } from "@ai-sdk/provider-utils";
+import { getErrorMessage as getErrorMessage2, isAbortError } from "@ai-sdk/provider-utils";
 
 // util/delay.ts
 async function delay(delayInMs) {
@@ -1254,7 +1190,7 @@ async function _retryWithExponentialBackoff(f, {
     if (maxRetries === 0) {
       throw error;
     }
-    const errorMessage = getErrorMessage3(error);
+    const errorMessage = getErrorMessage2(error);
     const newErrors = [...errors, error];
     const tryNumber = newErrors.length;
     if (tryNumber > maxRetries) {
@@ -1487,7 +1423,8 @@ async function streamUI({
       inputFormat: validatedPrompt.type,
       prompt: await convertToLanguageModelPrompt({
         prompt: validatedPrompt,
-        modelSupportsImageUrls: model.supportsImageUrls
+        modelSupportsImageUrls: model.supportsImageUrls,
+        modelSupportsUrl: model.supportsUrl
       }),
       providerMetadata,
       abortSignal,