ai 6.0.178 → 6.0.182

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai",
-  "version": "6.0.178",
+  "version": "6.0.182",
   "description": "AI SDK by Vercel - build apps like ChatGPT, Claude, Gemini, and more with a single interface for any model using the Vercel AI Gateway or go direct to OpenAI, Anthropic, Google, or any other model provider.",
   "license": "Apache-2.0",
   "sideEffects": false,
@@ -45,7 +45,7 @@
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.0",
-    "@ai-sdk/gateway": "3.0.112",
+    "@ai-sdk/gateway": "3.0.114",
     "@ai-sdk/provider": "3.0.10",
     "@ai-sdk/provider-utils": "4.0.27"
   },

package/src/prompt/convert-to-language-model-prompt.ts

@@ -25,7 +25,10 @@ import {
   createDefaultDownloadFunction,
   type DownloadFunction,
 } from '../util/download/download-function';
-import { convertToLanguageModelV3DataContent } from './data-content';
+import {
+  convertDataContentToBase64String,
+  convertToLanguageModelV3DataContent,
+} from './data-content';
 import { InvalidMessageRoleError } from './invalid-message-role-error';
 import type { StandardizedPrompt } from './standardize-prompt';
 import { asArray } from '../util/as-array';
@@ -283,7 +286,10 @@ export function convertToLanguageModelMessage({
                   type: 'tool-result' as const,
                   toolCallId: part.toolCallId,
                   toolName: part.toolName,
-                  output: mapToolResultOutput(part.output),
+                  output: mapToolResultOutput({
+                    output: part.output,
+                    downloadedAssets,
+                  }),
                   providerOptions,
                 };
               }
@@ -309,7 +315,10 @@ export function convertToLanguageModelMessage({
                   type: 'tool-result' as const,
                   toolCallId: part.toolCallId,
                   toolName: part.toolName,
-                  output: mapToolResultOutput(part.output),
+                  output: mapToolResultOutput({
+                    output: part.output,
+                    downloadedAssets,
+                  }),
                   providerOptions: part.providerOptions,
                 };
               }
@@ -344,28 +353,59 @@ async function downloadAssets(
 ): Promise<
   Record<string, { mediaType: string | undefined; data: Uint8Array }>
 > {
-  const plannedDownloads = messages
-    .filter(message => message.role === 'user')
-    .map(message => message.content)
-    .filter((content): content is Array<TextPart | ImagePart | FilePart> =>
-      Array.isArray(content),
-    )
-    .flat()
-    .filter(
-      (part): part is ImagePart | FilePart =>
-        part.type === 'image' || part.type === 'file',
-    )
-    .map(part => {
-      const mediaType =
-        part.mediaType ?? (part.type === 'image' ? 'image/*' : undefined);
-
-      let data = part.type === 'image' ? part.image : part.data;
-      if (typeof data === 'string') {
-        try {
-          data = new URL(data);
-        } catch (ignored) {}
+  type DownloadableFile = {
+    mediaType: string | undefined;
+    data: DataContent | URL;
+  };
+
+  const downloadableFiles: DownloadableFile[] = [];
+  for (const message of messages) {
+    if (message.role === 'user' && Array.isArray(message.content)) {
+      for (const part of message.content) {
+        if (part.type === 'image' || part.type === 'file') {
+          downloadableFiles.push({
+            data: part.type === 'image' ? part.image : part.data,
+            mediaType:
+              part.mediaType ?? (part.type === 'image' ? 'image/*' : undefined),
+          });
+        }
       }
+    }
+
+    if (message.role === 'tool' || message.role === 'assistant') {
+      if (!Array.isArray(message.content)) {
+        continue;
+      }
+
+      for (const part of message.content) {
+        if (part.type !== 'tool-result') {
+          continue;
+        }
 
+        if (part.output.type !== 'content') {
+          continue;
+        }
+
+        for (const contentPart of part.output.value) {
+          if (
+            contentPart.type === 'image-url' ||
+            contentPart.type === 'file-url'
+          ) {
+            downloadableFiles.push({
+              data: new URL(contentPart.url),
+              mediaType:
+                contentPart.type === 'image-url' ? 'image/*' : undefined,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  const plannedDownloads = downloadableFiles
+    .map(part => {
+      const mediaType = part.mediaType;
+      const { data } = convertToLanguageModelV3DataContent(part.data);
       return { mediaType, data };
     })
 
@@ -383,7 +423,6 @@ async function downloadAssets(
           supportedUrls,
         }),
     }));
-
   // download in parallel:
   const downloadedFiles = await download(plannedDownloads);
 
@@ -492,9 +531,16 @@ function convertPartToLanguageModelPart(
   }
 }
 
-function mapToolResultOutput(
-  output: ToolResultOutput,
-): LanguageModelV3ToolResultOutput {
+function mapToolResultOutput({
+  output,
+  downloadedAssets,
+}: {
+  output: ToolResultOutput;
+  downloadedAssets: Record<
+    string,
+    { mediaType: string | undefined; data: Uint8Array }
+  >;
+}): LanguageModelV3ToolResultOutput {
   if (output.type !== 'content') {
     return output;
   }
@@ -502,6 +548,34 @@ function mapToolResultOutput(
   return {
     type: 'content',
     value: output.value.map(item => {
+      if (item.type === 'image-url') {
+        const downloadedFile = downloadedAssets[new URL(item.url).toString()];
+        if (downloadedFile) {
+          return {
+            type: 'image-data' as const,
+            data: convertDataContentToBase64String(downloadedFile.data),
+            mediaType: downloadedFile.mediaType ?? 'image/*',
+            providerOptions: item.providerOptions,
+          };
+        }
+
+        return item;
+      }
+
+      if (item.type === 'file-url') {
+        const downloadedFile = downloadedAssets[new URL(item.url).toString()];
+        if (downloadedFile) {
+          return {
+            type: 'file-data' as const,
+            data: convertDataContentToBase64String(downloadedFile.data),
+            mediaType: downloadedFile.mediaType ?? 'application/octet-stream',
+            providerOptions: item.providerOptions,
+          };
+        }
+
+        return item;
+      }
+
       if (item.type !== 'media') {
         return item;
       }

package/src/ui/validate-ui-messages.ts

@@ -168,7 +168,7 @@ const uiMessagesSchema = lazySchema(() =>
                   toolCallId: z.string(),
                   toolMetadata: toolMetadataSchema.optional(),
                   state: z.literal('output-error'),
-                  input: z.unknown(),
+                  input: z.unknown().optional(),
                   rawInput: z.unknown().optional(),
                   providerExecuted: z.boolean().optional(),
                   output: z.never().optional(),
@@ -282,7 +282,7 @@ const uiMessagesSchema = lazySchema(() =>
                   toolMetadata: toolMetadataSchema.optional(),
                   state: z.literal('output-error'),
                   providerExecuted: z.boolean().optional(),
-                  input: z.unknown(),
+                  input: z.unknown().optional(),
                   rawInput: z.unknown().optional(),
                   output: z.never().optional(),
                   errorText: z.string(),

package/src/util/retry-with-exponential-backoff.ts

@@ -1,4 +1,5 @@
 import { APICallError } from '@ai-sdk/provider';
+import { GatewayError } from '@ai-sdk/gateway';
 import { delay, getErrorMessage, isAbortError } from '@ai-sdk/provider-utils';
 import { RetryError } from './retry-error';
 
@@ -10,10 +11,14 @@ function getRetryDelayInMs({
   error,
   exponentialBackoffDelay,
 }: {
-  error: APICallError;
+  error: APICallError | GatewayError;
   exponentialBackoffDelay: number;
 }): number {
-  const headers = error.responseHeaders;
+  const headers = APICallError.isInstance(error)
+    ? error.responseHeaders
+    : APICallError.isInstance(error.cause)
+      ? (error.cause as APICallError).responseHeaders
+      : undefined;
 
   if (!headers) return exponentialBackoffDelay;
 
@@ -117,13 +122,13 @@ async function _retryWithExponentialBackoff<OUTPUT>(
 
     if (
       error instanceof Error &&
-      APICallError.isInstance(error) &&
-      error.isRetryable === true &&
+      ((APICallError.isInstance(error) && error.isRetryable === true) ||
+        (GatewayError.isInstance(error) && error.isRetryable === true)) &&
       tryNumber <= maxRetries
     ) {
       await delay(
         getRetryDelayInMs({
-          error,
+          error: error as APICallError | GatewayError,
           exponentialBackoffDelay: delayInMs,
         }),
         { abortSignal },