ai 5.0.0-canary.3 → 5.0.0-canary.4

package/dist/index.mjs

@@ -830,12 +830,12 @@ import {
 var DefaultGeneratedFile = class {
   constructor({
     data,
-    mimeType
+    mediaType
   }) {
     const isUint8Array = data instanceof Uint8Array;
     this.base64Data = isUint8Array ? void 0 : data;
     this.uint8ArrayData = isUint8Array ? data : void 0;
-    this.mimeType = mimeType;
+    this.mediaType = mediaType;
   }
   // lazy conversion with caching to avoid unnecessary conversion overhead:
   get base64() {
@@ -859,45 +859,45 @@ var DefaultGeneratedFileWithType = class extends DefaultGeneratedFile {
   }
 };
 
-// core/util/detect-image-mimetype.ts
-var mimeTypeSignatures = [
+// core/util/detect-media-type.ts
+var imageMediaTypeSignatures = [
   {
-    mimeType: "image/gif",
+    mediaType: "image/gif",
     bytesPrefix: [71, 73, 70],
     base64Prefix: "R0lG"
   },
   {
-    mimeType: "image/png",
+    mediaType: "image/png",
     bytesPrefix: [137, 80, 78, 71],
     base64Prefix: "iVBORw"
   },
   {
-    mimeType: "image/jpeg",
+    mediaType: "image/jpeg",
     bytesPrefix: [255, 216],
     base64Prefix: "/9j/"
   },
   {
-    mimeType: "image/webp",
+    mediaType: "image/webp",
     bytesPrefix: [82, 73, 70, 70],
     base64Prefix: "UklGRg"
   },
   {
-    mimeType: "image/bmp",
+    mediaType: "image/bmp",
     bytesPrefix: [66, 77],
     base64Prefix: "Qk"
   },
   {
-    mimeType: "image/tiff",
+    mediaType: "image/tiff",
     bytesPrefix: [73, 73, 42, 0],
     base64Prefix: "SUkqAA"
   },
   {
-    mimeType: "image/tiff",
+    mediaType: "image/tiff",
     bytesPrefix: [77, 77, 0, 42],
     base64Prefix: "TU0AKg"
   },
   {
-    mimeType: "image/avif",
+    mediaType: "image/avif",
     bytesPrefix: [
       0,
       0,
@@ -915,7 +915,7 @@ var mimeTypeSignatures = [
     base64Prefix: "AAAAIGZ0eXBhdmlm"
   },
   {
-    mimeType: "image/heic",
+    mediaType: "image/heic",
     bytesPrefix: [
       0,
       0,
@@ -933,10 +933,45 @@ var mimeTypeSignatures = [
     base64Prefix: "AAAAIGZ0eXBoZWlj"
   }
 ];
-function detectImageMimeType(image) {
-  for (const signature of mimeTypeSignatures) {
-    if (typeof image === "string" ? image.startsWith(signature.base64Prefix) : image.length >= signature.bytesPrefix.length && signature.bytesPrefix.every((byte, index) => image[index] === byte)) {
-      return signature.mimeType;
+var audioMediaTypeSignatures = [
+  {
+    mediaType: "audio/mpeg",
+    bytesPrefix: [255, 251],
+    base64Prefix: "//s="
+  },
+  {
+    mediaType: "audio/wav",
+    bytesPrefix: [82, 73, 70, 70],
+    base64Prefix: "UklGR"
+  },
+  {
+    mediaType: "audio/ogg",
+    bytesPrefix: [79, 103, 103, 83],
+    base64Prefix: "T2dnUw"
+  },
+  {
+    mediaType: "audio/flac",
+    bytesPrefix: [102, 76, 97, 67],
+    base64Prefix: "ZkxhQw"
+  },
+  {
+    mediaType: "audio/aac",
+    bytesPrefix: [64, 21, 0, 0],
+    base64Prefix: "QBUA"
+  },
+  {
+    mediaType: "audio/mp4",
+    bytesPrefix: [102, 116, 121, 112],
+    base64Prefix: "ZnR5cA"
+  }
+];
+function detectMediaType({
+  data,
+  signatures
+}) {
+  for (const signature of signatures) {
+    if (typeof data === "string" ? data.startsWith(signature.base64Prefix) : data.length >= signature.bytesPrefix.length && signature.bytesPrefix.every((byte, index) => data[index] === byte)) {
+      return signature.mediaType;
     }
   }
   return void 0;
@@ -992,7 +1027,10 @@ async function generateImage({
           var _a18;
           return new DefaultGeneratedFile({
             data: image,
-            mimeType: (_a18 = detectImageMimeType(image)) != null ? _a18 : "image/png"
+            mediaType: (_a18 = detectMediaType({
+              data: image,
+              signatures: imageMediaTypeSignatures
+            })) != null ? _a18 : "image/png"
           });
         }
       )
@@ -1051,6 +1089,9 @@ var NoObjectGeneratedError = class extends AISDKError4 {
 };
 _a4 = symbol4;
 
+// core/prompt/convert-to-language-model-prompt.ts
+import { convertUint8ArrayToBase64 as convertUint8ArrayToBase643 } from "@ai-sdk/provider-utils";
+
 // util/download-error.ts
 import { AISDKError as AISDKError5 } from "@ai-sdk/provider";
 var name5 = "AI_DownloadError";
@@ -1092,7 +1133,7 @@ async function download({ url }) {
     }
     return {
       data: new Uint8Array(await response.arrayBuffer()),
-      mimeType: (_a17 = response.headers.get("content-type")) != null ? _a17 : void 0
+      mediaType: (_a17 = response.headers.get("content-type")) != null ? _a17 : void 0
     };
   } catch (error) {
     if (DownloadError.isInstance(error)) {
@@ -1208,12 +1249,12 @@ function splitDataUrl(dataUrl) {
   try {
     const [header, base64Content] = dataUrl.split(",");
     return {
-      mimeType: header.split(";")[0].split(":")[1],
+      mediaType: header.split(";")[0].split(":")[1],
       base64Content
     };
   } catch (error) {
     return {
-      mimeType: void 0,
+      mediaType: void 0,
       base64Content: void 0
     };
   }
@@ -1278,7 +1319,7 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
           // remove empty text parts:
           (part) => part.type !== "text" || part.text !== ""
         ).map((part) => {
-          var _a18;
+          var _a18, _b2;
           const providerOptions = (_a18 = part.providerOptions) != null ? _a18 : part.experimental_providerMetadata;
           switch (part.type) {
             case "file": {
@@ -1286,7 +1327,7 @@ function convertToLanguageModelMessage(message, downloadedAssets) {
                 type: "file",
                 data: part.data instanceof URL ? part.data : convertDataContentToBase64String(part.data),
                 filename: part.filename,
-                mimeType: part.mimeType,
+                mediaType: (_b2 = part.mediaType) != null ? _b2 : part.mimeType,
                 providerOptions
               };
             }
@@ -1374,7 +1415,7 @@ async function downloadAssets(messages, downloadImplementation, modelSupportsIma
   );
 }
 function convertPartToLanguageModelPart(part, downloadedAssets) {
-  var _a17, _b, _c, _d;
+  var _a17, _b, _c, _d, _e;
   if (part.type === "text") {
     return {
       type: "text",
@@ -1382,7 +1423,7 @@ function convertPartToLanguageModelPart(part, downloadedAssets) {
       providerOptions: (_a17 = part.providerOptions) != null ? _a17 : part.experimental_providerMetadata
     };
   }
-  let mimeType = part.mimeType;
+  let mediaType = (_b = part.mediaType) != null ? _b : part.mimeType;
   let data;
   let content;
   let normalizedData;
@@ -1404,19 +1445,19 @@ function convertPartToLanguageModelPart(part, downloadedAssets) {
   }
   if (content instanceof URL) {
     if (content.protocol === "data:") {
-      const { mimeType: dataUrlMimeType, base64Content } = splitDataUrl(
+      const { mediaType: dataUrlMediaType, base64Content } = splitDataUrl(
         content.toString()
       );
-      if (dataUrlMimeType == null || base64Content == null) {
+      if (dataUrlMediaType == null || base64Content == null) {
         throw new Error(`Invalid data URL format in part ${type}`);
       }
-      mimeType = dataUrlMimeType;
+      mediaType = dataUrlMediaType;
       normalizedData = convertDataContentToUint8Array(base64Content);
     } else {
       const downloadedFile = downloadedAssets[content.toString()];
       if (downloadedFile) {
         normalizedData = downloadedFile.data;
-        mimeType != null ? mimeType : mimeType = downloadedFile.mimeType;
+        mediaType != null ? mediaType : mediaType = downloadedFile.mediaType;
       } else {
         normalizedData = content;
       }
@@ -1427,25 +1468,30 @@ function convertPartToLanguageModelPart(part, downloadedAssets) {
   switch (type) {
     case "image": {
       if (normalizedData instanceof Uint8Array) {
-        mimeType = (_b = detectImageMimeType(normalizedData)) != null ? _b : mimeType;
+        mediaType = (_c = detectMediaType({
+          data: normalizedData,
+          signatures: imageMediaTypeSignatures
+        })) != null ? _c : mediaType;
       }
       return {
-        type: "image",
-        image: normalizedData,
-        mimeType,
-        providerOptions: (_c = part.providerOptions) != null ? _c : part.experimental_providerMetadata
+        type: "file",
+        mediaType: mediaType != null ? mediaType : "image/*",
+        // any image
+        filename: void 0,
+        data: normalizedData instanceof Uint8Array ? convertUint8ArrayToBase643(normalizedData) : normalizedData,
+        providerOptions: (_d = part.providerOptions) != null ? _d : part.experimental_providerMetadata
       };
     }
     case "file": {
-      if (mimeType == null) {
-        throw new Error(`Mime type is missing for file part`);
+      if (mediaType == null) {
+        throw new Error(`Media type is missing for file part`);
       }
       return {
         type: "file",
-        data: normalizedData instanceof Uint8Array ? convertDataContentToBase64String(normalizedData) : normalizedData,
+        mediaType,
         filename: part.filename,
-        mimeType,
-        providerOptions: (_d = part.providerOptions) != null ? _d : part.experimental_providerMetadata
+        data: normalizedData instanceof Uint8Array ? convertDataContentToBase64String(normalizedData) : normalizedData,
+        providerOptions: (_e = part.providerOptions) != null ? _e : part.experimental_providerMetadata
       };
     }
   }
@@ -1575,7 +1621,7 @@ function attachmentsToParts(attachments) {
           parts.push({
             type: "file",
             data: url,
-            mimeType: attachment.contentType
+            mediaType: attachment.contentType
           });
         }
         break;
@@ -1583,14 +1629,14 @@ function attachmentsToParts(attachments) {
       case "data:": {
         let header;
         let base64Content;
-        let mimeType;
+        let mediaType;
         try {
           [header, base64Content] = attachment.url.split(",");
-          mimeType = header.split(";")[0].split(":")[1];
+          mediaType = header.split(";")[0].split(":")[1];
         } catch (error) {
           throw new Error(`Error processing data URL: ${attachment.url}`);
         }
-        if (mimeType == null || base64Content == null) {
+        if (mediaType == null || base64Content == null) {
           throw new Error(`Invalid data URL format: ${attachment.url}`);
         }
         if ((_b = attachment.contentType) == null ? void 0 : _b.startsWith("image/")) {
@@ -1614,7 +1660,7 @@ function attachmentsToParts(attachments) {
           parts.push({
             type: "file",
             data: base64Content,
-            mimeType: attachment.contentType
+            mediaType: attachment.contentType
           });
         }
         break;
@@ -1689,14 +1735,23 @@ function convertToCoreMessages(messages, options) {
       case "assistant": {
         if (message.parts != null) {
           let processBlock2 = function() {
+            var _a18;
             const content2 = [];
             for (const part of block) {
               switch (part.type) {
-                case "file":
                 case "text": {
                   content2.push(part);
                   break;
                 }
+                case "file": {
+                  content2.push({
+                    type: "file",
+                    data: part.data,
+                    mediaType: (_a18 = part.mediaType) != null ? _a18 : part.mimeType
+                    // TODO migration, remove
+                  });
+                  break;
+                }
                 case "reasoning": {
                   for (const detail of part.details) {
                     switch (detail.type) {
@@ -1953,7 +2008,7 @@ var toolResultContentSchema = z4.array(
     z4.object({
       type: z4.literal("image"),
       data: z4.string(),
-      mimeType: z4.string().optional()
+      mediaType: z4.string().optional()
     })
   ])
 );
@@ -1968,6 +2023,7 @@ var textPartSchema = z5.object({
 var imagePartSchema = z5.object({
   type: z5.literal("image"),
   image: z5.union([dataContentSchema, z5.instanceof(URL)]),
+  mediaType: z5.string().optional(),
   mimeType: z5.string().optional(),
   providerOptions: providerMetadataSchema.optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
@@ -1976,7 +2032,8 @@ var filePartSchema = z5.object({
   type: z5.literal("file"),
   data: z5.union([dataContentSchema, z5.instanceof(URL)]),
   filename: z5.string().optional(),
-  mimeType: z5.string(),
+  mediaType: z5.string(),
+  mimeType: z5.string().optional(),
   providerOptions: providerMetadataSchema.optional(),
   experimental_providerMetadata: providerMetadataSchema.optional()
 });
@@ -3975,7 +4032,7 @@ function toResponseMessages({
       ...files.map((file) => ({
         type: "file",
         data: file.base64,
-        mimeType: file.mimeType
+        mediaType: file.mediaType
       })),
       { type: "text", text: text2 },
       ...toolCalls
@@ -4855,7 +4912,7 @@ function runToolsTransformation({
           controller.enqueue(
             new DefaultGeneratedFileWithType({
               data: chunk.data,
-              mimeType: chunk.mimeType
+              mediaType: chunk.mediaType
             })
           );
           break;
@@ -6035,7 +6092,7 @@ var DefaultStreamTextResult = class {
             case "file": {
               controller.enqueue(
                 formatDataStreamPart2("file", {
-                  mimeType: chunk.mimeType,
+                  mimeType: chunk.mediaType,
                   data: chunk.base64
                 })
               );
@@ -6239,6 +6296,70 @@ var DefaultStreamTextResult = class {
   }
 };
 
+// errors/no-transcript-generated-error.ts
+import { AISDKError as AISDKError18 } from "@ai-sdk/provider";
+var NoTranscriptGeneratedError = class extends AISDKError18 {
+  constructor(options) {
+    super({
+      name: "AI_NoTranscriptGeneratedError",
+      message: "No transcript generated."
+    });
+    this.responses = options.responses;
+  }
+};
+
+// core/transcribe/transcribe.ts
+async function transcribe({
+  model,
+  audio,
+  providerOptions = {},
+  maxRetries: maxRetriesArg,
+  abortSignal,
+  headers
+}) {
+  const { retry } = prepareRetries({ maxRetries: maxRetriesArg });
+  const audioData = audio instanceof URL ? (await download({ url: audio })).data : convertDataContentToUint8Array(audio);
+  const result = await retry(
+    () => {
+      var _a17;
+      return model.doGenerate({
+        audio: audioData,
+        abortSignal,
+        headers,
+        providerOptions,
+        mediaType: (_a17 = detectMediaType({
+          data: audioData,
+          signatures: audioMediaTypeSignatures
+        })) != null ? _a17 : "audio/wav"
+      });
+    }
+  );
+  if (!result.text) {
+    throw new NoTranscriptGeneratedError({ responses: [result.response] });
+  }
+  return new DefaultTranscriptionResult({
+    text: result.text,
+    segments: result.segments,
+    language: result.language,
+    durationInSeconds: result.durationInSeconds,
+    warnings: result.warnings,
+    responses: [result.response],
+    providerMetadata: result.providerMetadata
+  });
+}
+var DefaultTranscriptionResult = class {
+  constructor(options) {
+    var _a17;
+    this.text = options.text;
+    this.segments = options.segments;
+    this.language = options.language;
+    this.durationInSeconds = options.durationInSeconds;
+    this.warnings = options.warnings;
+    this.responses = options.responses;
+    this.providerMetadata = (_a17 = options.providerMetadata) != null ? _a17 : {};
+  }
+};
+
 // core/util/merge-objects.ts
 function mergeObjects(target, source) {
   if (target === void 0 && source === void 0) {
@@ -6556,13 +6677,13 @@ function appendClientMessage({
 import {
   extractMaxToolInvocationStep
 } from "@ai-sdk/ui-utils";
-import { AISDKError as AISDKError18 } from "@ai-sdk/provider";
+import { AISDKError as AISDKError19 } from "@ai-sdk/provider";
 function appendResponseMessages({
   messages,
   responseMessages,
   _internal: { currentDate = () => /* @__PURE__ */ new Date() } = {}
 }) {
-  var _a17, _b, _c, _d;
+  var _a17, _b, _c, _d, _e;
   const clonedMessages = structuredClone(messages);
   for (const message of responseMessages) {
     const role = message.role;
@@ -6639,14 +6760,14 @@ function appendResponseMessages({
                 break;
               case "file":
                 if (part.data instanceof URL) {
-                  throw new AISDKError18({
+                  throw new AISDKError19({
                     name: "InvalidAssistantFileData",
                     message: "File data cannot be a URL"
                   });
                 }
                 parts.push({
                   type: "file",
-                  mimeType: part.mimeType,
+                  mediaType: (_a17 = part.mediaType) != null ? _a17 : part.mimeType,
                   data: convertDataContentToBase64String(part.data)
                 });
                 break;
@@ -6657,12 +6778,12 @@ function appendResponseMessages({
           const maxStep = extractMaxToolInvocationStep(
             lastMessage.toolInvocations
           );
-          (_a17 = lastMessage.parts) != null ? _a17 : lastMessage.parts = [];
+          (_b = lastMessage.parts) != null ? _b : lastMessage.parts = [];
           lastMessage.content = textContent;
           lastMessage.reasoning = reasoningTextContent;
           lastMessage.parts.push(...parts);
           lastMessage.toolInvocations = [
-            ...(_b = lastMessage.toolInvocations) != null ? _b : [],
+            ...(_c = lastMessage.toolInvocations) != null ? _c : [],
             ...getToolInvocations2(maxStep === void 0 ? 0 : maxStep + 1)
           ];
           getToolInvocations2(maxStep === void 0 ? 0 : maxStep + 1).map((call) => ({
@@ -6692,13 +6813,13 @@ function appendResponseMessages({
         break;
       }
       case "tool": {
-        (_c = lastMessage.toolInvocations) != null ? _c : lastMessage.toolInvocations = [];
+        (_d = lastMessage.toolInvocations) != null ? _d : lastMessage.toolInvocations = [];
         if (lastMessage.role !== "assistant") {
           throw new Error(
             `Tool result must follow an assistant message: ${lastMessage.role}`
           );
         }
-        (_d = lastMessage.parts) != null ? _d : lastMessage.parts = [];
+        (_e = lastMessage.parts) != null ? _e : lastMessage.parts = [];
         for (const contentPart of message.content) {
           const toolCall = lastMessage.toolInvocations.find(
             (call) => call.toolCallId === contentPart.toolCallId
@@ -6773,7 +6894,7 @@ function customProvider({
 var experimental_customProvider = customProvider;
 
 // core/registry/no-such-provider-error.ts
-import { AISDKError as AISDKError19, NoSuchModelError as NoSuchModelError3 } from "@ai-sdk/provider";
+import { AISDKError as AISDKError20, NoSuchModelError as NoSuchModelError3 } from "@ai-sdk/provider";
 var name16 = "AI_NoSuchProviderError";
 var marker16 = `vercel.ai.error.${name16}`;
 var symbol16 = Symbol.for(marker16);
@@ -6792,7 +6913,7 @@ var NoSuchProviderError = class extends NoSuchModelError3 {
     this.availableProviders = availableProviders;
   }
   static isInstance(error) {
-    return AISDKError19.hasMarker(error, marker16);
+    return AISDKError20.hasMarker(error, marker16);
   }
 };
 _a16 = symbol16;
@@ -7777,6 +7898,7 @@ export {
   experimental_createProviderRegistry,
   experimental_customProvider,
   generateImage as experimental_generateImage,
+  transcribe as experimental_transcribe,
   experimental_wrapLanguageModel,
   extractReasoningMiddleware,
   formatDataStreamPart3 as formatDataStreamPart,