@fairyhunter13/ai-sdk 6.0.116-fork.1

package/src/prompt/convert-to-language-model-prompt.ts

@@ -0,0 +1,548 @@
+import {
+  LanguageModelV4FilePart,
+  LanguageModelV4Message,
+  LanguageModelV4Prompt,
+  LanguageModelV4TextPart,
+  LanguageModelV4ToolResultOutput,
+} from '@ai-sdk/provider';
+import {
+  CustomPart,
+  DataContent,
+  FilePart,
+  ImagePart,
+  isUrlSupported,
+  ModelMessage,
+  ReasoningFilePart,
+  ReasoningPart,
+  TextPart,
+  ToolCallPart,
+  ToolResultOutput,
+  ToolResultPart,
+} from '@ai-sdk/provider-utils';
+import {
+  detectMediaType,
+  imageMediaTypeSignatures,
+} from '../util/detect-media-type';
+import {
+  createDefaultDownloadFunction,
+  DownloadFunction,
+} from '../util/download/download-function';
+import { convertToLanguageModelV4DataContent } from './data-content';
+import { InvalidMessageRoleError } from './invalid-message-role-error';
+import { StandardizedPrompt } from './standardize-prompt';
+import { asArray } from '../util/as-array';
+import { MissingToolResultsError } from '../error/missing-tool-result-error';
+
+export async function convertToLanguageModelPrompt({
+  prompt,
+  supportedUrls,
+  download = createDefaultDownloadFunction(),
+}: {
+  prompt: StandardizedPrompt;
+  supportedUrls: Record<string, RegExp[]>;
+  download: DownloadFunction | undefined;
+}): Promise<LanguageModelV4Prompt> {
+  const downloadedAssets = await downloadAssets(
+    prompt.messages,
+    download,
+    supportedUrls,
+  );
+
+  const approvalIdToToolCallId = new Map<string, string>();
+  for (const message of prompt.messages) {
+    if (message.role === 'assistant' && Array.isArray(message.content)) {
+      for (const part of message.content) {
+        if (
+          part.type === 'tool-approval-request' &&
+          'approvalId' in part &&
+          'toolCallId' in part
+        ) {
+          approvalIdToToolCallId.set(
+            part.approvalId as string,
+            part.toolCallId as string,
+          );
+        }
+      }
+    }
+  }
+
+  const approvedToolCallIds = new Set<string>();
+  for (const message of prompt.messages) {
+    if (message.role === 'tool') {
+      for (const part of message.content) {
+        if (part.type === 'tool-approval-response') {
+          const toolCallId = approvalIdToToolCallId.get(part.approvalId);
+          if (toolCallId) {
+            approvedToolCallIds.add(toolCallId);
+          }
+        }
+      }
+    }
+  }
+
+  const messages = [
+    ...(prompt.system != null
+      ? typeof prompt.system === 'string'
+        ? [{ role: 'system' as const, content: prompt.system }]
+        : asArray(prompt.system).map(message => ({
+            role: 'system' as const,
+            content: message.content,
+            providerOptions: message.providerOptions,
+          }))
+      : []),
+    ...prompt.messages.map(message =>
+      convertToLanguageModelMessage({ message, downloadedAssets }),
+    ),
+  ];
+
+  // combine consecutive tool messages into a single tool message
+  const combinedMessages = [];
+  for (const message of messages) {
+    if (message.role !== 'tool') {
+      combinedMessages.push(message);
+      continue;
+    }
+
+    const lastCombinedMessage = combinedMessages.at(-1);
+    if (lastCombinedMessage?.role === 'tool') {
+      lastCombinedMessage.content.push(...message.content);
+    } else {
+      combinedMessages.push(message);
+    }
+  }
+
+  const toolCallIds = new Set<string>();
+
+  for (const message of combinedMessages) {
+    switch (message.role) {
+      case 'assistant': {
+        for (const content of message.content) {
+          if (content.type === 'tool-call' && !content.providerExecuted) {
+            toolCallIds.add(content.toolCallId);
+          }
+        }
+        break;
+      }
+      case 'tool': {
+        for (const content of message.content) {
+          if (content.type === 'tool-result') {
+            toolCallIds.delete(content.toolCallId);
+          }
+        }
+        break;
+      }
+      case 'user':
+      case 'system':
+        // remove approved tool calls from the set before checking:
+        for (const id of approvedToolCallIds) {
+          toolCallIds.delete(id);
+        }
+
+        if (toolCallIds.size > 0) {
+          throw new MissingToolResultsError({
+            toolCallIds: Array.from(toolCallIds),
+          });
+        }
+        break;
+    }
+  }
+
+  // remove approved tool calls from the set before checking:
+  for (const id of approvedToolCallIds) {
+    toolCallIds.delete(id);
+  }
+
+  if (toolCallIds.size > 0) {
+    throw new MissingToolResultsError({ toolCallIds: Array.from(toolCallIds) });
+  }
+
+  return combinedMessages.filter(
+    // Filter out empty tool messages (e.g. if they only contained
+    // tool-approval-response parts that were removed).
+    // This prevents sending invalid empty messages to the provider.
+    // Note: provider-executed tool-approval-response parts are preserved.
+    message => message.role !== 'tool' || message.content.length > 0,
+  );
+}
+
+/**
+ * Convert a ModelMessage to a LanguageModelV4Message.
+ *
+ * @param message - The ModelMessage to convert.
+ * @param downloadedAssets - A map of URLs to their downloaded data. Only
+ * available if the model does not support URLs, null otherwise.
+ */
+export function convertToLanguageModelMessage({
+  message,
+  downloadedAssets,
+}: {
+  message: ModelMessage;
+  downloadedAssets: Record<
+    string,
+    { mediaType: string | undefined; data: Uint8Array }
+  >;
+}): LanguageModelV4Message {
+  const role = message.role;
+  switch (role) {
+    case 'system': {
+      return {
+        role: 'system',
+        content: message.content,
+        providerOptions: message.providerOptions,
+      };
+    }
+
+    case 'user': {
+      if (typeof message.content === 'string') {
+        return {
+          role: 'user',
+          content: [{ type: 'text', text: message.content }],
+          providerOptions: message.providerOptions,
+        };
+      }
+
+      return {
+        role: 'user',
+        content: message.content
+          .map(part => convertPartToLanguageModelPart(part, downloadedAssets))
+          // remove empty text parts:
+          .filter(part => part.type !== 'text' || part.text !== ''),
+        providerOptions: message.providerOptions,
+      };
+    }
+
+    case 'assistant': {
+      if (typeof message.content === 'string') {
+        return {
+          role: 'assistant',
+          content: [{ type: 'text', text: message.content }],
+          providerOptions: message.providerOptions,
+        };
+      }
+
+      return {
+        role: 'assistant',
+        content: message.content
+          .filter(
+            // remove empty text parts (no text, and no provider options):
+            part =>
+              part.type !== 'text' ||
+              part.text !== '' ||
+              part.providerOptions != null,
+          )
+          .filter(
+            (
+              part,
+            ): part is
+              | CustomPart
+              | TextPart
+              | FilePart
+              | ReasoningPart
+              | ReasoningFilePart
+              | ToolCallPart
+              | ToolResultPart => part.type !== 'tool-approval-request',
+          )
+          .map(part => {
+            const providerOptions = part.providerOptions;
+
+            switch (part.type) {
+              case 'custom': {
+                return {
+                  type: 'custom' as const,
+                  kind: part.kind,
+                  providerOptions,
+                };
+              }
+              case 'file': {
+                const { data, mediaType } = convertToLanguageModelV4DataContent(
+                  part.data,
+                );
+                return {
+                  type: 'file',
+                  data,
+                  filename: part.filename,
+                  mediaType: mediaType ?? part.mediaType,
+                  providerOptions,
+                };
+              }
+              case 'reasoning': {
+                return {
+                  type: 'reasoning',
+                  text: part.text,
+                  providerOptions,
+                };
+              }
+              case 'reasoning-file': {
+                const { data, mediaType } = convertToLanguageModelV4DataContent(
+                  part.data,
+                );
+                return {
+                  type: 'reasoning-file' as const,
+                  data,
+                  mediaType: mediaType ?? part.mediaType,
+                  providerOptions,
+                };
+              }
+              case 'text': {
+                return {
+                  type: 'text' as const,
+                  text: part.text,
+                  providerOptions,
+                };
+              }
+              case 'tool-call': {
+                return {
+                  type: 'tool-call' as const,
+                  toolCallId: part.toolCallId,
+                  toolName: part.toolName,
+                  input: part.input,
+                  providerExecuted: part.providerExecuted,
+                  providerOptions,
+                };
+              }
+              case 'tool-result': {
+                return {
+                  type: 'tool-result' as const,
+                  toolCallId: part.toolCallId,
+                  toolName: part.toolName,
+                  output: mapToolResultOutput(part.output),
+                  providerOptions,
+                };
+              }
+            }
+          }),
+        providerOptions: message.providerOptions,
+      };
+    }
+
+    case 'tool': {
+      return {
+        role: 'tool',
+        content: message.content
+          .filter(
+            // Only include tool-approval-response for provider-executed tools
+            part =>
+              part.type !== 'tool-approval-response' || part.providerExecuted,
+          )
+          .map(part => {
+            switch (part.type) {
+              case 'tool-result': {
+                return {
+                  type: 'tool-result' as const,
+                  toolCallId: part.toolCallId,
+                  toolName: part.toolName,
+                  output: mapToolResultOutput(part.output),
+                  providerOptions: part.providerOptions,
+                };
+              }
+              case 'tool-approval-response': {
+                return {
+                  type: 'tool-approval-response' as const,
+                  approvalId: part.approvalId,
+                  approved: part.approved,
+                  reason: part.reason,
+                };
+              }
+            }
+          }),
+        providerOptions: message.providerOptions,
+      };
+    }
+
+    default: {
+      const _exhaustiveCheck: never = role;
+      throw new InvalidMessageRoleError({ role: _exhaustiveCheck });
+    }
+  }
+}
+
+/**
+ * Downloads images and files from URLs in the messages.
+ */
+async function downloadAssets(
+  messages: ModelMessage[],
+  download: DownloadFunction,
+  supportedUrls: Record<string, RegExp[]>,
+): Promise<
+  Record<string, { mediaType: string | undefined; data: Uint8Array }>
+> {
+  const plannedDownloads = messages
+    .filter(message => message.role === 'user')
+    .map(message => message.content)
+    .filter((content): content is Array<TextPart | ImagePart | FilePart> =>
+      Array.isArray(content),
+    )
+    .flat()
+    .filter(
+      (part): part is ImagePart | FilePart =>
+        part.type === 'image' || part.type === 'file',
+    )
+    .map(part => {
+      const mediaType =
+        part.mediaType ?? (part.type === 'image' ? 'image/*' : undefined);
+
+      let data = part.type === 'image' ? part.image : part.data;
+      if (typeof data === 'string') {
+        try {
+          data = new URL(data);
+        } catch (ignored) {}
+      }
+
+      return { mediaType, data };
+    })
+
+    .filter(
+      (part): part is { mediaType: string | undefined; data: URL } =>
+        part.data instanceof URL && part.data.protocol !== 'data:',
+    )
+    .map(part => ({
+      url: part.data,
+      isUrlSupportedByModel:
+        part.mediaType != null &&
+        isUrlSupported({
+          url: part.data.toString(),
+          mediaType: part.mediaType,
+          supportedUrls,
+        }),
+    }));
+
+  // download in parallel:
+  const downloadedFiles = await download(plannedDownloads);
+
+  return Object.fromEntries(
+    downloadedFiles
+      .map((file, index) =>
+        file == null
+          ? null
+          : [
+              plannedDownloads[index].url.toString(),
+              { data: file.data, mediaType: file.mediaType },
+            ],
+      )
+      .filter(file => file != null),
+  );
+}
+
+/**
+ * Convert part of a message to a LanguageModelV4Part.
+ *
+ * @param part - The part to convert.
+ * @param downloadedAssets - A map of URLs to their downloaded data. Only
+ * available if the model does not support URLs, null otherwise.
+ * @returns The converted part.
+ */
+function convertPartToLanguageModelPart(
+  part: TextPart | ImagePart | FilePart,
+  downloadedAssets: Record<
+    string,
+    { mediaType: string | undefined; data: Uint8Array }
+  >,
+): LanguageModelV4TextPart | LanguageModelV4FilePart {
+  if (part.type === 'text') {
+    return {
+      type: 'text',
+      text: part.text,
+      providerOptions: part.providerOptions,
+    };
+  }
+
+  let originalData: DataContent | URL;
+  const type = part.type;
+  switch (type) {
+    case 'image':
+      originalData = part.image;
+      break;
+    case 'file':
+      originalData = part.data;
+
+      break;
+    default:
+      throw new Error(`Unsupported part type: ${type}`);
+  }
+
+  const { data: convertedData, mediaType: convertedMediaType } =
+    convertToLanguageModelV4DataContent(originalData);
+
+  let mediaType: string | undefined = convertedMediaType ?? part.mediaType;
+  let data: Uint8Array | string | URL = convertedData; // binary | base64 | url
+
+  // If the content is a URL, we check if it was downloaded:
+  if (data instanceof URL) {
+    const downloadedFile = downloadedAssets[data.toString()];
+    if (downloadedFile) {
+      data = downloadedFile.data;
+      mediaType ??= downloadedFile.mediaType;
+    }
+  }
+
+  // Now that we have the normalized data either as a URL or a Uint8Array,
+  // we can create the LanguageModelV4Part.
+  switch (type) {
+    case 'image': {
+      // When possible, try to detect the media type automatically
+      // to deal with incorrect media type inputs.
+      // When detection fails, use provided media type.
+      if (data instanceof Uint8Array || typeof data === 'string') {
+        mediaType =
+          detectMediaType({ data, signatures: imageMediaTypeSignatures }) ??
+          mediaType;
+      }
+
+      return {
+        type: 'file',
+        mediaType: mediaType ?? 'image/*', // any image
+        filename: undefined,
+        data,
+        providerOptions: part.providerOptions,
+      };
+    }
+
+    case 'file': {
+      // We must have a mediaType for files, if not, throw an error.
+      if (mediaType == null) {
+        throw new Error(`Media type is missing for file part`);
+      }
+
+      return {
+        type: 'file',
+        mediaType,
+        filename: part.filename,
+        data,
+        providerOptions: part.providerOptions,
+      };
+    }
+  }
+}
+
+function mapToolResultOutput(
+  output: ToolResultOutput,
+): LanguageModelV4ToolResultOutput {
+  if (output.type !== 'content') {
+    return output;
+  }
+
+  return {
+    type: 'content',
+    value: output.value.map(item => {
+      if (item.type !== 'media') {
+        return item;
+      }
+
+      // AI SDK 5 tool backwards compatibility:
+      // map media type to image-data or file-data
+      if (item.mediaType.startsWith('image/')) {
+        return {
+          type: 'image-data' as const,
+          data: item.data,
+          mediaType: item.mediaType,
+        };
+      }
+
+      return {
+        type: 'file-data' as const,
+        data: item.data,
+        mediaType: item.mediaType,
+      };
+    }),
+  };
+}

package/src/prompt/create-tool-model-output.ts

@@ -0,0 +1,34 @@
+import { getErrorMessage, JSONValue } from '@ai-sdk/provider';
+import { Tool, ToolResultOutput } from '@ai-sdk/provider-utils';
+
+export async function createToolModelOutput({
+  toolCallId,
+  input,
+  output,
+  tool,
+  errorMode,
+}: {
+  toolCallId: string;
+  input: unknown;
+  output: unknown;
+  tool: Tool | undefined;
+  errorMode: 'none' | 'text' | 'json';
+}): Promise<ToolResultOutput> {
+  if (errorMode === 'text') {
+    return { type: 'error-text', value: getErrorMessage(output) };
+  } else if (errorMode === 'json') {
+    return { type: 'error-json', value: toJSONValue(output) };
+  }
+
+  if (tool?.toModelOutput) {
+    return await tool.toModelOutput({ toolCallId, input, output });
+  }
+
+  return typeof output === 'string'
+    ? { type: 'text', value: output }
+    : { type: 'json', value: toJSONValue(output) };
+}
+
+function toJSONValue(value: unknown): JSONValue {
+  return value === undefined ? null : (value as JSONValue);
+}

package/src/prompt/data-content.ts

@@ -0,0 +1,134 @@
+import { AISDKError, LanguageModelV4DataContent } from '@ai-sdk/provider';
+import {
+  convertBase64ToUint8Array,
+  convertUint8ArrayToBase64,
+  DataContent,
+} from '@ai-sdk/provider-utils';
+import { z } from 'zod/v4';
+import { InvalidDataContentError } from './invalid-data-content-error';
+import { splitDataUrl } from './split-data-url';
+
+/**
+ * @internal
+ */
+export const dataContentSchema: z.ZodType<DataContent> = z.union([
+  z.string(),
+  z.instanceof(Uint8Array),
+  z.instanceof(ArrayBuffer),
+  z.custom<Buffer>(
+    // Buffer might not be available in some environments such as CloudFlare:
+    (value: unknown): value is Buffer =>
+      globalThis.Buffer?.isBuffer(value) ?? false,
+    { message: 'Must be a Buffer' },
+  ),
+]);
+
+export function convertToLanguageModelV4DataContent(
+  content: DataContent | URL,
+): {
+  data: LanguageModelV4DataContent;
+  mediaType: string | undefined;
+} {
+  // Buffer & Uint8Array:
+  if (content instanceof Uint8Array) {
+    return { data: content, mediaType: undefined };
+  }
+
+  // ArrayBuffer needs conversion to Uint8Array (lightweight):
+  if (content instanceof ArrayBuffer) {
+    return { data: new Uint8Array(content), mediaType: undefined };
+  }
+
+  // Attempt to create a URL from the data. If it fails, we can assume the data
+  // is not a URL and likely some other sort of data.
+  if (typeof content === 'string') {
+    try {
+      content = new URL(content);
+    } catch (error) {
+      // ignored
+    }
+  }
+
+  // Extract data from data URL:
+  if (content instanceof URL && content.protocol === 'data:') {
+    const { mediaType: dataUrlMediaType, base64Content } = splitDataUrl(
+      content.toString(),
+    );
+
+    if (dataUrlMediaType == null || base64Content == null) {
+      throw new AISDKError({
+        name: 'InvalidDataContentError',
+        message: `Invalid data URL format in content ${content.toString()}`,
+      });
+    }
+
+    return { data: base64Content, mediaType: dataUrlMediaType };
+  }
+
+  return { data: content, mediaType: undefined };
+}
+
+/**
+ * Converts data content to a base64-encoded string.
+ *
+ * @param content - Data content to convert.
+ * @returns Base64-encoded string.
+ */
+export function convertDataContentToBase64String(content: DataContent): string {
+  if (typeof content === 'string') {
+    return content;
+  }
+
+  if (content instanceof ArrayBuffer) {
+    return convertUint8ArrayToBase64(new Uint8Array(content));
+  }
+
+  return convertUint8ArrayToBase64(content);
+}
+
+/**
+ * Converts data content to a Uint8Array.
+ *
+ * @param content - Data content to convert.
+ * @returns Uint8Array.
+ */
+export function convertDataContentToUint8Array(
+  content: DataContent,
+): Uint8Array {
+  if (content instanceof Uint8Array) {
+    return content;
+  }
+
+  if (typeof content === 'string') {
+    try {
+      return convertBase64ToUint8Array(content);
+    } catch (error) {
+      throw new InvalidDataContentError({
+        message:
+          'Invalid data content. Content string is not a base64-encoded media.',
+        content,
+        cause: error,
+      });
+    }
+  }
+
+  if (content instanceof ArrayBuffer) {
+    return new Uint8Array(content);
+  }
+
+  throw new InvalidDataContentError({ content });
+}
+
+/**
+ * Converts a Uint8Array to a string of text.
+ *
+ * @param uint8Array - The Uint8Array to convert.
+ * @returns The converted string.
+ */
+export function convertUint8ArrayToText(uint8Array: Uint8Array): string {
+  try {
+    return new TextDecoder().decode(uint8Array);
+  } catch (error) {
+    throw new Error('Error decoding Uint8Array to text');
+  }
+}