@ai-sdk/cohere 3.0.32 → 3.0.34

package/src/convert-to-cohere-chat-prompt.ts

@@ -1,20 +1,26 @@
 import {
   UnsupportedFunctionalityError,
-  type SharedV3Warning,
+  type LanguageModelV3FilePart,
   type LanguageModelV3Prompt,
+  type SharedV3Warning,
 } from '@ai-sdk/provider';
+import { convertToBase64, parseProviderOptions } from '@ai-sdk/provider-utils';
+import { cohereImagePartProviderOptions } from './cohere-chat-options';
 import type {
   CohereAssistantMessage,
   CohereChatPrompt,
+  CohereUserMessageContent,
 } from './cohere-chat-prompt';
 
-export function convertToCohereChatPrompt(prompt: LanguageModelV3Prompt): {
+export async function convertToCohereChatPrompt(
+  prompt: LanguageModelV3Prompt,
+): Promise<{
   messages: CohereChatPrompt;
   documents: Array<{
     data: { text: string; title?: string };
   }>;
   warnings: SharedV3Warning[];
-} {
+}> {
   const messages: CohereChatPrompt = [];
   const documents: Array<{ data: { text: string; title?: string } }> = [];
   const warnings: SharedV3Warning[] = [];
@@ -27,58 +33,86 @@ export function convertToCohereChatPrompt(prompt: LanguageModelV3Prompt): {
       }
 
       case 'user': {
-        messages.push({
-          role: 'user',
-          content: content
-            .map(part => {
-              switch (part.type) {
-                case 'text': {
-                  return part.text;
-                }
-                case 'file': {
-                  // Extract documents for RAG
-                  let textContent: string;
-
-                  if (typeof part.data === 'string') {
-                    // Base64 or text data
-                    textContent = part.data;
-                  } else if (part.data instanceof Uint8Array) {
-                    // Check if the media type is supported for text extraction
-                    if (
-                      !(
-                        part.mediaType?.startsWith('text/') ||
-                        part.mediaType === 'application/json'
-                      )
-                    ) {
-                      throw new UnsupportedFunctionalityError({
-                        functionality: `document media type: ${part.mediaType}`,
-                        message: `Media type '${part.mediaType}' is not supported. Supported media types are: text/* and application/json.`,
-                      });
-                    }
-                    textContent = new TextDecoder().decode(part.data);
-                  } else {
-                    throw new UnsupportedFunctionalityError({
-                      functionality: 'File URL data',
-                      message:
-                        'URLs should be downloaded by the AI SDK and not reach this point. This indicates a configuration issue.',
-                    });
-                  }
-
-                  documents.push({
-                    data: {
-                      text: textContent,
-                      title: part.filename,
-                    },
-                  });
+        const userContentParts: Array<CohereUserMessageContent> = [];
+        let hasImage = false;
 
-                  // Files are handled separately via the documents parameter
-                  // Return empty string to not include file content in message text
-                  return '';
+        for (const part of content) {
+          switch (part.type) {
+            case 'text': {
+              if (part.text.length > 0) {
+                userContentParts.push({ type: 'text', text: part.text });
+              }
+              break;
+            }
+            case 'file': {
+              if (isImageMediaType(part.mediaType)) {
+                hasImage = true;
+                const url = buildImageUrl({ part });
+                const cohereOptions =
+                  (await parseProviderOptions({
+                    provider: 'cohere',
+                    providerOptions: part.providerOptions,
+                    schema: cohereImagePartProviderOptions,
+                  })) ?? {};
+
+                userContentParts.push({
+                  type: 'image_url',
+                  image_url: {
+                    url,
+                    ...(cohereOptions.detail
+                      ? { detail: cohereOptions.detail }
+                      : {}),
+                  },
+                });
+                break;
+              }
+
+              let textContent: string;
+
+              if (typeof part.data === 'string') {
+                textContent = part.data;
+              } else if (part.data instanceof Uint8Array) {
+                if (
+                  !(
+                    part.mediaType?.startsWith('text/') ||
+                    part.mediaType === 'application/json'
+                  )
+                ) {
+                  throw new UnsupportedFunctionalityError({
+                    functionality: `document media type: ${part.mediaType}`,
+                    message: `Media type '${part.mediaType}' is not supported. Supported media types are: text/* and application/json.`,
+                  });
                 }
+                textContent = new TextDecoder().decode(part.data);
+              } else {
+                throw new UnsupportedFunctionalityError({
+                  functionality: 'File URL data',
+                  message:
+                    'URLs should be downloaded by the AI SDK and not reach this point. This indicates a configuration issue.',
+                });
               }
-            })
-            .join(''),
-        });
+
+              documents.push({
+                data: {
+                  text: textContent,
+                  title: part.filename,
+                },
+              });
+              break;
+            }
+          }
+        }
+
+        if (hasImage) {
+          messages.push({ role: 'user', content: userContentParts });
+        } else {
+          messages.push({
+            role: 'user',
+            content: userContentParts
+              .map(p => (p.type === 'text' ? p.text : ''))
+              .join(''),
+          });
+        }
         break;
       }
 
@@ -157,3 +191,20 @@ export function convertToCohereChatPrompt(prompt: LanguageModelV3Prompt): {
 
   return { messages, documents, warnings };
 }
+
+function isImageMediaType(mediaType: string | undefined): boolean {
+  return mediaType === 'image' || mediaType?.startsWith('image/') === true;
+}
+
+function buildImageUrl({ part }: { part: LanguageModelV3FilePart }): string {
+  if (part.data instanceof URL) {
+    return part.data.toString();
+  }
+
+  const mediaType =
+    part.mediaType === 'image' || part.mediaType === 'image/*'
+      ? 'image/jpeg'
+      : part.mediaType;
+
+  return `data:${mediaType};base64,${convertToBase64(part.data)}`;
+}