@tanstack/ai 0.3.0 → 0.4.0

package/src/activities/chat/messages.ts

@@ -1,16 +1,34 @@
 import type {
+  AudioPart,
   ContentPart,
+  DocumentPart,
+  ImagePart,
   MessagePart,
   ModelMessage,
   TextPart,
   ToolCallPart,
   ToolResultPart,
   UIMessage,
+  VideoPart,
 } from '../../types'
 // ===========================
 // Message Converters
 // ===========================
 
+/**
+ * Helper to check if a part is a multimodal content part (image, audio, video, document)
+ */
+function isMultimodalPart(
+  part: MessagePart,
+): part is ImagePart | AudioPart | VideoPart | DocumentPart {
+  return (
+    part.type === 'image' ||
+    part.type === 'audio' ||
+    part.type === 'video' ||
+    part.type === 'document'
+  )
+}
+
 /**
  * Helper to extract text content from string or ContentPart array
  * For multimodal content, this extracts only the text parts
@@ -52,7 +70,8 @@ export function convertMessagesToModelMessages(
  * Convert a UIMessage to ModelMessage(s)
  *
  * This conversion handles the parts-based structure:
- * - Text parts → content field
+ * - Text parts → content field (string or as part of ContentPart array)
+ * - Multimodal parts (image, audio, video, document) → ContentPart array
  * - ToolCall parts → toolCalls array
  * - ToolResult parts → separate role="tool" messages
  *
@@ -72,12 +91,17 @@ export function uiMessageToModelMessages(
   // Separate parts by type
   // Note: thinking parts are UI-only and not included in ModelMessages
   const textParts: Array<TextPart> = []
+  const multimodalParts: Array<
+    ImagePart | AudioPart | VideoPart | DocumentPart
+  > = []
   const toolCallParts: Array<ToolCallPart> = []
   const toolResultParts: Array<ToolResultPart> = []
 
   for (const part of uiMessage.parts) {
     if (part.type === 'text') {
       textParts.push(part)
+    } else if (isMultimodalPart(part)) {
+      multimodalParts.push(part)
     } else if (part.type === 'tool-call') {
       toolCallParts.push(part)
     } else if (part.type === 'tool-result') {
@@ -86,8 +110,26 @@ export function uiMessageToModelMessages(
     // thinking parts are skipped - they're UI-only
   }
 
-  // Build the main message (user or assistant)
-  const content = textParts.map((p) => p.content).join('') || null
+  // Build the content field
+  // If we have multimodal parts, use ContentPart array format
+  // Otherwise, use simple string format for backward compatibility
+  let content: string | null | Array<ContentPart>
+  if (multimodalParts.length > 0) {
+    // Build ContentPart array preserving the order of text and multimodal parts
+    const contentParts: Array<ContentPart> = []
+    for (const part of uiMessage.parts) {
+      if (part.type === 'text') {
+        contentParts.push(part)
+      } else if (isMultimodalPart(part)) {
+        contentParts.push(part)
+      }
+    }
+    content = contentParts
+  } else {
+    // Simple string content for text-only messages
+    content = textParts.map((p) => p.content).join('') || null
+  }
+
   const toolCalls =
     toolCallParts.length > 0
       ? toolCallParts
@@ -108,7 +150,9 @@ export function uiMessageToModelMessages(
       : undefined
 
   // Create the main message
-  if (uiMessage.role !== 'assistant' || content || !toolCalls) {
+  // For multimodal content, we always create a message even if content is an empty array
+  const hasContent = Array.isArray(content) ? true : content !== null
+  if (uiMessage.role !== 'assistant' || hasContent || !toolCalls) {
     messageList.push({
       role: uiMessage.role,
       content,
@@ -123,7 +167,11 @@ export function uiMessageToModelMessages(
     })
   }
 
-  // Add tool result messages (only completed ones)
+  // Add tool result messages for completed tool calls
+  // This includes:
+  // 1. Explicit tool-result parts (from server tools)
+  // 2. Client tool calls with output set
+  // 3. Approval-responded tool calls (approval result)
   for (const toolResultPart of toolResultParts) {
     if (
       toolResultPart.state === 'complete' ||
@@ -137,6 +185,41 @@ export function uiMessageToModelMessages(
     }
   }
 
+  // Add tool result messages for client tool results (tools with output)
+  // and approval responses (so iteration tracking works correctly)
+  for (const toolCallPart of toolCallParts) {
+    // Client tool with output - add as tool result
+    if (toolCallPart.output !== undefined && !toolCallPart.approval) {
+      messageList.push({
+        role: 'tool',
+        content: JSON.stringify(toolCallPart.output),
+        toolCallId: toolCallPart.id,
+      })
+    }
+
+    // Approval response - add as tool result for iteration tracking
+    // For APPROVED: includes pendingExecution marker so the tool still executes
+    // For DENIED: just marks the tool as complete (no execution needed)
+    if (
+      toolCallPart.state === 'approval-responded' &&
+      toolCallPart.approval?.approved !== undefined
+    ) {
+      const approved = toolCallPart.approval.approved
+      messageList.push({
+        role: 'tool',
+        content: JSON.stringify({
+          approved,
+          // Mark approved tools as pending execution - they still need to run
+          ...(approved && { pendingExecution: true }),
+          message: approved
+            ? 'User approved this action'
+            : 'User denied this action',
+        }),
+        toolCallId: toolCallPart.id,
+      })
+    }
+  }
+
   return messageList
 }
 

package/src/activities/chat/stream/processor.ts

@@ -35,10 +35,13 @@ import type {
   ToolResultState,
 } from './types'
 import type {
+  ContentPart,
+  MessagePart,
   ModelMessage,
   StreamChunk,
   ToolCall,
   ToolCallPart,
+  ToolResultPart,
   UIMessage,
 } from '../../../types'
 
@@ -164,13 +167,42 @@ export class StreamProcessor {
   }
 
   /**
-   * Add a user message to the conversation
-   */
-  addUserMessage(content: string): UIMessage {
+   * Add a user message to the conversation.
+   * Supports both simple string content and multimodal content arrays.
+   *
+   * @param content - The message content (string or array of content parts)
+   * @param id - Optional custom message ID (generated if not provided)
+   * @returns The created UIMessage
+   *
+   * @example
+   * ```ts
+   * // Simple text message
+   * processor.addUserMessage('Hello!')
+   *
+   * // Multimodal message with image
+   * processor.addUserMessage([
+   *   { type: 'text', content: 'What is in this image?' },
+   *   { type: 'image', source: { type: 'url', value: 'https://example.com/photo.jpg' } }
+   * ])
+   *
+   * // With custom ID
+   * processor.addUserMessage('Hello!', 'custom-id-123')
+   * ```
+   */
+  addUserMessage(content: string | Array<ContentPart>, id?: string): UIMessage {
+    // Convert content to message parts
+    const parts: Array<MessagePart> =
+      typeof content === 'string'
+        ? [{ type: 'text', content }]
+        : content.map((part) => {
+            // ContentPart types (text, image, audio, video, document) are compatible with MessagePart
+            return part as MessagePart
+          })
+
     const userMessage: UIMessage = {
-      id: generateMessageId(),
+      id: id ?? generateMessageId(),
       role: 'user',
-      parts: [{ type: 'text', content }],
+      parts,
       createdAt: new Date(),
     }
 
@@ -296,11 +328,23 @@ export class StreamProcessor {
 
     if (toolParts.length === 0) return true
 
+    // Get tool result parts to check for server tool completion
+    const toolResultIds = new Set(
+      lastAssistant.parts
+        .filter((p): p is ToolResultPart => p.type === 'tool-result')
+        .map((p) => p.toolCallId),
+    )
+
     // All tool calls must be in a terminal state
+    // A tool call is complete if:
+    // 1. It was approved/denied (approval-responded state)
+    // 2. It has an output field set (client tool completed via addToolResult)
+    // 3. It has a corresponding tool-result part (server tool completed)
     return toolParts.every(
       (part) =>
         part.state === 'approval-responded' ||
-        (part.output !== undefined && !part.approval),
+        (part.output !== undefined && !part.approval) ||
+        toolResultIds.has(part.id),
     )
   }
 

package/src/index.ts

@@ -73,6 +73,9 @@ export {
 // All types
 export * from './types'
 
+// Utility functions
+export { detectImageMimeType } from './utils'
+
 // Event client + event types
 export * from './event-client'
 

package/src/types.ts

@@ -108,24 +108,52 @@ export interface ToolCall {
 export type Modality = 'text' | 'image' | 'audio' | 'video' | 'document'
 
 /**
- * Source specification for multimodal content.
- * Supports both inline data (base64) and URL-based content.
+ * Source specification for inline data content (base64).
+ * Requires a mimeType to ensure providers receive proper content type information.
+ */
+export interface ContentPartDataSource {
+  /**
+   * Indicates this is inline data content.
+   */
+  type: 'data'
+  /**
+   * The base64-encoded content value.
+   */
+  value: string
+  /**
+   * The MIME type of the content (e.g., 'image/png', 'audio/wav').
+   * Required for data sources to ensure proper handling by providers.
+   */
+  mimeType: string
+}
+
+/**
+ * Source specification for URL-based content.
+ * mimeType is optional as it can often be inferred from the URL or response headers.
  */
-export interface ContentPartSource {
+export interface ContentPartUrlSource {
   /**
-   * The type of source:
-   * - 'data': Inline data (typically base64 encoded)
-   * - 'url': URL reference to the content
+   * Indicates this is URL-referenced content.
    */
-  type: 'data' | 'url'
+  type: 'url'
   /**
-   * The actual content value:
-   * - For 'data': base64-encoded string
-   * - For 'url': HTTP(S) URL or data URI
+   * HTTP(S) URL or data URI pointing to the content.
    */
   value: string
+  /**
+   * Optional MIME type hint for cases where providers can't infer it from the URL.
+   */
+  mimeType?: string
 }
 
+/**
+ * Source specification for multimodal content.
+ * Discriminated union supporting both inline data (base64) and URL-based content.
+ * - For 'data' sources: mimeType is required
+ * - For 'url' sources: mimeType is optional
+ */
+export type ContentPartSource = ContentPartDataSource | ContentPartUrlSource
+
 /**
  * Image content part for multimodal messages.
  * @template TMetadata - Provider-specific metadata type (e.g., OpenAI's detail level)
@@ -282,6 +310,10 @@ export interface ThinkingPart {
 
 export type MessagePart =
   | TextPart
+  | ImagePart
+  | AudioPart
+  | VideoPart
+  | DocumentPart
   | ToolCallPart
   | ToolResultPart
   | ThinkingPart

package/src/utils.ts

@@ -0,0 +1,41 @@
+/**
+ * Detect image mime type from base64 data using magic bytes.
+ * Returns undefined if the format cannot be detected.
+ *
+ * This function analyzes the first few bytes of base64-encoded image data
+ * to determine the image format based on file signature (magic bytes).
+ *
+ * @param base64Data - The base64-encoded image data
+ * @returns The detected mime type, or undefined if unrecognized
+ *
+ * @example
+ * ```ts
+ * const mimeType = detectImageMimeType(imageBase64)
+ * // Returns 'image/jpeg', 'image/png', 'image/gif', 'image/webp', or undefined
+ * ```
+ */
+export function detectImageMimeType(
+  base64Data: string,
+): 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | undefined {
+  // Get first few bytes (base64 encoded)
+  const prefix = base64Data.substring(0, 20)
+
+  // JPEG: starts with /9j/ (FFD8FF in base64)
+  if (prefix.startsWith('/9j/')) {
+    return 'image/jpeg'
+  }
+  // PNG: starts with iVBORw0KGgo (89504E47 in base64)
+  if (prefix.startsWith('iVBORw0KGgo')) {
+    return 'image/png'
+  }
+  // GIF: starts with R0lGOD (474946 in base64)
+  if (prefix.startsWith('R0lGOD')) {
+    return 'image/gif'
+  }
+  // WebP: starts with UklGR (52494646 in base64, followed by WEBP)
+  if (prefix.startsWith('UklGR')) {
+    return 'image/webp'
+  }
+
+  return undefined
+}