clawport-ui 0.1.0

package/lib/multimodal.test.ts

@@ -0,0 +1,219 @@
+import { describe, it, expect } from 'vitest'
+import { buildApiContent } from './multimodal'
+import type { Message, MediaAttachment } from './conversations'
+
+function msg(overrides: Partial<Message> = {}): Message {
+  return {
+    id: 'msg-1',
+    role: 'user',
+    content: 'hello',
+    timestamp: 1000,
+    ...overrides,
+  }
+}
+
+function imageAttachment(overrides: Partial<MediaAttachment> = {}): MediaAttachment {
+  return {
+    type: 'image',
+    url: 'data:image/png;base64,iVBORw0KGgoAAAA',
+    name: 'photo.png',
+    mimeType: 'image/png',
+    ...overrides,
+  }
+}
+
+function fileAttachment(overrides: Partial<MediaAttachment> = {}): MediaAttachment {
+  return {
+    type: 'file',
+    url: 'data:application/pdf;base64,JVBERi0',
+    name: 'report.pdf',
+    mimeType: 'application/pdf',
+    size: 245000,
+    ...overrides,
+  }
+}
+
+function audioAttachment(overrides: Partial<MediaAttachment> = {}): MediaAttachment {
+  return {
+    type: 'audio',
+    url: 'data:audio/webm;base64,GkXfo59C',
+    name: 'Voice message',
+    mimeType: 'audio/webm',
+    duration: 5,
+    waveform: Array(50).fill(0.3),
+    ...overrides,
+  }
+}
+
+// --- plain text messages ---
+
+describe('buildApiContent — plain text', () => {
+  it('returns string content when no media is attached', () => {
+    const result = buildApiContent(msg({ content: 'just text' }))
+    expect(result).toBe('just text')
+  })
+
+  it('returns string content when media array is empty', () => {
+    const result = buildApiContent(msg({ content: 'text', media: [] }))
+    expect(result).toBe('text')
+  })
+
+  it('returns empty string for empty content with no media', () => {
+    const result = buildApiContent(msg({ content: '' }))
+    expect(result).toBe('')
+  })
+})
+
+// --- image attachments ---
+
+describe('buildApiContent — images', () => {
+  it('converts a message with one image to content parts array', () => {
+    const result = buildApiContent(msg({
+      content: 'what is this?',
+      media: [imageAttachment()],
+    }))
+    expect(Array.isArray(result)).toBe(true)
+    const parts = result as Array<{ type: string }>
+    expect(parts).toHaveLength(2)
+    expect(parts[0]).toEqual({ type: 'text', text: 'what is this?' })
+    expect(parts[1]).toEqual({ type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgoAAAA' } })
+  })
+
+  it('includes multiple images as separate image_url parts', () => {
+    const result = buildApiContent(msg({
+      content: 'compare these',
+      media: [
+        imageAttachment({ url: 'data:image/png;base64,AAA' }),
+        imageAttachment({ url: 'data:image/png;base64,BBB' }),
+      ],
+    }))
+    const parts = result as Array<{ type: string }>
+    expect(parts).toHaveLength(3) // 1 text + 2 images
+    expect(parts.filter(p => p.type === 'image_url')).toHaveLength(2)
+  })
+
+  it('handles image-only message with no text content', () => {
+    const result = buildApiContent(msg({
+      content: '',
+      media: [imageAttachment()],
+    }))
+    const parts = result as Array<{ type: string }>
+    // No text part since content is empty, just the image
+    expect(parts).toHaveLength(1)
+    expect(parts[0].type).toBe('image_url')
+  })
+})
+
+// --- file attachments ---
+
+describe('buildApiContent — files', () => {
+  it('adds a text label for binary files (PDF)', () => {
+    const result = buildApiContent(msg({
+      content: 'here is a doc',
+      media: [fileAttachment()],
+    }))
+    const parts = result as Array<{ type: string; text?: string }>
+    expect(parts).toHaveLength(2)
+    expect(parts[1].type).toBe('text')
+    expect(parts[1].text).toContain('report.pdf')
+    expect(parts[1].text).toContain('239 KB') // 245000 / 1024 ≈ 239
+  })
+
+  it('inlines text file content from base64 data URL', () => {
+    const textContent = 'Hello, world!'
+    const base64 = btoa(textContent)
+    const result = buildApiContent(msg({
+      content: 'check this file',
+      media: [fileAttachment({
+        name: 'notes.txt',
+        mimeType: 'text/plain',
+        url: `data:text/plain;base64,${base64}`,
+      })],
+    }))
+    const parts = result as Array<{ type: string; text?: string }>
+    expect(parts).toHaveLength(2)
+    expect(parts[1].text).toContain('Hello, world!')
+    expect(parts[1].text).toContain('Contents of notes.txt')
+  })
+
+  it('inlines JSON file based on extension even without text/ mimeType', () => {
+    const jsonContent = '{"key": "value"}'
+    const base64 = btoa(jsonContent)
+    const result = buildApiContent(msg({
+      content: '',
+      media: [fileAttachment({
+        name: 'config.json',
+        mimeType: 'application/json',
+        url: `data:application/json;base64,${base64}`,
+      })],
+    }))
+    const parts = result as Array<{ type: string; text?: string }>
+    expect(parts).toHaveLength(1)
+    expect(parts[0].text).toContain('"key": "value"')
+  })
+
+  it('falls back to label when file has no name', () => {
+    const result = buildApiContent(msg({
+      content: '',
+      media: [fileAttachment({ name: undefined })],
+    }))
+    const parts = result as Array<{ type: string; text?: string }>
+    expect(parts[0].text).toContain('unknown')
+  })
+})
+
+// --- audio attachments ---
+
+describe('buildApiContent — audio', () => {
+  it('returns plain string for audio-only message (transcript from Whisper)', () => {
+    const result = buildApiContent(msg({
+      content: 'Hello this is my transcribed message',
+      media: [audioAttachment()],
+    }))
+    // Audio is skipped and no other non-text parts exist, so return plain string
+    // to avoid wrapping in ContentPart[] which the gateway may not handle
+    expect(result).toBe('Hello this is my transcribed message')
+  })
+
+  it('returns plain string when audio is the only attachment', () => {
+    const result = buildApiContent(msg({
+      content: 'transcribed words',
+      media: [audioAttachment()],
+    }))
+    expect(typeof result).toBe('string')
+    expect(result).toBe('transcribed words')
+  })
+})
+
+// --- mixed media ---
+
+describe('buildApiContent — mixed media', () => {
+  it('handles image + file together', () => {
+    const result = buildApiContent(msg({
+      content: 'look at both',
+      media: [
+        imageAttachment(),
+        fileAttachment(),
+      ],
+    }))
+    const parts = result as Array<{ type: string }>
+    expect(parts).toHaveLength(3) // text + image + file label
+    expect(parts[0].type).toBe('text')
+    expect(parts[1].type).toBe('image_url')
+    expect(parts[2].type).toBe('text')
+  })
+
+  it('handles image + audio (audio skipped, text preserved)', () => {
+    const result = buildApiContent(msg({
+      content: 'here is what I see',
+      media: [
+        imageAttachment(),
+        audioAttachment(),
+      ],
+    }))
+    const parts = result as Array<{ type: string }>
+    expect(parts).toHaveLength(2) // text + image (audio skipped)
+    expect(parts[0].type).toBe('text')
+    expect(parts[1].type).toBe('image_url')
+  })
+})

package/lib/multimodal.ts

@@ -0,0 +1,68 @@
+import type { Message, MediaAttachment } from './conversations'
+import type { ContentPart, MessageContent } from './validation'
+
+/**
+ * Convert a Message (with optional MediaAttachments) into the API content format.
+ * - No media → plain string (backward compatible)
+ * - Images → ContentPart[] with image_url entries (OpenAI vision format)
+ * - Text files → content inlined as text parts
+ * - Binary files → descriptive label
+ * - Audio → skipped (transcript is already in msg.content from Whisper)
+ */
+export function buildApiContent(msg: Message): MessageContent {
+  const media = msg.media
+  if (!media || media.length === 0) return msg.content
+
+  const parts: ContentPart[] = []
+  let attachmentAdded = false
+
+  if (msg.content) {
+    parts.push({ type: 'text', text: msg.content })
+  }
+
+  for (const attachment of media) {
+    if (attachment.type === 'image') {
+      parts.push({ type: 'image_url', image_url: { url: attachment.url } })
+      attachmentAdded = true
+    } else if (attachment.type === 'file') {
+      const label = attachment.name || 'unknown'
+      const sizeNote = attachment.size ? ` (${Math.round(attachment.size / 1024)} KB)` : ''
+      // Attempt to inline text file content from base64 data URL
+      const inlined = tryExtractText(attachment)
+      if (inlined) {
+        parts.push({ type: 'text', text: `--- Contents of ${label} ---\n${inlined}\n--- End of file ---` })
+      } else {
+        parts.push({ type: 'text', text: `[Attached file: ${label}${sizeNote}]` })
+      }
+      attachmentAdded = true
+    }
+    // Audio: transcript already in msg.content via Whisper — skip binary
+  }
+
+  // If no attachment actually contributed to parts (e.g., audio-only message
+  // where the transcript is already in msg.content), return a plain string
+  // so the gateway doesn't receive an unnecessary ContentPart[] wrapper.
+  if (!attachmentAdded) return msg.content
+
+  return parts.length > 0 ? parts : msg.content
+}
+
+const TEXT_EXTENSIONS = ['txt', 'csv', 'json', 'md', 'log', 'xml', 'yaml', 'yml', 'toml']
+
+function isTextFile(att: MediaAttachment): boolean {
+  if (att.mimeType?.startsWith('text/')) return true
+  const ext = att.name?.split('.').pop()?.toLowerCase() || ''
+  return TEXT_EXTENSIONS.includes(ext)
+}
+
+function tryExtractText(att: MediaAttachment): string | null {
+  if (!isTextFile(att)) return null
+  if (!att.url.startsWith('data:')) return null
+  try {
+    const base64 = att.url.split(',')[1]
+    if (!base64) return null
+    return atob(base64)
+  } catch {
+    return null
+  }
+}

package/lib/pipeline.integration.test.ts

@@ -0,0 +1,343 @@
+/**
+ * Integration tests for the full image pipeline:
+ * buildApiContent → JSON serialize → parse → validateChatMessages → OpenAI message mapping
+ *
+ * These tests verify that image data survives the complete journey from
+ * user attachment to the format sent to the OpenAI SDK.
+ */
+import { describe, it, expect } from 'vitest'
+import { buildApiContent } from './multimodal'
+import { validateChatMessages, validateMessages } from './validation'
+import type { Message, MediaAttachment } from './conversations'
+import type { ContentPart, MessageContent } from './validation'
+
+// --- Helpers ---
+
+function userMessage(overrides: Partial<Message> = {}): Message {
+  return {
+    id: 'msg-1',
+    role: 'user',
+    content: 'what is this?',
+    timestamp: Date.now(),
+    ...overrides,
+  }
+}
+
+function assistantMessage(overrides: Partial<Message> = {}): Message {
+  return {
+    id: 'msg-0',
+    role: 'assistant',
+    content: 'Hello, how can I help?',
+    timestamp: Date.now(),
+    ...overrides,
+  }
+}
+
+function imageAttachment(overrides: Partial<MediaAttachment> = {}): MediaAttachment {
+  return {
+    type: 'image',
+    url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk',
+    name: 'screenshot.png',
+    mimeType: 'image/png',
+    ...overrides,
+  }
+}
+
+// --- Full pipeline simulation ---
+
+/**
+ * Simulates the exact pipeline:
+ * 1. ConversationView builds apiMessages via buildApiContent
+ * 2. JSON.stringify for fetch body
+ * 3. Server parses with request.json()
+ * 4. Server validates with validateChatMessages
+ * 5. Server maps messages for OpenAI SDK
+ */
+function simulateFullPipeline(conversationMessages: Message[]) {
+  // Step 1: Client-side — buildApiContent for each message (ConversationView line 368-371)
+  const apiMessages = conversationMessages.map(m => ({
+    role: m.role,
+    content: buildApiContent(m),
+  }))
+
+  // Step 2: Client-side — JSON.stringify for fetch body (ConversationView line 377)
+  const jsonBody = JSON.stringify({ messages: apiMessages })
+
+  // Step 3: Server-side — request.json() (route.ts line 29)
+  const parsedBody = JSON.parse(jsonBody)
+
+  // Step 4: Server-side — validateChatMessages (route.ts line 37)
+  const result = validateChatMessages(parsedBody)
+
+  if (!result.ok) {
+    throw new Error(`Validation failed: ${result.error}`)
+  }
+
+  // Step 5: Server-side — map for OpenAI SDK (route.ts line 55-58)
+  const systemPrompt = 'You are a helpful assistant.'
+  const openaiMessages = [
+    { role: 'system' as const, content: systemPrompt },
+    ...result.messages.map(m => ({ role: m.role, content: m.content })),
+  ]
+
+  return { apiMessages, jsonBody, parsedBody, validatedMessages: result.messages, openaiMessages }
+}
+
+// =============================================================
+// Pipeline integration tests
+// =============================================================
+
+describe('full image pipeline — end-to-end', () => {
+  it('preserves image_url data through the entire pipeline', () => {
+    const img = imageAttachment()
+    const msgs: Message[] = [
+      assistantMessage(),
+      userMessage({ media: [img] }),
+    ]
+
+    const { openaiMessages } = simulateFullPipeline(msgs)
+
+    // The last message should be the user message with multimodal content
+    const userMsg = openaiMessages[openaiMessages.length - 1]
+    expect(userMsg.role).toBe('user')
+
+    // content must be an array (multimodal), NOT a string
+    expect(Array.isArray(userMsg.content)).toBe(true)
+
+    const parts = userMsg.content as ContentPart[]
+    // Must have text + image_url parts
+    const imageParts = parts.filter(p => p.type === 'image_url')
+    expect(imageParts.length).toBeGreaterThanOrEqual(1)
+
+    // The image_url must contain the actual data URL, not "[object Object]" or empty
+    const imgPart = imageParts[0] as { type: 'image_url'; image_url: { url: string } }
+    expect(imgPart.image_url.url).toBe(img.url)
+    expect(imgPart.image_url.url).toContain('data:image/png;base64,')
+  })
+
+  it('preserves image data through JSON serialization round-trip', () => {
+    const img = imageAttachment()
+    const msg = userMessage({ media: [img] })
+    const content = buildApiContent(msg)
+
+    // Simulate JSON round-trip (fetch body → request.json())
+    const serialized = JSON.stringify({ role: 'user', content })
+    const deserialized = JSON.parse(serialized)
+
+    // content must still be an array after round-trip
+    expect(Array.isArray(deserialized.content)).toBe(true)
+    expect(deserialized.content).toHaveLength(2) // text + image
+
+    // image_url part must survive
+    const imgPart = deserialized.content.find((p: ContentPart) => p.type === 'image_url')
+    expect(imgPart).toBeDefined()
+    expect(imgPart.image_url.url).toBe(img.url)
+  })
+
+  it('validation preserves multimodal content arrays — does not stringify them', () => {
+    const imageUrl = 'data:image/png;base64,iVBORw0KGgoAAAA'
+    const messages = [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'what is this?' },
+          { type: 'image_url', image_url: { url: imageUrl } },
+        ],
+      },
+    ]
+
+    const validated = validateMessages(messages)
+    expect(validated).toHaveLength(1)
+
+    // content must be the ARRAY, not a stringified version
+    const content = validated[0].content
+    expect(Array.isArray(content)).toBe(true)
+    expect(typeof content).not.toBe('string')
+
+    const parts = content as ContentPart[]
+    expect(parts).toHaveLength(2)
+
+    const imgPart = parts[1] as { type: 'image_url'; image_url: { url: string } }
+    expect(imgPart.type).toBe('image_url')
+    expect(imgPart.image_url.url).toBe(imageUrl)
+  })
+
+  it('route message mapping preserves multimodal content as-is', () => {
+    // Simulate what the route does: map validated messages
+    const imageUrl = 'data:image/jpeg;base64,/9j/4AAQSkZJRg'
+    const validatedMessages = [
+      {
+        role: 'user' as const,
+        content: [
+          { type: 'text' as const, text: 'describe this image' },
+          { type: 'image_url' as const, image_url: { url: imageUrl } },
+        ] as ContentPart[],
+      },
+    ]
+
+    // This is what the route does on line 57
+    const mapped = validatedMessages.map(m => ({ role: m.role, content: m.content }))
+
+    // content must still be the array
+    expect(Array.isArray(mapped[0].content)).toBe(true)
+    const parts = mapped[0].content as ContentPart[]
+    const imgPart = parts[1] as { type: 'image_url'; image_url: { url: string } }
+    expect(imgPart.image_url.url).toBe(imageUrl)
+  })
+
+  it('handles image-only message (no text) through full pipeline', () => {
+    // When user sends image without typing anything, content becomes a label
+    const img = imageAttachment()
+    const msgs: Message[] = [
+      userMessage({ content: '[screenshot.png]', media: [img] }),
+    ]
+
+    const { openaiMessages } = simulateFullPipeline(msgs)
+    const userMsg = openaiMessages[1] // index 0 is system
+
+    expect(Array.isArray(userMsg.content)).toBe(true)
+    const parts = userMsg.content as ContentPart[]
+
+    // Should have text part (the label) + image_url part
+    const imageParts = parts.filter(p => p.type === 'image_url')
+    expect(imageParts.length).toBe(1)
+
+    const imgPart = imageParts[0] as { type: 'image_url'; image_url: { url: string } }
+    expect(imgPart.image_url.url).toBe(img.url)
+  })
+
+  it('handles multiple images in a single message', () => {
+    const img1 = imageAttachment({ url: 'data:image/png;base64,AAA', name: 'first.png' })
+    const img2 = imageAttachment({ url: 'data:image/jpeg;base64,BBB', name: 'second.jpg' })
+    const msgs: Message[] = [
+      userMessage({ content: 'compare these', media: [img1, img2] }),
+    ]
+
+    const { openaiMessages } = simulateFullPipeline(msgs)
+    const userMsg = openaiMessages[1]
+
+    expect(Array.isArray(userMsg.content)).toBe(true)
+    const parts = userMsg.content as ContentPart[]
+    const imageParts = parts.filter(p => p.type === 'image_url') as Array<{ type: 'image_url'; image_url: { url: string } }>
+
+    expect(imageParts).toHaveLength(2)
+    expect(imageParts[0].image_url.url).toBe('data:image/png;base64,AAA')
+    expect(imageParts[1].image_url.url).toBe('data:image/jpeg;base64,BBB')
+  })
+
+  it('plain text messages remain strings through the pipeline (not wrapped in array)', () => {
+    const msgs: Message[] = [
+      assistantMessage(),
+      userMessage({ content: 'just text, no images' }),
+    ]
+
+    const { openaiMessages } = simulateFullPipeline(msgs)
+
+    // Assistant message (index 1, after system) should be a string
+    expect(typeof openaiMessages[1].content).toBe('string')
+
+    // User message (index 2) should also be a string (no media)
+    expect(typeof openaiMessages[2].content).toBe('string')
+    expect(openaiMessages[2].content).toBe('just text, no images')
+  })
+
+  it('conversation with mix of plain and multimodal messages', () => {
+    const img = imageAttachment()
+    const msgs: Message[] = [
+      assistantMessage({ content: 'Hi there!' }),
+      userMessage({ id: 'u1', content: 'hello' }),  // plain
+      assistantMessage({ id: 'a1', content: 'How can I help?' }),
+      userMessage({ id: 'u2', content: 'what is this?', media: [img] }),  // multimodal
+    ]
+
+    const { openaiMessages } = simulateFullPipeline(msgs)
+
+    // system (0), assistant (1), user plain (2), assistant (3), user multimodal (4)
+    expect(openaiMessages).toHaveLength(5)
+
+    // Plain user message
+    expect(typeof openaiMessages[2].content).toBe('string')
+
+    // Multimodal user message
+    expect(Array.isArray(openaiMessages[4].content)).toBe(true)
+    const parts = openaiMessages[4].content as ContentPart[]
+    expect(parts.some(p => p.type === 'image_url')).toBe(true)
+  })
+})
+
+// =============================================================
+// Specific regression tests for common bugs
+// =============================================================
+
+describe('regression — content must not become [object Object]', () => {
+  it('buildApiContent result does not stringify to [object Object]', () => {
+    const msg = userMessage({ media: [imageAttachment()] })
+    const content = buildApiContent(msg)
+
+    // If content were accidentally coerced to string, it would be "[object Object]"
+    const asString = String(content)
+    expect(asString).not.toBe('[object Object]')
+
+    // The actual content should be an array
+    expect(Array.isArray(content)).toBe(true)
+  })
+
+  it('multimodal content survives template literal (catches accidental string coercion)', () => {
+    const msg = userMessage({ media: [imageAttachment()] })
+    const content = buildApiContent(msg)
+
+    // This simulates an accidental `${content}` in code
+    if (Array.isArray(content)) {
+      // Verify the array is properly structured, not accidentally stringified
+      const jsonStr = JSON.stringify(content)
+      expect(jsonStr).not.toContain('[object Object]')
+      expect(jsonStr).toContain('image_url')
+      expect(jsonStr).toContain('data:image/png;base64,')
+    }
+  })
+})
+
+describe('regression — validation must not strip image_url parts', () => {
+  it('validates and returns all content part types', () => {
+    const result = validateChatMessages({
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: 'look at this' },
+          { type: 'image_url', image_url: { url: 'data:image/png;base64,abc' } },
+        ],
+      }],
+    })
+
+    expect(result.ok).toBe(true)
+    if (!result.ok) return
+
+    const content = result.messages[0].content
+    expect(Array.isArray(content)).toBe(true)
+    const parts = content as ContentPart[]
+
+    // Both parts must survive validation
+    expect(parts).toHaveLength(2)
+    expect(parts[0].type).toBe('text')
+    expect(parts[1].type).toBe('image_url')
+  })
+
+  it('preserves the exact image URL through validation', () => {
+    const longBase64Url = 'data:image/png;base64,' + 'A'.repeat(1000)
+    const result = validateChatMessages({
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'image_url', image_url: { url: longBase64Url } },
+        ],
+      }],
+    })
+
+    expect(result.ok).toBe(true)
+    if (!result.ok) return
+
+    const parts = result.messages[0].content as ContentPart[]
+    const imgPart = parts[0] as { type: 'image_url'; image_url: { url: string } }
+    expect(imgPart.image_url.url).toBe(longBase64Url)
+  })
+})