ethagent 2.3.0 → 3.0.0

package/src/models/modelPickerOptions.ts

@@ -23,6 +23,9 @@ export type LocalHfPickerModel = {
   risk: HfRisk
   task: HfTask
   status: 'ready' | 'incomplete'
+  mmprojPath?: string
+  mmprojAvailable?: boolean
+  mmprojSizeBytes?: number
 }
 
 export type CloudCredentialKind = 'apikey' | 'oauth'
@@ -58,7 +61,9 @@ const CHILD_INDENT = 4
 export function buildModelPickerOptions(
   data: ModelPickerOptionsData,
   context: ModelPickerOptionsContext,
+  options_: { localOnly?: boolean } = {},
 ): SelectOption<string>[] {
+  const localOnly = options_.localOnly === true
   const options: SelectOption<string>[] = []
 
   options.push(sectionOption('hdr:local', 'Local Models'))
@@ -69,49 +74,53 @@ export function buildModelPickerOptions(
     options.push(utilityOption('local:uninstall', 'Uninstall Downloaded GGUF'))
   }
 
-  options.push(sectionOption('hdr:cloud', 'Cloud'))
-  for (const provider of MODEL_PICKER_CLOUD_PROVIDERS) {
-    options.push(groupOption(`hdr:cloud:${provider}`, cloudProviderDisplayName(provider)))
-    const keySet = data.cloudKeys[provider] === true
-    if (!keySet) {
-      if (provider === 'openai') {
-        options.push(utilityOption('oauth:openai', 'Sign in with ChatGPT', 'Use your ChatGPT subscription'))
+  if (!localOnly) {
+    options.push(sectionOption('hdr:cloud', 'Cloud'))
+    for (const provider of MODEL_PICKER_CLOUD_PROVIDERS) {
+      options.push(groupOption(`hdr:cloud:${provider}`, cloudProviderDisplayName(provider)))
+      const keySet = data.cloudKeys[provider] === true
+      if (!keySet) {
+        if (provider === 'openai') {
+          options.push(utilityOption('oauth:openai', 'Sign in with ChatGPT', 'Use your ChatGPT subscription'))
+        }
+        options.push(utilityOption(`key:set:${provider}`, 'Add API Key'))
+        continue
       }
-      options.push(utilityOption(`key:set:${provider}`, 'Add API Key'))
-      continue
-    }
 
-    const catalog = data.cloudCatalogs[provider]
-    if (catalog?.status === 'fallback') {
-      const reason = catalog.error ? ` · ${catalog.error}` : ''
-      options.push(noticeOption(
-        `hdr:cloud-fallback:${provider}`,
-        `Catalog unavailable${reason} · showing configured model`,
-        CHILD_INDENT,
-      ))
-    }
+      const catalog = data.cloudCatalogs[provider]
+      if (catalog?.status === 'fallback') {
+        const reason = catalog.error ? ` · ${catalog.error}` : ''
+        options.push(noticeOption(
+          `hdr:cloud-fallback:${provider}`,
+          `Catalog unavailable${reason} · showing configured model`,
+          CHILD_INDENT,
+        ))
+      }
 
-    const models = orderModelsForContextFit(provider, cloudPickerModels(provider, catalog, context), context.contextFit)
-    if (models.length === 0) {
-      options.push(noticeOption(`hdr:cloud-empty:${provider}`, 'No selectable models', CHILD_INDENT))
-    }
-    for (const model of models) {
-      const active = context.currentProvider === provider && context.currentModel === model
-      const displayName = formatModelDisplayName(provider, model, { maxLength: 58 })
-      options.push(rowOption(
-        `c:${provider}:${model}`,
-        contextFitLabel(provider, model, `${displayName}${active ? '  *' : ''}`, context.contextFit),
-      ))
+      const models = orderModelsForContextFit(provider, cloudPickerModels(provider, catalog, context), context.contextFit)
+      if (models.length === 0) {
+        options.push(noticeOption(`hdr:cloud-empty:${provider}`, 'No selectable models', CHILD_INDENT))
+      }
+      for (const model of models) {
+        const active = context.currentProvider === provider && context.currentModel === model
+        const displayName = formatModelDisplayName(provider, model, { maxLength: 58 })
+        options.push(rowOption(
+          `c:${provider}:${model}`,
+          contextFitLabel(provider, model, `${displayName}${active ? '  *' : ''}`, context.contextFit),
+        ))
+      }
+      options.push(utilityOption(`catalog:${provider}`, 'Full Catalog'))
+      const manageLabel = provider === 'openai' && data.cloudCredentialKinds?.openai === 'oauth'
+        ? 'Manage ChatGPT Sign-in'
+        : 'Manage API Key'
+      options.push(utilityOption(`key:manage:${provider}`, manageLabel))
     }
-    options.push(utilityOption(`catalog:${provider}`, 'Full Catalog'))
-    const manageLabel = provider === 'openai' && data.cloudCredentialKinds?.openai === 'oauth'
-      ? 'Manage ChatGPT Sign-in'
-      : 'Manage API Key'
-    options.push(utilityOption(`key:manage:${provider}`, manageLabel))
   }
 
-  options.push(sectionOption('hdr:exit', 'Exit'))
-  options.push(utilityOption('cancel', 'Close Model Picker', 'Return to chat without changing model'))
+  if (!localOnly) {
+    options.push(sectionOption('hdr:exit', 'Exit'))
+    options.push(utilityOption('cancel', 'Close Model Picker', 'Return to chat'))
+  }
 
   return options
 }
@@ -185,12 +194,22 @@ function appendHfModelOptions(
       displayName: model.displayName,
       maxLength,
     })
+    const tags = ['Installed']
+    if (model.mmprojPath) tags.push('Vision encoder loaded')
     options.push(rowOption(
       `hf:${id}`,
       contextFitLabel('llamacpp', id, `${active ? '* ' : '  '}${displayName}`, context.contextFit),
       undefined,
-      modelMetadataSubtext(size, ['Installed']),
+      modelMetadataSubtext(size, tags),
     ))
+    if (model.mmprojAvailable && !model.mmprojPath) {
+      const projectorSize = model.mmprojSizeBytes ? ` (+${formatSize(model.mmprojSizeBytes)})` : ''
+      options.push(rowOption(
+        `hfmmproj:${id}`,
+        `    + Add Vision Encoder${projectorSize}`,
+        'Enable image input on this local model',
+      ))
+    }
   }
 }
 

package/src/providers/anthropic.ts

@@ -4,6 +4,7 @@ import { ProviderError } from './contracts.js'
 import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseEvents } from './sse.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 
 export type AnthropicToolDefinition = {
   name: string
@@ -75,7 +76,22 @@ export class AnthropicProvider implements Provider {
       return
     }
 
-    const { system, conversation } = splitMessages(messages)
+    if (hasImageBlocks(messages) && !supportsAnthropicImages(this.model)) {
+      yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+      return
+    }
+
+    let split: { system?: string; conversation: Awaited<ReturnType<typeof splitMessages>>['conversation'] }
+    try {
+      split = await splitMessages(messages)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
+    const { system, conversation } = split
 
     let response: Response
     try {
@@ -195,22 +211,24 @@ export class AnthropicProvider implements Provider {
   }
 }
 
-function splitMessages(messages: Message[]): {
+async function splitMessages(messages: Message[]): Promise<{
   system?: string
   conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
   }>
-} {
+}> {
   const systemParts: string[] = []
   const conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
@@ -226,11 +244,16 @@ function splitMessages(messages: Message[]): {
     }
     conversation.push({
       role: message.role,
-      content: blocks.map(block => {
+      content: await Promise.all(blocks.map(async block => {
         if (block.type === 'text') return { type: 'text', text: block.text }
+        if (block.type === 'image') {
+          const loaded = await loadImageBlock(block)
+          if (!loaded.dataBase64 || !loaded.mimeType) throw new Error(`could not load image: ${block.path}`)
+          return { type: 'image', source: { type: 'base64', media_type: loaded.mimeType, data: loaded.dataBase64 } }
+        }
         if (block.type === 'tool_use') return { type: 'tool_use', id: block.id, name: block.name, input: block.input }
         return { type: 'tool_result', tool_use_id: block.toolUseId, content: block.content, is_error: block.isError }
-      }),
+      })),
     })
   }
 
@@ -251,6 +274,14 @@ function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   })
 }
 
+export function supportsAnthropicImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return normalized.includes('claude-3')
+    || normalized.includes('claude-sonnet-4')
+    || normalized.includes('claude-opus-4')
+    || normalized.includes('claude-haiku-4')
+}
+
 function normalizeStopReason(value?: string): 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' | 'unknown' {
   if (value === 'end_turn' || value === 'tool_use' || value === 'max_tokens' || value === 'stop_sequence') {
     return value

package/src/providers/contracts.ts

@@ -8,6 +8,14 @@ export type TextBlock = {
   text: string
 }
 
+export type ImageBlock = {
+  type: 'image'
+  path: string
+  mimeType?: string
+  url?: string
+  dataBase64?: string
+}
+
 export type ToolUseBlock = {
   type: 'tool_use'
   id: string
@@ -22,7 +30,7 @@ export type ToolResultBlock = {
   isError?: boolean
 }
 
-export type MessageContentBlock = TextBlock | ToolUseBlock | ToolResultBlock
+export type MessageContentBlock = TextBlock | ImageBlock | ToolUseBlock | ToolResultBlock
 
 export type Message = {
   role: Role
@@ -34,6 +42,7 @@ export type ProviderRetryStreamEvent = { type: 'retry' } & RetryEvent
 export type StreamEvent =
   | { type: 'text'; delta: string }
   | { type: 'thinking'; delta: string }
+  | { type: 'thinking_end' }
   | ProviderRetryStreamEvent
   | { type: 'tool_use_start'; id: string; name: string }
   | { type: 'tool_use_delta'; id: string; delta: string }

package/src/providers/gemini.ts

@@ -4,6 +4,7 @@ import { ProviderError } from './contracts.js'
 import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseFrames } from './sse.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 
 export type GeminiToolDefinition = {
   name: string
@@ -41,6 +42,7 @@ type GeminiChunk = {
 
 type GeminiContentPart =
   | { text: string }
+  | { inlineData: { mimeType: string; data: string } }
   | { functionCall: { name: string; args: Record<string, unknown> } }
   | { functionResponse: { name: string; response: Record<string, unknown> } }
 
@@ -92,8 +94,21 @@ export class GeminiProvider implements Provider {
       yield { type: 'error', message: error.message }
       return
     }
+    if (hasImageBlocks(messages) && !supportsGeminiImages(this.model)) {
+      yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+      return
+    }
 
-    const payload = buildGeminiPayload(messages, this.tools, options)
+    let payload: GeminiPayload
+    try {
+      payload = await buildGeminiPayload(messages, this.tools, options)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
     const modelName = this.model.replace(/^models\//, '')
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(modelName)}:streamGenerateContent?alt=sse`
 
@@ -181,11 +196,11 @@ export class GeminiProvider implements Provider {
   }
 }
 
-export function buildGeminiPayload(
+export async function buildGeminiPayload(
   messages: Message[],
   tools: GeminiToolDefinition[] = [],
   options: ProviderCompleteOptions = {},
-): GeminiPayload {
+): Promise<GeminiPayload> {
   const systemParts: string[] = []
   const contents: GeminiContent[] = []
   const toolUseNamesById = new Map<string, string>()
@@ -222,6 +237,10 @@ export function buildGeminiPayload(
     for (const block of blocks) {
       if (block.type === 'text') {
         parts.push({ text: block.text })
+      } else if (block.type === 'image') {
+        const loaded = await loadImageBlock(block)
+        if (!loaded.dataBase64 || !loaded.mimeType) throw new Error(`could not load image: ${block.path}`)
+        parts.push({ inlineData: { mimeType: loaded.mimeType, data: loaded.dataBase64 } })
       } else if (block.type === 'tool_result') {
         const name = toolUseNamesById.get(block.toolUseId) ?? 'unknown'
         const response: Record<string, unknown> = block.isError
@@ -258,6 +277,13 @@ function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   })
 }
 
+export function supportsGeminiImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return normalized.includes('gemini-1.5')
+    || normalized.includes('gemini-2.0')
+    || normalized.includes('gemini-2.5')
+}
+
 function normalizeFinishReason(reason: string, sawToolCall: boolean): DoneStopReason {
   if (sawToolCall) return 'tool_use'
   switch (reason) {

package/src/providers/openai-chat.ts

@@ -5,6 +5,7 @@ import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseFrames } from './sse.js'
 import { messageTextContent } from '../utils/messages.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 import { providerDisplayName } from '../models/providerDisplay.js'
 
 export type OpenAIToolDefinition = {
@@ -28,6 +29,7 @@ type Options = {
   loadApiKey?: () => Promise<string | null>
   tools?: OpenAIToolDefinition[]
   maxRetries?: number
+  hasVisionProjector?: boolean
 }
 
 type ChatChunk = {
@@ -76,6 +78,7 @@ export class OpenAIChatProvider implements Provider {
   private readonly loadApiKey?: () => Promise<string | null>
   private readonly tools: OpenAIToolDefinition[]
   private readonly maxRetries?: number
+  private readonly hasVisionProjector: boolean
 
   constructor(opts: Options) {
     this.id = opts.id
@@ -86,6 +89,7 @@ export class OpenAIChatProvider implements Provider {
     this.tools = opts.tools ?? []
     this.maxRetries = opts.maxRetries
     this.supportsTools = this.tools.length > 0
+    this.hasVisionProjector = opts.hasVisionProjector ?? false
   }
 
   async *complete(
@@ -99,6 +103,19 @@ export class OpenAIChatProvider implements Provider {
       yield { type: 'error', message: error.message }
       return
     }
+    if (hasImageBlocks(messages)) {
+      if (this.id === 'llamacpp' && !this.hasVisionProjector) {
+        const hint = localModelNameHintsVision(this.model)
+          ? '; open alt+p and run "Add Vision Encoder" on this model to enable image input'
+          : ''
+        yield { type: 'error', message: `image input is not enabled for local model "${this.model}" (no vision projector loaded)${hint}` }
+        return
+      }
+      if (this.id === 'openai' && !supportsOpenAIImages(this.model)) {
+        yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+        return
+      }
+    }
 
     const headers: Record<string, string> = {
       'Content-Type': 'application/json',
@@ -106,6 +123,17 @@ export class OpenAIChatProvider implements Provider {
     }
     if (apiKey) headers.Authorization = `Bearer ${apiKey}`
 
+    let wireMessages: Array<Record<string, unknown>>
+    try {
+      wireMessages = await toWireMessages(messages)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
+
     let response: Response
     try {
       response = yield* fetchWithRetryStreamEvents(`${this.baseUrl}/chat/completions`, {
@@ -113,7 +141,7 @@ export class OpenAIChatProvider implements Provider {
         headers,
         body: JSON.stringify({
           model: this.model,
-          messages: toWireMessages(messages),
+          messages: wireMessages,
           tools: this.tools.length > 0 ? this.tools : undefined,
           tool_choice: this.tools.length > 0 ? 'auto' : undefined,
           stream: true,
@@ -143,6 +171,7 @@ export class OpenAIChatProvider implements Provider {
     let stopReason: DoneStopReason = 'unknown'
     const toolCalls = new Map<number, StreamingToolCall>()
     const contentThinkingParser = new ContentThinkingParser(this.id)
+    let reasoningPending = false
 
     try {
       for await (const frame of iterSseFrames(response.body, signal, READ_TIMEOUT_MS)) {
@@ -166,18 +195,34 @@ export class OpenAIChatProvider implements Provider {
                 ? delta.thinking
                 : ''
 
-        if (reasoning.length > 0) yield { type: 'thinking', delta: reasoning }
+        if (reasoning.length > 0) {
+          yield { type: 'thinking', delta: reasoning }
+          reasoningPending = true
+        }
         if (text.length > 0) {
+          if (reasoningPending) {
+            yield { type: 'thinking_end' }
+            reasoningPending = false
+          }
           for (const event of contentThinkingParser.push(text)) {
             yield event
           }
         }
 
-        for (const event of applyStreamingToolCallDelta(toolCalls, delta?.tool_calls ?? [])) {
+        const toolCallDeltas = delta?.tool_calls ?? []
+        if (toolCallDeltas.length > 0 && reasoningPending) {
+          yield { type: 'thinking_end' }
+          reasoningPending = false
+        }
+        for (const event of applyStreamingToolCallDelta(toolCalls, toolCallDeltas)) {
           yield event
         }
 
         if (choice?.finish_reason) {
+          if (reasoningPending) {
+            yield { type: 'thinking_end' }
+            reasoningPending = false
+          }
           stopReason = normalizeFinishReason(choice.finish_reason)
         }
         if (parsed.usage) {
@@ -195,6 +240,10 @@ export class OpenAIChatProvider implements Provider {
     for (const event of contentThinkingParser.flush()) {
       yield event
     }
+    if (reasoningPending) {
+      yield { type: 'thinking_end' }
+      reasoningPending = false
+    }
 
     let streamEmittedToolUses = 0
     if (stopReason === 'tool_use' || toolCalls.size > 0) {
@@ -221,7 +270,7 @@ export class OpenAIChatProvider implements Provider {
 
 }
 
-export function toWireMessages(messages: Message[]): Array<Record<string, unknown>> {
+export async function toWireMessages(messages: Message[]): Promise<Array<Record<string, unknown>>> {
   const out: Array<Record<string, unknown>> = []
 
   for (const message of messages) {
@@ -230,6 +279,26 @@ export function toWireMessages(messages: Message[]): Array<Record<string, unknow
       continue
     }
 
+    if (message.role === 'user') {
+      const toolResults = message.content.filter(isToolResultBlock)
+      if (toolResults.length > 0) {
+        for (const block of toolResults) {
+          out.push({
+            role: 'tool',
+            tool_call_id: block.toolUseId,
+            content: block.content,
+          })
+        }
+        const nonToolBlocks = message.content.filter(block => block.type !== 'tool_result')
+        if (nonToolBlocks.length > 0) {
+          out.push({ role: 'user', content: await toOpenAIUserContent(nonToolBlocks) })
+        }
+        continue
+      }
+      out.push({ role: 'user', content: await toOpenAIUserContent(message.content) })
+      continue
+    }
+
     if (message.role === 'assistant') {
       const textParts = message.content.filter(isTextBlock).map(block => block.text)
       const toolCalls = message.content.filter(isToolUseBlock).map(block => ({
@@ -266,6 +335,37 @@ export function toWireMessages(messages: Message[]): Array<Record<string, unknow
   return normalizeSystemMessages(out)
 }
 
+async function toOpenAIUserContent(blocks: MessageContentBlock[]): Promise<Array<Record<string, unknown>>> {
+  const parts: Array<Record<string, unknown>> = []
+  for (const block of blocks) {
+    if (block.type === 'text') {
+      if (block.text.length > 0) parts.push({ type: 'text', text: block.text })
+      continue
+    }
+    if (block.type === 'image') {
+      const loaded = await loadImageBlock(block)
+      if (loaded.url) {
+        parts.push({ type: 'image_url', image_url: { url: loaded.url } })
+      } else if (loaded.dataBase64 && loaded.mimeType) {
+        parts.push({ type: 'image_url', image_url: { url: `data:${loaded.mimeType};base64,${loaded.dataBase64}` } })
+      }
+      continue
+    }
+  }
+  return parts.length > 0 ? parts : [{ type: 'text', text: '' }]
+}
+
+export function supportsOpenAIImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  if (normalized.includes('gpt-3.5')) return false
+  return /gpt-4o|gpt-4\.1|gpt-4-turbo|gpt-4-vision|gpt-5|o1|o3|o4|chatgpt-4/.test(normalized)
+}
+
+export function localModelNameHintsVision(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return /llava|bakllava|qwen[-_.]?vl|qwen2[-_.]?vl|qwen2\.5[-_.]?vl|minicpm-?v|llama-3\.2.*vision|mllama|cogvlm|internvl|moondream|pixtral|phi-?3[\.-]?vision|phi-?3\.5[\.-]?vision|smolvlm/.test(normalized)
+}
+
 function normalizeSystemMessages(messages: Array<Record<string, unknown>>): Array<Record<string, unknown>> {
   const systemContents: string[] = []
   const nonSystem: Array<Record<string, unknown>> = []
@@ -304,17 +404,35 @@ function isToolResultBlock(block: MessageContentBlock): block is Extract<Message
 
 function parseToolArguments(inputJson: string): Record<string, unknown> {
   if (!inputJson.trim()) return {}
+  const direct = tryParseJsonOnce(inputJson)
+  if (direct !== undefined) return coerceToToolArguments(direct)
+  const repaired = repairJsonObject(inputJson)
+  if (!repaired) return {}
+  const parsedRepaired = tryParseJsonOnce(repaired)
+  return parsedRepaired === undefined ? {} : coerceToToolArguments(parsedRepaired)
+}
+
+function tryParseJsonOnce(value: string): unknown {
   try {
-    return JSON.parse(inputJson) as Record<string, unknown>
+    return JSON.parse(value)
   } catch {
-    const repaired = repairJsonObject(inputJson)
-    if (!repaired) return {}
-    try {
-      return JSON.parse(repaired) as Record<string, unknown>
-    } catch {
-      return {}
+    return undefined
+  }
+}
+
+function coerceToToolArguments(value: unknown): Record<string, unknown> {
+  if (typeof value === 'string') {
+    const trimmed = value.trim()
+    if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+      const inner = tryParseJsonOnce(trimmed)
+      if (inner !== undefined) return coerceToToolArguments(inner)
     }
+    return {}
+  }
+  if (value && typeof value === 'object' && !Array.isArray(value)) {
+    return value as Record<string, unknown>
   }
+  return {}
 }
 
 function* applyStreamingToolCallDelta(
@@ -407,7 +525,9 @@ class ContentThinkingParser {
           yield { type: this.state === 'thinking' ? 'thinking' : 'text', delta: before }
         }
         this.buffer = this.buffer.slice(tagIndex + tag.length)
+        const wasThinking = this.state === 'thinking'
         this.state = this.state === 'text' ? 'thinking' : 'text'
+        if (wasThinking) yield { type: 'thinking_end' }
         continue
       }
 

package/src/providers/openai-responses-format.ts

@@ -1,9 +1,11 @@
 import type { Message, MessageContentBlock } from './contracts.js'
 import { messageTextContent } from '../utils/messages.js'
 import type { OpenAIToolDefinition } from './openai-chat.js'
+import { loadImageBlock } from '../utils/images.js'
 
 export type ResponsesInputContent =
   | { type: 'input_text'; text: string }
+  | { type: 'input_image'; image_url: string }
   | { type: 'output_text'; text: string }
 
 export type ResponsesInputItem =
@@ -30,13 +32,13 @@ export type ResponsesRequestBody = {
   max_output_tokens?: number
 }
 
-export function buildResponsesBody(args: {
+export async function buildResponsesBody(args: {
   model: string
   messages: Message[]
   tools: OpenAIToolDefinition[]
   maxOutputTokens?: number
-}): ResponsesRequestBody {
-  const { instructions, items } = splitMessages(args.messages)
+}): Promise<ResponsesRequestBody> {
+  const { instructions, items } = await splitMessages(args.messages)
   const body: ResponsesRequestBody = {
     model: args.model,
     input: items,
@@ -60,10 +62,10 @@ export function buildResponsesBody(args: {
   return body
 }
 
-function splitMessages(messages: Message[]): {
+async function splitMessages(messages: Message[]): Promise<{
   instructions?: string
   items: ResponsesInputItem[]
-} {
+}> {
   const instructions: string[] = []
   const items: ResponsesInputItem[] = []
 
@@ -100,12 +102,12 @@ function splitMessages(messages: Message[]): {
         }
         continue
       }
-      const text = blocks.filter(isTextBlock).map(block => block.text).join('')
-      if (text) {
+      const content = await toOpenAIResponsesUserContent(blocks)
+      if (content.length > 0) {
         items.push({
           type: 'message',
           role: 'user',
-          content: [{ type: 'input_text', text }],
+          content,
         })
       }
       continue
@@ -136,6 +138,25 @@ function splitMessages(messages: Message[]): {
   }
 }
 
+async function toOpenAIResponsesUserContent(blocks: MessageContentBlock[]): Promise<ResponsesInputContent[]> {
+  const content: ResponsesInputContent[] = []
+  for (const block of blocks) {
+    if (block.type === 'text') {
+      if (block.text) content.push({ type: 'input_text', text: block.text })
+      continue
+    }
+    if (block.type === 'image') {
+      const loaded = await loadImageBlock(block)
+      if (loaded.url) {
+        content.push({ type: 'input_image', image_url: loaded.url })
+      } else if (loaded.dataBase64 && loaded.mimeType) {
+        content.push({ type: 'input_image', image_url: `data:${loaded.mimeType};base64,${loaded.dataBase64}` })
+      }
+    }
+  }
+  return content
+}
+
 function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   if (typeof content === 'string') {
     return content ? [{ type: 'text', text: content }] : []