@strav/brain 1.0.0-alpha.15 → 1.0.0-alpha.17

package/src/providers/gemini_provider.ts

@@ -60,11 +60,21 @@ import type { AgentResult } from '../agent_result.ts'
 import { BrainError } from '../brain_error.ts'
 import type { GeminiProviderConfig } from '../brain_config.ts'
 import type { MCPServer } from '../mcp_server.ts'
+import type { AgentGenerateResult } from '../agent_generate_result.ts'
+import type { AgentStreamEvent } from '../agent_stream_event.ts'
+import type {
+  AudioSource,
+  EmbedOptions,
+  EmbedResult,
+  ServerTool,
+  TranscribeOptions,
+  TranscribeResult,
+} from '../types.ts'
 import { resolveMcpTools, type ResolveMcpToolsOptions } from '../mcp/resolve_mcp_tools.ts'
 import { parseGenerated, type OutputSchema } from '../output_schema.ts'
+import { runToolWithRecovery } from '../tool_runner.ts'
 import type { Provider, RunWithToolsOptions } from '../provider.ts'
 import type { Tool } from '../tool.ts'
-import { ToolExecutionError } from '../tool_execution_error.ts'
 import type {
   ChatOptions,
   ChatResult,
@@ -80,6 +90,7 @@ import type {
 } from '../types.ts'
 
 const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash'
+const DEFAULT_GEMINI_EMBED_MODEL = 'text-embedding-004'
 
 /**
  * The slice of `GoogleGenAI` the provider exercises. Narrowed so
@@ -91,6 +102,17 @@ export interface GeminiModelsClient {
     params: GenerateContentParameters,
   ): Promise<AsyncIterable<GenerateContentResponse>>
   countTokens(params: { model: string; contents: Content[] }): Promise<{ totalTokens?: number }>
+  /**
+   * Optional on the test seam — the real SDK always provides it,
+   * but tests that don't exercise embed don't need to stub it.
+   * `embed()` calls this directly; missing it throws a clear
+   * TypeError if invoked.
+   */
+  embedContent?(params: {
+    model: string
+    contents: string[]
+    config?: { outputDimensionality?: number; abortSignal?: AbortSignal }
+  }): Promise<{ embeddings?: Array<{ values?: number[] }> }>
 }
 
 export interface GeminiProviderOptions {
@@ -104,12 +126,14 @@ export class GeminiProvider implements Provider {
   private readonly models: GeminiModelsClient
   private readonly defaultModel: string
   private readonly defaultMaxTokens: number
+  private readonly defaultEmbedModel: string
   private readonly mcpClientFactory?: ResolveMcpToolsOptions['clientFactory']
 
   constructor(name: string, config: GeminiProviderConfig, options: GeminiProviderOptions = {}) {
     this.name = name
     this.defaultModel = config.defaultModel ?? DEFAULT_GEMINI_MODEL
     this.defaultMaxTokens = config.defaultMaxTokens ?? 4096
+    this.defaultEmbedModel = config.defaultEmbedModel ?? DEFAULT_GEMINI_EMBED_MODEL
     this.mcpClientFactory = options.mcpClientFactory
     if (options.client) {
       this.models = options.client.models
@@ -169,6 +193,86 @@ export class GeminiProvider implements Provider {
     return response.totalTokens ?? 0
   }
 
+  /**
+   * Gemini embeddings via `ai.models.embedContent`. Returns one
+   * vector per input text. `usage.inputTokens` is `0` — Gemini's
+   * embed endpoint doesn't surface token counts in the response
+   * for the Gemini Developer API tier (Vertex's request-level
+   * metadata exposes billable characters, but that's a different
+   * accounting unit and not the framework's contract). Apps that
+   * need exact embed-token usage call `countTokens` separately
+   * before the call.
+   */
+  /**
+   * Gemini has no dedicated transcription endpoint, so we wrap a
+   * chat call: an AudioBlock + a system message that tells the
+   * model to transcribe verbatim. Apps that want OpenAI-style
+   * Whisper transcription with `language` / `duration` metadata
+   * route to OpenAI (or local Whisper via Ollama).
+   *
+   * `options.prompt` threads into the system instruction —
+   * useful for style/vocabulary hints. `options.language` is
+   * surfaced to the model in the system prompt (Gemini doesn't
+   * have a dedicated language field).
+   */
+  async transcribe(
+    audio: AudioSource,
+    options: TranscribeOptions = {},
+  ): Promise<TranscribeResult> {
+    const lines = [
+      'Transcribe the attached audio verbatim. Output ONLY the transcribed text — no preamble, no quotes, no commentary.',
+      options.language ? `Audio language: ${options.language}.` : undefined,
+      options.prompt ? `Style / vocabulary hints: ${options.prompt}` : undefined,
+    ].filter((s): s is string => s !== undefined)
+    const system = lines.join(' ')
+    const chatResult = await this.chat(
+      [
+        {
+          role: 'user',
+          content: [{ type: 'audio', source: audio }],
+        },
+      ],
+      {
+        system,
+        ...(options.model !== undefined ? { model: options.model } : {}),
+        ...(options.signal !== undefined ? { signal: options.signal } : {}),
+      },
+    )
+    return {
+      text: chatResult.text,
+      model: chatResult.model,
+      raw: chatResult.raw,
+    }
+  }
+
+  async embed(
+    texts: readonly string[],
+    options: EmbedOptions = {},
+  ): Promise<EmbedResult<{ embeddings?: Array<{ values?: number[] }> }>> {
+    const model = options.model ?? this.defaultEmbedModel
+    const config: { outputDimensionality?: number; abortSignal?: AbortSignal } = {}
+    if (options.dimensions !== undefined) config.outputDimensionality = options.dimensions
+    if (options.signal !== undefined) config.abortSignal = options.signal
+    if (!this.models.embedContent) {
+      throw new BrainError(
+        `GeminiProvider.embed: underlying SDK does not implement embedContent. This usually means a test stub omitted it.`,
+        { context: { provider: this.name } },
+      )
+    }
+    const response = await this.models.embedContent({
+      model,
+      contents: texts as string[],
+      ...(Object.keys(config).length > 0 ? { config } : {}),
+    })
+    const embeddings = (response.embeddings ?? []).map((e) => e.values ?? [])
+    return {
+      embeddings,
+      model,
+      usage: { inputTokens: 0 },
+      raw: response,
+    }
+  }
+
   async runWithTools(
     messages: readonly Message[],
     tools: readonly Tool[],
@@ -205,6 +309,7 @@ export class GeminiProvider implements Provider {
     let iterations = 0
 
     while (true) {
+      checkAborted(options.signal)
       const params = this.buildParams(workingMessages, options, tools)
       const response = await this.models.generateContent(params)
       addUsage(aggregated, response.usageMetadata)
@@ -235,36 +340,132 @@ export class GeminiProvider implements Provider {
 
       const resultBlocks: ContentBlock[] = []
       for (const call of toolUses) {
-        const tool = toolMap.get(call.name)
-        if (!tool) {
-          throw new ToolExecutionError(
-            call.name,
-            call.id,
-            new Error(`Tool "${call.name}" is not registered.`),
-          )
+        const { content, isError } = await runToolWithRecovery(
+          toolMap.get(call.name),
+          call.name,
+          call.id,
+          call.input,
+          options,
+        )
+        resultBlocks.push({
+          type: 'tool_result',
+          toolUseId: call.id,
+          content,
+          ...(isError ? { isError: true } : {}),
+        } satisfies ToolResultBlock)
+      }
+      workingMessages.push({ role: 'user', content: resultBlocks })
+
+      iterations++
+      if (iterations >= maxIterations) {
+        return {
+          text: candidateText(candidate),
+          messages: workingMessages,
+          iterations,
+          stopReason: 'max_iterations',
+          usage: aggregated,
         }
-        let output: unknown
-        try {
-          output = await tool.execute(call.input, {
-            callId: call.id,
-            context: options.context ?? {},
+      }
+    }
+  }
+
+  async runWithToolsAndSchema<T>(
+    messages: readonly Message[],
+    tools: readonly Tool[],
+    schema: OutputSchema<T>,
+    options: RunWithToolsOptions = {},
+  ): Promise<AgentGenerateResult<T>> {
+    const mcpServers: readonly MCPServer[] = options.mcpServers ?? []
+    const resolved =
+      mcpServers.length > 0
+        ? await resolveMcpTools(mcpServers, {
+            ...(this.mcpClientFactory ? { clientFactory: this.mcpClientFactory } : {}),
           })
-        } catch (cause) {
-          throw new ToolExecutionError(call.name, call.id, cause)
+        : { tools: [] as Tool[], close: async () => {} }
+    try {
+      return await this._runLoopWithSchema([...tools, ...resolved.tools], messages, schema, options)
+    } finally {
+      await resolved.close()
+    }
+  }
+
+  private async _runLoopWithSchema<T>(
+    tools: readonly Tool[],
+    messages: readonly Message[],
+    schema: OutputSchema<T>,
+    options: RunWithToolsOptions,
+  ): Promise<AgentGenerateResult<T>> {
+    const maxIterations = options.maxIterations ?? 10
+    const toolMap = new Map<string, Tool>(tools.map((t) => [t.name, t]))
+    const workingMessages: Message[] = [...messages]
+    const aggregated: ChatUsage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheCreationTokens: 0,
+    }
+    let iterations = 0
+
+    while (true) {
+      const params = this.buildParams(workingMessages, options, tools)
+      params.config = {
+        ...(params.config ?? {}),
+        responseMimeType: 'application/json',
+        responseJsonSchema: schema.jsonSchema,
+      }
+      const response = await this.models.generateContent(params)
+      addUsage(aggregated, response.usageMetadata)
+
+      const candidate = response.candidates?.[0]
+      if (!candidate) {
+        throw new BrainError('GeminiProvider: response had no candidates.')
+      }
+      const parts = candidate.content?.parts ?? []
+      const assistantContent = fromGeminiParts(parts)
+      workingMessages.push({ role: 'assistant', content: assistantContent })
+
+      const toolUses = (Array.isArray(assistantContent) ? assistantContent : []).filter(
+        (b): b is ToolUseBlock => b.type === 'tool_use',
+      )
+
+      if (toolUses.length === 0) {
+        const text = typeof assistantContent === 'string'
+          ? assistantContent
+          : candidateText(candidate)
+        return {
+          value: parseGenerated(text, schema),
+          text,
+          messages: workingMessages,
+          iterations,
+          stopReason: candidate.finishReason ? String(candidate.finishReason) : 'stop',
+          usage: aggregated,
         }
-        const resultBlock: ToolResultBlock = {
+      }
+
+      const resultBlocks: ContentBlock[] = []
+      for (const call of toolUses) {
+        const { content, isError } = await runToolWithRecovery(
+          toolMap.get(call.name),
+          call.name,
+          call.id,
+          call.input,
+          options,
+        )
+        resultBlocks.push({
           type: 'tool_result',
           toolUseId: call.id,
-          content: typeof output === 'string' ? output : JSON.stringify(output),
-        }
-        resultBlocks.push(resultBlock)
+          content,
+          ...(isError ? { isError: true } : {}),
+        } satisfies ToolResultBlock)
       }
       workingMessages.push({ role: 'user', content: resultBlocks })
 
       iterations++
       if (iterations >= maxIterations) {
+        const text = candidateText(candidate)
         return {
-          text: candidateText(candidate),
+          value: parseGenerated(text, schema),
+          text,
           messages: workingMessages,
           iterations,
           stopReason: 'max_iterations',
@@ -274,6 +475,276 @@ export class GeminiProvider implements Provider {
     }
   }
 
+  async *streamWithTools(
+    messages: readonly Message[],
+    tools: readonly Tool[],
+    options: RunWithToolsOptions = {},
+  ): AsyncIterable<AgentStreamEvent> {
+    const mcpServers: readonly MCPServer[] = options.mcpServers ?? []
+    const resolved =
+      mcpServers.length > 0
+        ? await resolveMcpTools(mcpServers, {
+            ...(this.mcpClientFactory ? { clientFactory: this.mcpClientFactory } : {}),
+          })
+        : { tools: [] as Tool[], close: async () => {} }
+    try {
+      yield* this._streamLoop(messages, [...tools, ...resolved.tools], options)
+    } finally {
+      await resolved.close()
+    }
+  }
+
+  private async *_streamLoop(
+    messages: readonly Message[],
+    tools: readonly Tool[],
+    options: RunWithToolsOptions,
+  ): AsyncIterable<AgentStreamEvent> {
+    const maxIterations = options.maxIterations ?? 10
+    const toolMap = new Map<string, Tool>(tools.map((t) => [t.name, t]))
+    const workingMessages: Message[] = [...messages]
+    const aggregated: ChatUsage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheCreationTokens: 0,
+    }
+    let iterations = 0
+
+    while (true) {
+      checkAborted(options.signal)
+      yield { type: 'iteration_start', iteration: iterations }
+
+      const params = this.buildParams(workingMessages, options, tools)
+      const stream = await this.models.generateContentStream(params)
+
+      const accumulatedParts: Part[] = []
+      let finishReason: string | null = null
+      let lastUsage: ChatUsage | undefined
+
+      for await (const chunk of stream) {
+        const candidate = chunk.candidates?.[0]
+        const chunkParts = candidate?.content?.parts ?? []
+        for (const part of chunkParts) {
+          if (typeof part.text === 'string' && part.text.length > 0) {
+            yield { type: 'text', delta: part.text }
+          }
+        }
+        accumulatedParts.push(...chunkParts)
+        if (candidate?.finishReason) finishReason = String(candidate.finishReason)
+        if (chunk.usageMetadata) lastUsage = toUsage(chunk.usageMetadata)
+      }
+      if (lastUsage) {
+        aggregated.inputTokens += lastUsage.inputTokens
+        aggregated.outputTokens += lastUsage.outputTokens
+        aggregated.cacheReadTokens += lastUsage.cacheReadTokens
+      }
+
+      yield { type: 'iteration_end', iteration: iterations, stopReason: finishReason }
+
+      const assistantContent = fromGeminiParts(accumulatedParts)
+      workingMessages.push({ role: 'assistant', content: assistantContent })
+
+      const toolUses = (Array.isArray(assistantContent) ? assistantContent : []).filter(
+        (b): b is ToolUseBlock => b.type === 'tool_use',
+      )
+
+      if (toolUses.length === 0) {
+        yield {
+          type: 'stop',
+          stopReason: finishReason ?? 'stop',
+          iterations,
+          usage: aggregated,
+          messages: workingMessages,
+        }
+        return
+      }
+
+      const resultBlocks: ContentBlock[] = []
+      for (const call of toolUses) {
+        yield { type: 'tool_use', id: call.id, name: call.name, input: call.input }
+        const { content, isError } = await runToolWithRecovery(
+          toolMap.get(call.name),
+          call.name,
+          call.id,
+          call.input,
+          options,
+        )
+        resultBlocks.push({
+          type: 'tool_result',
+          toolUseId: call.id,
+          content,
+          ...(isError ? { isError: true } : {}),
+        } satisfies ToolResultBlock)
+        yield {
+          type: 'tool_result',
+          id: call.id,
+          name: call.name,
+          content,
+          isError,
+        }
+      }
+      workingMessages.push({ role: 'user', content: resultBlocks })
+
+      iterations++
+      if (iterations >= maxIterations) {
+        yield {
+          type: 'stop',
+          stopReason: 'max_iterations',
+          iterations,
+          usage: aggregated,
+          messages: workingMessages,
+        }
+        return
+      }
+    }
+  }
+
+  async *streamWithToolsAndSchema<T>(
+    messages: readonly Message[],
+    tools: readonly Tool[],
+    schema: OutputSchema<T>,
+    options: RunWithToolsOptions = {},
+  ): AsyncIterable<AgentStreamEvent<T>> {
+    const mcpServers: readonly MCPServer[] = options.mcpServers ?? []
+    const resolved =
+      mcpServers.length > 0
+        ? await resolveMcpTools(mcpServers, {
+            ...(this.mcpClientFactory ? { clientFactory: this.mcpClientFactory } : {}),
+          })
+        : { tools: [] as Tool[], close: async () => {} }
+    try {
+      yield* this._streamLoopWithSchema(
+        [...tools, ...resolved.tools],
+        messages,
+        schema,
+        options,
+      )
+    } finally {
+      await resolved.close()
+    }
+  }
+
+  private async *_streamLoopWithSchema<T>(
+    tools: readonly Tool[],
+    messages: readonly Message[],
+    schema: OutputSchema<T>,
+    options: RunWithToolsOptions,
+  ): AsyncIterable<AgentStreamEvent<T>> {
+    const maxIterations = options.maxIterations ?? 10
+    const toolMap = new Map<string, Tool>(tools.map((t) => [t.name, t]))
+    const workingMessages: Message[] = [...messages]
+    const aggregated: ChatUsage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      cacheReadTokens: 0,
+      cacheCreationTokens: 0,
+    }
+    let iterations = 0
+
+    while (true) {
+      checkAborted(options.signal)
+      yield { type: 'iteration_start', iteration: iterations }
+
+      const params = this.buildParams(workingMessages, options, tools)
+      params.config = {
+        ...(params.config ?? {}),
+        responseMimeType: 'application/json',
+        responseJsonSchema: schema.jsonSchema,
+      }
+      const stream = await this.models.generateContentStream(params)
+
+      const accumulatedParts: Part[] = []
+      let textBuf = ''
+      let finishReason: string | null = null
+      let lastUsage: ChatUsage | undefined
+
+      for await (const chunk of stream) {
+        const candidate = chunk.candidates?.[0]
+        const chunkParts = candidate?.content?.parts ?? []
+        for (const part of chunkParts) {
+          if (typeof part.text === 'string' && part.text.length > 0) {
+            textBuf += part.text
+            yield { type: 'text', delta: part.text }
+          }
+        }
+        accumulatedParts.push(...chunkParts)
+        if (candidate?.finishReason) finishReason = String(candidate.finishReason)
+        if (chunk.usageMetadata) lastUsage = toUsage(chunk.usageMetadata)
+      }
+      if (lastUsage) {
+        aggregated.inputTokens += lastUsage.inputTokens
+        aggregated.outputTokens += lastUsage.outputTokens
+        aggregated.cacheReadTokens += lastUsage.cacheReadTokens
+      }
+
+      yield { type: 'iteration_end', iteration: iterations, stopReason: finishReason }
+
+      const assistantContent = fromGeminiParts(accumulatedParts)
+      workingMessages.push({ role: 'assistant', content: assistantContent })
+
+      const toolUses = (Array.isArray(assistantContent) ? assistantContent : []).filter(
+        (b): b is ToolUseBlock => b.type === 'tool_use',
+      )
+
+      if (toolUses.length === 0) {
+        const text = textBuf
+        const value = parseGenerated(text, schema)
+        yield {
+          type: 'stop',
+          stopReason: finishReason ?? 'stop',
+          iterations,
+          usage: aggregated,
+          messages: workingMessages,
+          value,
+          text,
+        } as AgentStreamEvent<T>
+        return
+      }
+
+      const resultBlocks: ContentBlock[] = []
+      for (const call of toolUses) {
+        yield { type: 'tool_use', id: call.id, name: call.name, input: call.input }
+        const { content, isError } = await runToolWithRecovery(
+          toolMap.get(call.name),
+          call.name,
+          call.id,
+          call.input,
+          options,
+        )
+        resultBlocks.push({
+          type: 'tool_result',
+          toolUseId: call.id,
+          content,
+          ...(isError ? { isError: true } : {}),
+        } satisfies ToolResultBlock)
+        yield {
+          type: 'tool_result',
+          id: call.id,
+          name: call.name,
+          content,
+          isError,
+        }
+      }
+      workingMessages.push({ role: 'user', content: resultBlocks })
+
+      iterations++
+      if (iterations >= maxIterations) {
+        const text = textBuf
+        const value = parseGenerated(text, schema)
+        yield {
+          type: 'stop',
+          stopReason: 'max_iterations',
+          iterations,
+          usage: aggregated,
+          messages: workingMessages,
+          value,
+          text,
+        } as AgentStreamEvent<T>
+        return
+      }
+    }
+  }
+
   async generate<T>(
     messages: readonly Message[],
     schema: OutputSchema<T>,
@@ -317,18 +788,27 @@ export class GeminiProvider implements Provider {
       config.systemInstruction = systemText
     }
 
+    const configTools: NonNullable<GenerateContentConfig['tools']> = []
     if (tools.length > 0) {
       const functionDeclarations: FunctionDeclaration[] = tools.map((t) => ({
         name: t.name,
         description: t.description,
         parametersJsonSchema: t.inputSchema,
       }))
-      config.tools = [{ functionDeclarations }]
+      configTools.push({ functionDeclarations })
+    }
+    if (options.serverTools && options.serverTools.length > 0) {
+      configTools.push(...geminiServerTools(options.serverTools))
+    }
+    if (configTools.length > 0) {
+      config.tools = configTools
     }
 
     const thinking = buildThinkingConfig(options)
     if (thinking !== undefined) config.thinkingConfig = thinking
 
+    if (options.signal !== undefined) config.abortSignal = options.signal
+
     return { model, contents, config }
   }
 
@@ -356,6 +836,13 @@ export class GeminiProvider implements Provider {
 
 // ─── Shape converters ─────────────────────────────────────────────────────
 
+/** Throw a DOMException-shaped abort error if the signal has fired. */
+function checkAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted) {
+    throw signal.reason ?? new DOMException('Aborted', 'AbortError')
+  }
+}
+
 function systemPromptText(system: SystemPrompt | undefined): string {
   if (system === undefined) return ''
   if (typeof system === 'string') return system
@@ -369,6 +856,25 @@ function toGeminiParts(content: string | ContentBlock[]): Part[] {
   for (const block of content) {
     if (block.type === 'text') {
       parts.push({ text: block.text })
+    } else if (block.type === 'image' || block.type === 'document' || block.type === 'audio') {
+      // All three media block types share Gemini's inlineData /
+      // fileData wire shape; only the MIME differs. Base64 →
+      // inlineData. URL → fileData with fileUri. Gemini's
+      // fileData accepts public HTTPS and gs:// URIs; arbitrary
+      // private URLs need to be fetched and converted to base64
+      // by the app.
+      if (block.source.type === 'base64') {
+        parts.push({
+          inlineData: { mimeType: block.source.mediaType, data: block.source.data },
+        })
+      } else {
+        parts.push({
+          fileData: {
+            fileUri: block.source.url,
+            mimeType: guessMimeFromUrl(block.source.url, block.type),
+          },
+        })
+      }
     } else if (block.type === 'tool_use') {
       parts.push({
         functionCall: {
@@ -394,6 +900,69 @@ function toGeminiParts(content: string | ContentBlock[]): Part[] {
   return parts
 }
 
+/**
+ * Gemini's `fileData.mimeType` is required, but our media-block
+ * URL-source variants don't carry it (the app may not know).
+ * Best-effort from the file extension. Default falls back to the
+ * block type's most-common MIME (jpeg for images, pdf for
+ * documents, mp3 for audio).
+ */
+/**
+ * Translate framework `ServerTool[]` into Gemini's typed entries
+ * (`googleSearch` / `codeExecution` / `urlContext`). Anthropic-
+ * specific tools (`web_fetch`) throw with clear guidance.
+ *
+ * Gemini's server tools have no per-tool config — they're enabled
+ * with empty `{}` objects. Domain allowlists / max_uses /
+ * blocked_domains on `web_search` are silently dropped (Gemini
+ * doesn't accept them).
+ */
+function geminiServerTools(
+  serverTools: readonly ServerTool[],
+): NonNullable<GenerateContentConfig['tools']> {
+  const out: NonNullable<GenerateContentConfig['tools']> = []
+  for (const t of serverTools) {
+    if (t.type === 'web_search') {
+      out.push({ googleSearch: {} })
+    } else if (t.type === 'code_execution') {
+      out.push({ codeExecution: {} })
+    } else if (t.type === 'url_context') {
+      out.push({ urlContext: {} })
+    } else if (t.type === 'web_fetch') {
+      throw new BrainError(
+        'GeminiProvider: server tool `web_fetch` is Anthropic-only. Use `url_context` for Gemini or route the call to Anthropic.',
+        { context: { provider: 'google' } },
+      )
+    }
+  }
+  return out
+}
+
+function guessMimeFromUrl(
+  url: string,
+  kind: 'image' | 'document' | 'audio',
+): string {
+  const lower = url.toLowerCase().split('?')[0] ?? ''
+  if (kind === 'image') {
+    if (lower.endsWith('.png')) return 'image/png'
+    if (lower.endsWith('.webp')) return 'image/webp'
+    if (lower.endsWith('.gif')) return 'image/gif'
+    if (lower.endsWith('.jpg') || lower.endsWith('.jpeg')) return 'image/jpeg'
+    return 'image/jpeg'
+  }
+  if (kind === 'document') {
+    return 'application/pdf'
+  }
+  // audio
+  if (lower.endsWith('.mp3')) return 'audio/mp3'
+  if (lower.endsWith('.wav')) return 'audio/wav'
+  if (lower.endsWith('.ogg')) return 'audio/ogg'
+  if (lower.endsWith('.flac')) return 'audio/flac'
+  if (lower.endsWith('.webm')) return 'audio/webm'
+  if (lower.endsWith('.aac') || lower.endsWith('.m4a')) return 'audio/aac'
+  return 'audio/mp3'
+}
+
 function fromGeminiParts(parts: readonly Part[]): string | ContentBlock[] {
   const blocks: ContentBlock[] = []
   for (const part of parts) {