mohdel 1.3.2 → 1.5.0

package/js/client/coalesce.js

@@ -0,0 +1,42 @@
+/**
+ * Coalesce the delta events of a `call()` stream, with the facade's
+ * `bufferOpts` rules (`createRealtimeDeltaBuffer`): deltas of one kind
+ * accumulate until `maxChars` is reached or `maxMs` has passed since the
+ * last flush, checked as each delta arrives — there is no timer. A change
+ * of kind, any other event, and the end of the stream flush first, so
+ * order is kept and the terminal event is never held back.
+ *
+ * @module client/coalesce
+ */
+
+/**
+ * @param {AsyncIterable<import('#core/events.js').Event>} events
+ * @param {{maxChars?: number, maxMs?: number}} [bufferOpts]
+ * @returns {AsyncGenerator<import('#core/events.js').Event>}
+ */
+export async function * coalesce (events, { maxChars = 250, maxMs = 10_000 } = {}) {
+  let buffer = ''
+  let kind = 'message'
+  let lastFlush = Date.now()
+
+  const flush = () => {
+    const ev = { type: 'delta', delta: { type: kind, delta: buffer } }
+    buffer = ''
+    lastFlush = Date.now()
+    return ev
+  }
+
+  for await (const ev of events) {
+    if (ev.type !== 'delta') {
+      if (buffer) yield flush()
+      yield ev
+      continue
+    }
+    if (!ev.delta.delta) continue
+    if (buffer && ev.delta.type !== kind) yield flush()
+    kind = ev.delta.type
+    buffer += ev.delta.delta
+    if (buffer.length >= maxChars || Date.now() - lastFlush >= maxMs) yield flush()
+  }
+  if (buffer) yield flush()
+}

package/js/client/index.js

@@ -14,6 +14,7 @@
  */
 
 export { call } from './call.js'
+export { coalesce } from './coalesce.js'
 export { callImage } from './call_image.js'
 export { callTranscription } from './call_transcription.js'
 export { callEmbedding } from './call_embedding.js'

package/js/core/envelope.js

@@ -119,8 +119,14 @@
 
 /**
  * @typedef {object} MessagePart
- * @property {('text'|'reasoning')} type
- * @property {string} text
+ * @property {('text'|'reasoning'|'image')} type
+ * @property {string} [text]
+ *   Required on `text` and `reasoning` parts.
+ * @property {string} [fileUri]
+ *   Required on `image` parts. Same schemes and local-read rules as
+ *   an envelope `images` ref. Accepted on `user` and `tool` messages.
+ * @property {string} [mimeType]
+ *   Required on `image` parts.
  * @property {('5m'|'1h')} [cache]
  *   Prompt-cache marker. On system parts: a breakpoint at this block.
  *   On non-system parts: opts the whole conversation into prefix

package/js/session/adapters/_chat_completions.js

@@ -19,6 +19,8 @@
 import { getSpec } from './_catalog.js'
 import { capOutput } from './_output_cap.js'
 import { classifyProviderError } from './_errors.js'
+import { hasImagePart, loadImage, loadImageParts } from './_images.js'
+import { isTrustedMedia, mediaError, mediaScheme } from './_media.js'
 import { costFor } from './_pricing.js'
 import { cancelledDone } from './_cancelled.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
@@ -75,6 +77,9 @@ const DSML_PARAM_RE = /<\uFF5CDSML\uFF5Cparameter\s+name="([^"]+)"(?:\s+string="
  * @property {(envelope: any, args: any) => void} [mutateArgs]
  *   Last-mile hook to splice provider-specific fields into the
  *   request (e.g. OpenRouter routing prefs).
+ * @property {boolean} [inlineImagesOnly]
+ *   The provider takes images as data URIs only; an `https://` image
+ *   is refused before dispatch.
  */
 
 /**
@@ -88,7 +93,17 @@ export async function * runChatCompletions (envelope, client, config, deps = {})
   const spec = getSpec(catalogKey(envelope.model)) || {}
   const start = String(process.hrtime.bigint())
 
-  const args = buildRequest(envelope, spec, config)
+  let images
+  try {
+    images = await loadChatImages(envelope, config)
+  } catch (e) {
+    deps.log?.warn({ err: e }, `[mohdel:${config.provider}] image load failed`)
+    const typed = /** @type {any} */(e).typed
+    yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: config.provider }) }
+    return
+  }
+
+  const args = buildRequest(envelope, spec, config, images)
   if (config.mutateArgs) config.mutateArgs(envelope, args)
 
   if (config.stream) {
@@ -323,18 +338,65 @@ function finalize ({ envelope, content, toolCalls, usage, finishReason, start, f
   return done
 }
 
+/**
+ * @typedef {object} ChatImages
+ * @property {Map<object, any>} parts  `image` part → rendered `image_url` part
+ * @property {any[]} envelope          Rendered envelope `images` refs
+ */
+
+/**
+ * @param {import('#core/envelope.js').CallEnvelope} envelope
+ * @param {ChatCompletionsConfig} config
+ * @returns {Promise<ChatImages>}
+ */
+async function loadChatImages (envelope, config) {
+  const trusted = isTrustedMedia(envelope)
+  const parts = new Map()
+  for (const [part, loaded] of await loadImageParts(envelope.prompt, { trusted })) {
+    parts.set(part, toChatImagePart(part, loaded, config))
+  }
+  const rendered = []
+  for (const ref of envelope.images ?? []) {
+    rendered.push(toChatImagePart(ref, await loadImage(ref, { trusted }), config))
+  }
+  return { parts, envelope: rendered }
+}
+
+/**
+ * A `data:` URI goes out exactly as the caller sent it.
+ *
+ * @param {{fileUri: string}} ref
+ * @param {import('./_images.js').LoadedImage} loaded
+ * @param {ChatCompletionsConfig} config
+ */
+function toChatImagePart (ref, loaded, config) {
+  if (loaded.url && config.inlineImagesOnly) {
+    throw mediaError(
+      `${config.provider} does not accept image URLs`,
+      'SESSION_INVALID_IMAGE',
+      'send the image as a data: URI or a file:// path'
+    )
+  }
+  const url = mediaScheme(ref.fileUri) === 'file'
+    ? `data:${loaded.mimeType};base64,${loaded.base64}`
+    : ref.fileUri
+  return { type: 'image_url', image_url: { url, detail: 'high' } }
+}
+
 /**
  * @param {import('#core/envelope.js').CallEnvelope} envelope
  * @param {any} spec
  * @param {ChatCompletionsConfig} config
+ * @param {ChatImages} images
  */
-function buildRequest (envelope, spec, config) {
+function buildRequest (envelope, spec, config, images) {
   /** @type {Record<string, any>} */
   const args = {
     model: spec?.model ?? bareOf(envelope.model),
     temperature: 0,
     messages: toChatMessages(envelope.prompt, {
-      reasoningPad: typeof spec?.reasoningContentPlaceholder === 'string' ? spec.reasoningContentPlaceholder : null
+      reasoningPad: typeof spec?.reasoningContentPlaceholder === 'string' ? spec.reasoningContentPlaceholder : null,
+      imageParts: images.parts
     })
   }
 
@@ -342,8 +404,8 @@ function buildRequest (envelope, spec, config) {
     args.max_tokens = envelope.outputBudget
   }
 
-  if (envelope.images?.length) {
-    injectImages(args, envelope.images)
+  if (images.envelope.length) {
+    injectImages(args, images.envelope)
   }
 
   if (envelope.tools?.length) {
@@ -393,8 +455,11 @@ function buildRequest (envelope, spec, config) {
 }
 
 /**
+ * A `tool` message carries text only, so a tool result's images go
+ * into one user message after the run of tool results.
+ *
  * @param {string | import('#core/envelope.js').Message[]} prompt
- * @param {{ reasoningPad?: string | null }} [opts]
+ * @param {{ reasoningPad?: string | null, imageParts: Map<object, any> }} opts
  *   `reasoningPad`: when a string (including `''`), assistant messages without
  *   extractable reasoning get `reasoning_content: <pad>` so providers that
  *   require the field on every assistant turn (e.g. deepseek-v4-pro) accept
@@ -406,42 +471,74 @@ function buildRequest (envelope, spec, config) {
  *   own adapters and have their own roundtrip rules.
  * @returns {Array<any>}
  */
-function toChatMessages (prompt, opts = {}) {
+function toChatMessages (prompt, opts) {
   const pad = typeof opts.reasoningPad === 'string' ? opts.reasoningPad : null
   if (typeof prompt === 'string') return [{ role: 'user', content: prompt }]
-  return prompt.map(m => {
+  const out = []
+  let hoisted = []
+  for (const m of prompt) {
     if (m.role === 'tool') {
-      return {
-        role: 'tool',
-        tool_call_id: m.toolCallId,
-        content: flattenText(m.content)
+      if (Array.isArray(m.content)) {
+        hoisted.push(...m.content.filter(p => p.type === 'image').map(p => opts.imageParts.get(p)))
       }
+    } else if (hoisted.length) {
+      out.push({ role: 'user', content: hoisted })
+      hoisted = []
     }
-    const reasoning = m.role === 'assistant' ? extractReasoning(m.content) : null
-    const reasoningField = reasoning ?? (pad !== null && m.role === 'assistant' ? pad : null)
-    if (m.role === 'assistant' && m.toolCalls?.length) {
-      // Chat Completions assistant turn: optional `content` + the
-      // `tool_calls` array. `arguments` must be a JSON string on
-      // the wire.
-      const msg = {
-        role: 'assistant',
-        content: flattenText(m.content) || '',
-        tool_calls: m.toolCalls.map(tc => ({
-          id: tc.id,
-          type: 'function',
-          function: {
-            name: tc.name,
-            arguments: stringifyToolArgs(tc.arguments)
-          }
-        }))
-      }
-      if (reasoningField !== null) msg.reasoning_content = reasoningField
-      return msg
+    out.push(toChatMessage(m, pad, opts.imageParts))
+  }
+  if (hoisted.length) out.push({ role: 'user', content: hoisted })
+  return out
+}
+
+/**
+ * @param {import('#core/envelope.js').Message} m
+ * @param {string | null} pad
+ * @param {Map<object, any>} imageParts
+ */
+function toChatMessage (m, pad, imageParts) {
+  if (m.role === 'tool') {
+    return {
+      role: 'tool',
+      tool_call_id: m.toolCallId,
+      content: flattenText(m.content)
+    }
+  }
+  const reasoning = m.role === 'assistant' ? extractReasoning(m.content) : null
+  const reasoningField = reasoning ?? (pad !== null && m.role === 'assistant' ? pad : null)
+  if (m.role === 'assistant' && m.toolCalls?.length) {
+    // Chat Completions assistant turn: optional `content` + the
+    // `tool_calls` array. `arguments` must be a JSON string on
+    // the wire.
+    const msg = {
+      role: 'assistant',
+      content: flattenText(m.content) || '',
+      tool_calls: m.toolCalls.map(tc => ({
+        id: tc.id,
+        type: 'function',
+        function: {
+          name: tc.name,
+          arguments: stringifyToolArgs(tc.arguments)
+        }
+      }))
     }
-    const msg = { role: m.role, content: flattenText(m.content) }
     if (reasoningField !== null) msg.reasoning_content = reasoningField
     return msg
-  })
+  }
+  const content = hasImagePart(m.content) ? toChatContent(m.content, imageParts) : flattenText(m.content)
+  const msg = { role: m.role, content }
+  if (reasoningField !== null) msg.reasoning_content = reasoningField
+  return msg
+}
+
+/**
+ * @param {import('#core/envelope.js').MessagePart[]} content
+ * @param {Map<object, any>} imageParts
+ */
+function toChatContent (content, imageParts) {
+  return content
+    .filter(p => p.type === 'image' || (p.type === 'text' && p.text))
+    .map(p => p.type === 'image' ? imageParts.get(p) : { type: 'text', text: p.text })
 }
 
 /** @param {unknown} args */
@@ -466,17 +563,9 @@ function extractReasoning (content) {
 
 /**
  * @param {any} args
- * @param {import('#core/envelope.js').MediaRef[]} images
+ * @param {any[]} blocks  Rendered `image_url` parts
  */
-function injectImages (args, images) {
-  const blocks = images
-    .filter(i => i?.fileUri && i?.mimeType)
-    .map(i => ({
-      type: 'image_url',
-      image_url: { url: i.fileUri, detail: 'high' }
-    }))
-  if (!blocks.length) return
-
+function injectImages (args, blocks) {
   // Append to the last user message; fallback to creating one.
   for (let i = args.messages.length - 1; i >= 0; i--) {
     const m = args.messages[i]

package/js/session/adapters/_images.js

@@ -37,13 +37,43 @@ const IMAGE_ERROR = 'SESSION_INVALID_IMAGE'
 export async function loadImages (images, opts = {}) {
   if (!images || !Array.isArray(images)) return []
   const out = []
-  for (const img of images) {
-    if (!img?.fileUri || !img?.mimeType) continue
-    out.push(await loadImage(img, opts))
+  for (const img of images) out.push(await loadImage(img, opts))
+  return out
+}
+
+/**
+ * Loads every `image` part in a message prompt, keyed by the part
+ * object so a message builder can render each one where it sits.
+ *
+ * @param {string | import('#core/envelope.js').Message[]} prompt
+ * @param {{trusted?: boolean}} [opts]
+ * @returns {Promise<Map<object, LoadedImage>>}
+ */
+export async function loadImageParts (prompt, opts = {}) {
+  const out = new Map()
+  if (typeof prompt === 'string') return out
+  for (const m of prompt) {
+    if (!Array.isArray(m.content)) continue
+    for (const p of m.content) {
+      if (p.type !== 'image') continue
+      if (m.role !== 'user' && m.role !== 'tool') {
+        throw mediaError(
+          `image parts are not accepted on ${m.role} messages`,
+          IMAGE_ERROR,
+          'put images on user or tool messages'
+        )
+      }
+      out.set(p, await loadImage(p, opts))
+    }
   }
   return out
 }
 
+/** @param {string | import('#core/envelope.js').MessagePart[]} content */
+export function hasImagePart (content) {
+  return Array.isArray(content) && content.some(p => p.type === 'image')
+}
+
 /**
  * @param {{fileUri: string, mimeType: string}} image
  * @param {{trusted?: boolean}} [opts]
@@ -51,6 +81,9 @@ export async function loadImages (images, opts = {}) {
  */
 export async function loadImage (image, opts = {}) {
   const { fileUri, mimeType } = image
+  if (!fileUri || !mimeType) {
+    throw mediaError('image is missing fileUri or mimeType', IMAGE_ERROR)
+  }
   switch (mediaScheme(fileUri)) {
     case 'file': {
       const { bytes } = await readLocalMedia(fileUri, { type: IMAGE_ERROR, trusted: opts.trusted })

package/js/session/adapters/anthropic.js

@@ -27,7 +27,7 @@ import {
 import { cancelledDone } from './_cancelled.js'
 import { getSpec } from './_catalog.js'
 import { classifyProviderError } from './_errors.js'
-import { loadImages } from './_images.js'
+import { hasImagePart, loadImageParts, loadImages } from './_images.js'
 import { isTrustedMedia } from './_media.js'
 import { costFor } from './_pricing.js'
 import { catalogKey, bareOf } from '#core/model-id.js'
@@ -92,22 +92,24 @@ export async function * anthropic (envelope, deps = {}) {
   const start = String(process.hrtime.bigint())
   let first = null
 
-  const { system, conversation, conversationCacheTtl } = splitPrompt(envelope.prompt)
-
-  // Attach images to the last user message before building the request.
-  if (envelope.images?.length) {
-    try {
-      const loaded = await loadImages(envelope.images, { trusted: isTrustedMedia(envelope) })
-      const blocks = loaded.map(toAnthropicImageBlock).filter(Boolean)
-      if (blocks.length) injectImageBlocks(conversation, blocks)
-    } catch (e) {
-      log?.warn({ err: e }, '[mohdel:anthropic] image load failed')
-      const typed = /** @type {any} */(e).typed
-      yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'anthropic' }) }
-      return
+  let imageParts
+  let imageBlocks = []
+  try {
+    const trusted = isTrustedMedia(envelope)
+    imageParts = await loadImageParts(envelope.prompt, { trusted })
+    if (envelope.images?.length) {
+      imageBlocks = (await loadImages(envelope.images, { trusted })).map(toAnthropicImageBlock)
     }
+  } catch (e) {
+    log?.warn({ err: e }, '[mohdel:anthropic] image load failed')
+    const typed = /** @type {any} */(e).typed
+    yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'anthropic' }) }
+    return
   }
 
+  const { system, conversation, conversationCacheTtl } = splitPrompt(envelope.prompt, imageParts)
+  if (imageBlocks.length) injectImageBlocks(conversation, imageBlocks)
+
   const request = buildRequest(envelope, conversation, system, conversationCacheTtl)
 
   // Array + join, not `+=`: per-delta cons-strings are the cost on a long stream.
@@ -435,7 +437,7 @@ function placeConversationBreakpoints (messages, ttl) {
 }
 
 /** @param {string | import('#core/envelope.js').Message[]} prompt */
-function splitPrompt (prompt) {
+function splitPrompt (prompt, imageParts) {
   if (typeof prompt === 'string') {
     return { system: '', conversation: [{ role: 'user', content: prompt }], conversationCacheTtl: null }
   }
@@ -488,7 +490,7 @@ function splitPrompt (prompt) {
         content: [{
           type: 'tool_result',
           tool_use_id: m.toolCallId ?? '',
-          content: flattenText(m.content)
+          content: hasImagePart(m.content) ? toAnthropicContent(m.content, imageParts) : flattenText(m.content)
         }]
       })
     } else if (m.role === 'assistant' && m.toolCalls?.length) {
@@ -509,7 +511,7 @@ function splitPrompt (prompt) {
     } else {
       conversation.push({
         role: m.role,
-        content: toAnthropicContent(m.content)
+        content: toAnthropicContent(m.content, imageParts)
       })
     }
   }
@@ -529,11 +531,15 @@ function flattenText (content) {
   return content.filter(p => p.type === 'text' && p.text).map(p => p.text).join('\n')
 }
 
-/** @param {string | import('#core/envelope.js').MessagePart[]} content */
-function toAnthropicContent (content) {
+/**
+ * @param {string | import('#core/envelope.js').MessagePart[]} content
+ * @param {Map<object, import('./_images.js').LoadedImage>} imageParts
+ */
+function toAnthropicContent (content, imageParts) {
   if (typeof content === 'string') return content
-  return content.map(p => {
+  return content.filter(p => p.type !== 'reasoning').map(p => {
     if (p.type === 'text') return { type: 'text', text: p.text ?? '' }
+    if (p.type === 'image') return toAnthropicImageBlock(imageParts.get(p))
     throw new Error(`unsupported content part type: ${p.type}`)
   })
 }

package/js/session/adapters/cerebras.js

@@ -20,7 +20,8 @@ export async function * cerebras (envelope, deps = {}) {
     stream: true,
     provider: 'cerebras',
     toolChoiceFlavor: 'cerebras',
-    reasoningField: 'cerebras_zai'
+    reasoningField: 'cerebras_zai',
+    inlineImagesOnly: true
   }, {
     signal: deps.signal,
     log: deps.log,

package/js/session/adapters/gemini.js

@@ -28,7 +28,7 @@ import {
 import { cancelledDone } from './_cancelled.js'
 import { getSpec } from './_catalog.js'
 import { classifyProviderError } from './_errors.js'
-import { loadImages } from './_images.js'
+import { loadImageParts, loadImages } from './_images.js'
 import { isTrustedMedia } from './_media.js'
 import { loadVideos } from './_videos.js'
 import { costFor } from './_pricing.js'
@@ -51,21 +51,24 @@ export async function * gemini (envelope, deps = {}) {
   const start = String(process.hrtime.bigint())
   let first = null
 
-  const { systemInstruction, contents } = buildContents(envelope.prompt)
-
-  if (envelope.images?.length) {
-    try {
-      const loaded = await loadImages(envelope.images, { trusted: isTrustedMedia(envelope) })
-      const parts = loaded.map(toGeminiImagePart).filter(Boolean)
-      if (parts.length) injectParts(contents, parts)
-    } catch (e) {
-      log?.warn({ err: e }, '[mohdel:gemini] image load failed')
-      const typed = /** @type {any} */(e).typed
-      yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'gemini' }) }
-      return
+  let imageParts
+  let imageInputs = []
+  try {
+    const trusted = isTrustedMedia(envelope)
+    imageParts = await loadImageParts(envelope.prompt, { trusted })
+    if (envelope.images?.length) {
+      imageInputs = (await loadImages(envelope.images, { trusted })).map(toGeminiImagePart)
     }
+  } catch (e) {
+    log?.warn({ err: e }, '[mohdel:gemini] image load failed')
+    const typed = /** @type {any} */(e).typed
+    yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'gemini' }) }
+    return
   }
 
+  const { systemInstruction, contents } = buildContents(envelope.prompt, imageParts)
+  if (imageInputs.length) injectParts(contents, imageInputs)
+
   if (envelope.videos?.length) {
     try {
       const parts = await loadVideos(envelope.videos, {
@@ -278,7 +281,15 @@ function buildRequest (envelope, contents, systemInstruction) {
 }
 
 /** @param {string | import('#core/envelope.js').Message[]} prompt */
-function buildContents (prompt) {
+/**
+ * A tool result's images go into one user content after the run of
+ * tool results: only Gemini 3 accepts media inside a
+ * `functionResponse`.
+ *
+ * @param {string | import('#core/envelope.js').Message[]} prompt
+ * @param {Map<object, import('./_images.js').LoadedImage>} imageParts
+ */
+function buildContents (prompt, imageParts) {
   if (typeof prompt === 'string') {
     return {
       systemInstruction: '',
@@ -289,10 +300,20 @@ function buildContents (prompt) {
   const systemParts = []
   /** @type {Array<{role: string, parts: any[]}>} */
   const contents = []
+  /** @type {Array<any>} */
+  let hoisted = []
+  const flushHoisted = () => {
+    if (hoisted.length) contents.push({ role: 'user', parts: hoisted })
+    hoisted = []
+  }
   for (const m of prompt) {
+    if (m.role !== 'tool') flushHoisted()
     if (m.role === 'system') {
       systemParts.push(flattenText(m.content))
     } else if (m.role === 'tool') {
+      if (Array.isArray(m.content)) {
+        hoisted.push(...m.content.filter(p => p.type === 'image').map(p => toGeminiImagePart(imageParts.get(p))))
+      }
       contents.push({
         role: 'user',
         parts: [{
@@ -322,10 +343,11 @@ function buildContents (prompt) {
     } else {
       contents.push({
         role: mapRole(m.role),
-        parts: toGeminiParts(m.content)
+        parts: toGeminiParts(m.content, imageParts)
       })
     }
   }
+  flushHoisted()
   return {
     systemInstruction: systemParts.filter(Boolean).join('\n\n'),
     contents
@@ -355,11 +377,15 @@ function flattenText (content) {
   return content.filter(p => p.type === 'text' && p.text).map(p => p.text).join('\n')
 }
 
-/** @param {string | import('#core/envelope.js').MessagePart[]} content */
-function toGeminiParts (content) {
+/**
+ * @param {string | import('#core/envelope.js').MessagePart[]} content
+ * @param {Map<object, import('./_images.js').LoadedImage>} imageParts
+ */
+function toGeminiParts (content, imageParts) {
   if (typeof content === 'string') return [{ text: content }]
-  return content.map(p => {
+  return content.filter(p => p.type !== 'reasoning').map(p => {
     if (p.type === 'text') return { text: p.text ?? '' }
+    if (p.type === 'image') return toGeminiImagePart(imageParts.get(p))
     throw new Error(`unsupported content part type: ${p.type}`)
   })
 }

package/js/session/adapters/openai.js

@@ -28,7 +28,7 @@ import {
 import { cancelledDone } from './_cancelled.js'
 import { getSpec } from './_catalog.js'
 import { classifyProviderError } from './_errors.js'
-import { loadImages } from './_images.js'
+import { hasImagePart, loadImageParts, loadImages } from './_images.js'
 import { isTrustedMedia } from './_media.js'
 import { costFor } from './_pricing.js'
 import { catalogKey, providerOf, bareOf } from '#core/model-id.js'
@@ -54,21 +54,26 @@ export async function * openai (envelope, deps = {}) {
   const start = String(process.hrtime.bigint())
   let first = null
 
-  const { instructions, input } = splitPrompt(envelope.prompt)
-
-  if (envelope.images?.length) {
-    try {
-      const loaded = await loadImages(envelope.images, { trusted: isTrustedMedia(envelope) })
-      const parts = loaded.map(toOpenAIImagePart).filter(Boolean)
-      if (parts.length) injectImageParts(input, parts)
-    } catch (e) {
-      log?.warn({ err: e }, '[mohdel:openai] image load failed')
-      const typed = /** @type {any} */(e).typed
-      yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'openai' }) }
-      return
+  let imageParts
+  let imageInputs = []
+  try {
+    const trusted = isTrustedMedia(envelope)
+    imageParts = await loadImageParts(envelope.prompt, { trusted })
+    if (envelope.images?.length) {
+      imageInputs = (await loadImages(envelope.images, { trusted })).map(toOpenAIImagePart)
     }
+  } catch (e) {
+    log?.warn({ err: e }, '[mohdel:openai] image load failed')
+    const typed = /** @type {any} */(e).typed
+    yield { type: 'error', error: typed || classifyProviderError(e, envelope.auth?.key, { provider: 'openai' }) }
+    return
   }
 
+  const { instructions, input } = splitPrompt(envelope.prompt, imageParts, {
+    toolResultImages: providerOf(envelope.model) === 'openai'
+  })
+  if (imageInputs.length) injectImageParts(input, imageInputs)
+
   const request = buildRequest(envelope, input, instructions)
 
   // Array + join, not `+=`: per-delta cons-strings are the cost on a long stream.
@@ -358,8 +363,16 @@ function buildRequest (envelope, input, instructions) {
   return request
 }
 
-/** @param {string | import('#core/envelope.js').Message[]} prompt */
-function splitPrompt (prompt) {
+/**
+ * `toolResultImages`: whether a `function_call_output` may carry
+ * images. Where it may not, a tool result's images go into one user
+ * message after the run of tool results.
+ *
+ * @param {string | import('#core/envelope.js').Message[]} prompt
+ * @param {Map<object, import('./_images.js').LoadedImage>} imageParts
+ * @param {{toolResultImages: boolean}} opts
+ */
+function splitPrompt (prompt, imageParts, opts) {
   if (typeof prompt === 'string') {
     return { instructions: '', input: [{ role: 'user', content: prompt }] }
   }
@@ -367,14 +380,30 @@ function splitPrompt (prompt) {
   const systemParts = []
   /** @type {Array<any>} */
   const input = []
+  /** @type {Array<any>} */
+  let hoisted = []
+  const flushHoisted = () => {
+    if (hoisted.length) input.push({ role: 'user', content: hoisted })
+    hoisted = []
+  }
   for (const m of prompt) {
+    if (m.role !== 'tool') flushHoisted()
     if (m.role === 'system') {
       systemParts.push(flattenText(m.content))
     } else if (m.role === 'tool') {
+      let output = flattenText(m.content)
+      if (hasImagePart(m.content)) {
+        const content = toInputContent('user', m.content, imageParts)
+        if (opts.toolResultImages) {
+          output = content
+        } else {
+          hoisted.push(...content.filter(p => p.type === 'input_image'))
+        }
+      }
       input.push({
         type: 'function_call_output',
         call_id: m.toolCallId ?? '',
-        output: flattenText(m.content)
+        output
       })
     } else if (m.role === 'assistant' && m.toolCalls?.length) {
       // Responses API wants a message item (if any text) followed
@@ -398,10 +427,11 @@ function splitPrompt (prompt) {
     } else {
       input.push({
         role: m.role,
-        content: toInputContent(m.role, m.content)
+        content: toInputContent(m.role, m.content, imageParts)
       })
     }
   }
+  flushHoisted()
   return { instructions: systemParts.filter(Boolean).join('\n\n'), input }
 }
 
@@ -430,12 +460,14 @@ function stringifyToolArgs (args) {
 /**
  * @param {string} role
  * @param {string | import('#core/envelope.js').MessagePart[]} content
+ * @param {Map<object, import('./_images.js').LoadedImage>} imageParts
  */
-function toInputContent (role, content) {
+function toInputContent (role, content, imageParts) {
   if (typeof content === 'string') return content
   const partType = role === 'assistant' ? 'output_text' : 'input_text'
-  return content.map(p => {
+  return content.filter(p => p.type !== 'reasoning').map(p => {
     if (p.type === 'text') return { type: partType, text: p.text ?? '' }
+    if (p.type === 'image') return toOpenAIImagePart(imageParts.get(p))
     throw new Error(`unsupported content part type: ${p.type}`)
   })
 }

package/js/session/driver.js

@@ -19,6 +19,7 @@ import { run } from './run.js'
 import { runImage } from './run_image.js'
 import { runTranscription } from './run_transcription.js'
 import { runEmbedding } from './run_embedding.js'
+import { runInfo } from './run_info.js'
 import { setCatalog } from './adapters/_catalog.js'
 
 // Bounded memory for pre-dequeue cancels. Hostile/buggy supervisors
@@ -232,6 +233,13 @@ export async function drive (stdin, stdout) {
         } else {
           await writeLine({ type: 'error', error: out.error })
         }
+      } else if (envelope.op === 'info') {
+        const out = await runInfo(envelope)
+        if (out.ok) {
+          await writeLine({ type: 'info_done', result: out.result })
+        } else {
+          await writeLine({ type: 'error', error: out.error })
+        }
       } else {
         for await (const ev of run(envelope, { signal: controller.signal })) {
           await writeLine(ev)

package/js/session/run.js

@@ -61,7 +61,7 @@ import { STATUS_INCOMPLETE, WARNING_CANCELLED } from '#core/status.js'
 // name is checked against the known list before it reaches the import path —
 // it comes off the envelope, and a computed specifier must never take an
 // arbitrary string.
-const loadAdapter = async (provider) => {
+export const loadAdapter = async (provider) => {
   if (!ADAPTER_NAMES.includes(provider)) throw new Error(`unknown provider: ${provider}`)
   const module = await import(`./adapters/${provider}.js`)
   return module[provider]
@@ -312,7 +312,7 @@ export async function * run (envelope, {
  *   error?: import('#core/events.js').ErrorEvent
  * }}
  */
-function normalizeModelId (envelope, resolveSpec) {
+export function normalizeModelId (envelope, resolveSpec) {
   // A bare id that itself contains `:` or `@` is a catalog key in its
   // own right; resolving the whole string first stops it being split
   // into a base plus a suffix that was never meant as one.
@@ -374,7 +374,7 @@ function normalizeModelId (envelope, resolveSpec) {
  * @param {any} adapter
  * @returns {import('#core/events.js').ErrorEvent | undefined}
  */
-function speedError (key, speed, spec, provider, adapter) {
+export function speedError (key, speed, spec, provider, adapter) {
   if (!hasSpeed(spec, speed)) {
     const available = speedNames(spec)
     const detail = available.length

package/js/session/run_info.js

@@ -0,0 +1,58 @@
+/**
+ * Catalog entry lookup for a model id, answered by the session that
+ * holds the catalog. An embedder of the gate supplies or loads the
+ * catalog but never reads it back; this is how it asks.
+ *
+ * The answer is the entry a call with that `model` would run on: the
+ * same lookup and the same effort and speed-lane checks as `run.js`,
+ * so an id this accepts is one a call accepts. Shape follows the
+ * facade's `info()`: the entry, plus `speed` when a lane is named.
+ *
+ * @module session/run_info
+ */
+
+import { getSpec } from './adapters/_catalog.js'
+import { providerOf } from '#core/model-id.js'
+import { loadAdapter, normalizeModelId, speedError } from './run.js'
+
+/**
+ * @param {{model: string}} request
+ * @param {{
+ *   resolveSpec?: (key: string) => any,
+ *   resolveAdapter?: (provider: string) => Promise<any>
+ * }} [options]
+ * @returns {Promise<
+ *   | {ok: true, result: Record<string, any> | null}
+ *   | {ok: false, error: import('#core/errors.js').TypedError}
+ * >}
+ */
+export async function runInfo ({ model }, {
+  resolveSpec = getSpec,
+  resolveAdapter = loadAdapter
+} = {}) {
+  const norm = normalizeModelId(/** @type {any} */({ model }), resolveSpec)
+  if (norm.error) return { ok: false, error: norm.error.error }
+  if (!norm.spec) return { ok: true, result: null }
+
+  const speed = norm.envelope.speed
+  if (!speed) return { ok: true, result: { ...norm.spec } }
+
+  const provider = providerOf(norm.key)
+  let adapter
+  try {
+    adapter = await resolveAdapter(provider)
+  } catch (e) {
+    return {
+      ok: false,
+      error: {
+        message: e instanceof Error ? e.message : String(e),
+        severity: 'error',
+        retryable: false,
+        type: 'SESSION_UNKNOWN_PROVIDER'
+      }
+    }
+  }
+  const denied = speedError(norm.key, speed, norm.spec, provider, adapter)
+  if (denied) return { ok: false, error: denied.error }
+  return { ok: true, result: { ...norm.spec, speed } }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mohdel",
-  "version": "1.3.2",
+  "version": "1.5.0",
   "license": "MIT",
   "author": {
     "name": "Christophe Le Bars",
@@ -109,6 +109,7 @@
     "test:provider": "vitest run test/integration/provider.test.js",
     "test:multiturn": "vitest run test/integration/multiturn.test.js",
     "test:vision": "vitest run test/integration/vision.test.js",
+    "test:vision:catalog": "vitest run test/integration/vision-catalog.test.js",
     "test:live": "vitest run test/live"
   },
   "release-it": {
@@ -135,17 +136,17 @@
     "@opentelemetry/exporter-trace-otlp-grpc": "^0.222.0",
     "@opentelemetry/sdk-node": "^0.222.0",
     "chalk": "^6.0.0",
-    "mohdel-thin-gate-linux-x64-gnu": "1.3.2"
+    "mohdel-thin-gate-linux-x64-gnu": "1.5.0"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "^0.125.0",
+    "@anthropic-ai/sdk": "^0.128.0",
     "@cerebras/cerebras_cloud_sdk": "^1.91.0",
     "@clack/prompts": "^1.8.1",
-    "@google/genai": "^2.22.0",
+    "@google/genai": "^2.24.0",
     "@opentelemetry/api": "^1.9.1",
     "env-paths": "^4.0.0",
     "groq-sdk": "^1.6.0",
-    "openai": "^7.15.0",
+    "openai": "^7.23.0",
     "undici": "^7.29.0"
   },
   "lint-staged": {

package/src/lib/utils.js

@@ -64,7 +64,9 @@ export const createRealtimeDeltaBuffer = (handler, opts = {}) => {
 
   const push = (type, delta) => {
     if (!handler || !delta) return
-    lastType = type || lastType || 'message'
+    const kind = type || lastType
+    if (buffer && kind !== lastType) flushInternal(true)
+    lastType = kind
     buffer += delta
     flushInternal(false)
   }