ask-llm-providers 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ceebc12e911bceb83ec1528b66716605dbdb2e838d69051751ea9455f21cb018
-  data.tar.gz: f435785c44e2cc895c8dce3280ad1220dd19445dea044fd11e0070f11605b476
+  metadata.gz: d6670dc1a64e5b158fa6cb25b8acca8df2daad65bd866d1b67594425d6407c30
+  data.tar.gz: 195892b5274ea261223f22d221f462c1229f719c3834812cd510918a21ce1f5f
 SHA512:
-  metadata.gz: 299eb23798c50249fe2dce48981418516bf9630235eaaa25ffec01e6753aeaefe9c2867c02c697ee97084e092ed1e5ee4b7051cf1ef29b08efd095af402c9f3c
-  data.tar.gz: 196fe1897eeccd769a1ce8b097a0ce238199abb50fd4d2e0d16378a4e83a13b6599765f60a2708d3a52f6f32245f96f098247bcd31a20591e1ba4ef50dbb591f
+  metadata.gz: f8bcd6ca9421703ec7a075dcc5431e487d420c610fa03c93ddf9c65918b798455334b92aa545e69da71831eca98eb993c10b0cf0b49e5fb4862d7b3810783980
+  data.tar.gz: 4272c5ccce88ee16f61097556838f40d25d64436f2c3d2e3d3db6cd08434b344944180c56e9ae66df98341ce66aff3b6379d0a563ad3b6913efea9eacab09b22

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [0.10.0] — 2026-07-26
+
+### Added
+
+- **Multi-modal content block support** — OpenAI and Anthropic providers now detect content block arrays in messages and serialize them to their native wire formats.
+
+  **OpenAI**: `image` (URL/base64/file_id) → `image_url`, `audio` (URL/base64) → `input_audio`, `video` → `image_url`, `file` → text.
+
+  **Anthropic**: `image` (base64/URL) → native `source` blocks, `image` with `file_id` → URL, `audio`/`video` → text fallback, `file` → text.
+
+### Changed
+
+- `format_message` in both providers now handles Array `content` (from `Ask::Content` blocks in `Message#to_h`). Plain string content is unchanged — fully backward compatible.
+
+### Tested
+
+- 20 new tests for provider-specific content block formatting
+- 503 total tests, 0 failures
+
 ## [0.9.0] — 2026-07-21
 
 ### Added

data/lib/ask/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module LLM
-    VERSION = "0.9.0"
+    VERSION = "0.10.0"
   end
 end

data/lib/ask/provider/anthropic.rb

@@ -163,10 +163,11 @@ module Ask
 
       def format_message(msg)
         role = (msg[:role] || msg["role"]).to_s
-        content = msg[:content] || msg["content"]
+        raw_content = msg[:content] || msg["content"]
 
         if msg[:tool_calls] || msg["tool_calls"]
           tc = msg[:tool_calls] || msg["tool_calls"]
+          content = raw_content.is_a?(Array) ? raw_content.map { |b| format_anthropic_content_block(b) } : raw_content
           return {
             role:,
             content:,
@@ -182,17 +183,63 @@ module Ask
         end
 
         if msg[:tool_call_id] || msg["tool_call_id"]
+          content = if raw_content.is_a?(Array)
+            raw_content.map { |b| format_anthropic_content_block(b) }
+          else
+            raw_content || ""
+          end
           return {
             role: "user",
             content: [{
               type: "tool_result",
               tool_use_id: msg[:tool_call_id] || msg["tool_call_id"],
-              content: content || ""
+              content: content
             }]
           }
         end
 
-        { role:, content: }.compact
+        # Multi-modal content blocks
+        if raw_content.is_a?(Array)
+          content = raw_content.map { |b| format_anthropic_content_block(b) }
+          return { role:, content: }
+        end
+
+        { role:, content: raw_content }.compact
+      end
+
+      # Transform a generic content block hash into Anthropic's wire format.
+      # https://docs.anthropic.com/en/docs/build-with-claude/vision
+      def format_anthropic_content_block(block)
+        block = block.transform_keys(&:to_sym) if block.respond_to?(:transform_keys)
+        type = block[:type] || block["type"]
+
+        case type
+        when "text"
+          { type: "text", text: block[:text] || block["text"] }
+        when "image"
+          if block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "image/png"
+            { type: "image", source: { type: "base64", media_type: mime, data: block[:base64] || block["base64"] } }
+          elsif block[:url] || block["url"]
+            mime = block[:mime_type] || block["mime_type"] || "image/jpeg"
+            { type: "image", source: { type: "url", url: block[:url] || block["url"] } }
+          elsif block[:file_id] || block["file_id"]
+            # Anthropic doesn't support file_id for images; pass as URL
+            { type: "image", source: { type: "url", url: block[:file_id] || block["file_id"] } }
+          else
+            block
+          end
+        when "audio", "video"
+          # Anthropic doesn't support audio/video content blocks in messages
+          # Fall back to text description
+          { type: "text", text: "[#{type} content not supported by Anthropic]" }
+        when "file"
+          data = block[:data] || block["data"] || ""
+          filename = block[:filename] ? "[#{block[:filename]}] " : ""
+          { type: "text", text: "#{filename}#{data}" }
+        else
+          block
+        end
       end
 
       private

data/lib/ask/provider/openai.rb

@@ -177,7 +177,16 @@ module Ask
 
       def format_message(msg)
         role = msg[:role] || msg["role"] || :user
-        { role: role.to_s, content: msg[:content] || msg["content"] }.tap do |fm|
+        raw_content = msg[:content] || msg["content"]
+
+        # Multi-modal content blocks — transform to OpenAI's format
+        if raw_content.is_a?(Array)
+          content = raw_content.map { |block| format_openai_content_block(block) }
+        else
+          content = raw_content
+        end
+
+        { role: role.to_s, content: content }.tap do |fm|
           if (tc = msg[:tool_calls] || msg["tool_calls"]) && tc.respond_to?(:any?) && tc.any?
             calls = tc.is_a?(Hash) ? tc.values : tc
             fm[:tool_calls] = calls.map { |t|
@@ -362,6 +371,63 @@ module Ask
         messages.map { |msg| format_message(msg) }
       end
 
+      # Transform a generic content block hash into OpenAI's wire format.
+      # https://platform.openai.com/docs/guides/vision
+      def format_openai_content_block(block)
+        block = block.transform_keys(&:to_sym) if block.respond_to?(:transform_keys)
+        type = block[:type] || block["type"]
+
+        case type
+        when "text"
+          { type: "text", text: block[:text] || block["text"] }
+        when "image"
+          if block[:url] || block["url"]
+            { type: "image_url", image_url: { url: block[:url] || block["url"] } }
+          elsif block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "image/png"
+            data = block[:base64] || block["base64"]
+            { type: "image_url", image_url: { url: "data:#{mime};base64,#{data}" } }
+          elsif block[:file_id] || block["file_id"]
+            { type: "image_url", image_url: { url: block[:file_id] || block["file_id"] } }
+          else
+            block
+          end
+        when "audio"
+          if block[:url] || block["url"]
+            { type: "input_audio", input_audio: { data: block[:url] || block["url"], format: detect_audio_format(block) } }
+          elsif block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "audio/wav"
+            { type: "input_audio", input_audio: { data: block[:base64] || block["base64"], format: mime.split("/").last } }
+          else
+            block
+          end
+        when "file"
+          # OpenAI doesn't have a generic file content block; send as text
+          data = block[:data] || block["data"] || ""
+          filename = block[:filename] ? "[#{block[:filename]}] " : ""
+          { type: "text", text: "#{filename}#{data}" }
+        when "video"
+          # OpenAI supports video via URLs (same as images)
+          if block[:url] || block["url"]
+            { type: "image_url", image_url: { url: block[:url] || block["url"] } }
+          else
+            block
+          end
+        else
+          block
+        end
+      end
+
+      def detect_audio_format(block)
+        mime = block[:mime_type] || block["mime_type"] || ""
+        case mime
+        when /mpeg|mp3/ then "mp3"
+        when /wav/ then "wav"
+        when /opus/ then "opus"
+        else mime.split("/").last || "wav"
+        end
+      end
+
       def chat_nonstream(payload, model)
         response = @http.post("chat/completions") { |r| r.body = payload }
         raise LLM::HTTP.map_error(response.status, response.body, provider: "OpenAI") unless response.success?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-llm-providers
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: ask-auth
   requirement: !ruby/object:Gem::Requirement