ask-llm-providers 0.8.7 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d0b181ef0d5094ee52f234c097cbce2fecad3dcad729c91e6923036483a47cf
-  data.tar.gz: 8afd2d428cbe66ffa17bacdfe65e459acdef03ec8052ab764f7a742f7f453be4
+  metadata.gz: d6670dc1a64e5b158fa6cb25b8acca8df2daad65bd866d1b67594425d6407c30
+  data.tar.gz: 195892b5274ea261223f22d221f462c1229f719c3834812cd510918a21ce1f5f
 SHA512:
-  metadata.gz: 4a4c3741d5d530624c8acde90a60cc01b9c0b5742f219f2a237d8d9de45291b1da98d7cb43d34678c7597865dac54574b0dd36c6f31eae4ec31eeb5f281de57c
-  data.tar.gz: 2577b18aaa21cd12ce02788dbfd53d1a05196bace3e6443127deca449ba477ef9342fa0bd11de43fe3c411cab2784794efbb4707fa60d183a19cbdf11f30bcfc
+  metadata.gz: f8bcd6ca9421703ec7a075dcc5431e487d420c610fa03c93ddf9c65918b798455334b92aa545e69da71831eca98eb993c10b0cf0b49e5fb4862d7b3810783980
+  data.tar.gz: 4272c5ccce88ee16f61097556838f40d25d64436f2c3d2e3d3db6cd08434b344944180c56e9ae66df98341ce66aff3b6379d0a563ad3b6913efea9eacab09b22

data/CHANGELOG.md

@@ -1,3 +1,44 @@
+## [0.10.0] — 2026-07-26
+
+### Added
+
+- **Multi-modal content block support** — OpenAI and Anthropic providers now detect content block arrays in messages and serialize them to their native wire formats.
+
+  **OpenAI**: `image` (URL/base64/file_id) → `image_url`, `audio` (URL/base64) → `input_audio`, `video` → `image_url`, `file` → text.
+
+  **Anthropic**: `image` (base64/URL) → native `source` blocks, `image` with `file_id` → URL, `audio`/`video` → text fallback, `file` → text.
+
+### Changed
+
+- `format_message` in both providers now handles Array `content` (from `Ask::Content` blocks in `Message#to_h`). Plain string content is unchanged — fully backward compatible.
+
+### Tested
+
+- 20 new tests for provider-specific content block formatting
+- 503 total tests, 0 failures
+
+## [0.9.0] — 2026-07-21
+
+### Added
+
+- **Prompt caching support for Anthropic** — `build_request` now accepts `prompt_caching: true`. When enabled, the system prompt is sent as an array with `cache_control: { type: "ephemeral" }` and the last user message content is wrapped to include `cache_control`. Response metadata includes `cache_creation_input_tokens` and `cache_read_input_tokens` from the provider.
+
+- **Prompt caching support for OpenAI** — `parse_response` now extracts `usage.prompt_tokens_details.cached_tokens` into response metadata as `cached_tokens`.
+
+- **Both providers advertise `prompt_caching: true`** in their capabilities.
+
+## [0.8.7] — 2026-07-18
+
+### Fixed
+
+- **`OpenAI#format_message` no longer sends empty `tool_calls: []` on assistant messages** — When a model response has no tool calls, the formatted message previously included `tool_calls: []` in the output. Some providers (notably opencode_go) reject messages with an empty `tool_calls` array. Now the field is only added when there are actual tool calls to report. Fixes multi-turn conversations breaking on all providers that reject empty `tool_calls`.
+
+## [0.8.6] — 2026-07-18
+
+### Added
+
+- **`OpenAICompatible#resolve_credential_from_env_name` now tries both flat key and path segments** — The credential fallback resolves the `api_key_env` name (e.g., `OPENCODE_API_KEY`) both as a flat key (`:opencode_api_key`) and as a nested path (`[:opencode, :api_key]`). This works with the new `Ask::Auth.resolve` multi-name and path segment support from ask-auth 0.2.3.
+
 ## [0.8.5] — 2026-07-18
 
 ### Fixed

data/lib/ask/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module LLM
-    VERSION = "0.8.7"
+    VERSION = "0.10.0"
   end
 end

data/lib/ask/provider/anthropic.rb

@@ -69,7 +69,7 @@ module Ask
 
       def build_request(messages, model:, tools: nil, temperature: nil, stream: nil, schema: nil, **params)
         system_msgs, chat_msgs = messages.partition { |m| (m[:role] || m["role"]).to_s == "system" }
-        system_content = format_system_content(system_msgs)
+        prompt_caching = params.delete(:prompt_caching) || false
 
         payload = {
           model:,
@@ -78,7 +78,21 @@ module Ask
           stream: stream || false
         }
 
-        payload[:system] = system_content if system_content
+        if prompt_caching
+          payload[:system] = format_system_with_caching(system_msgs, chat_msgs)
+          # Mark the last user message for caching (required by Anthropic for conversation caching)
+          if payload[:messages].any?
+            last_user_idx = payload[:messages].rindex { |m| m[:role] == "user" }
+            if last_user_idx
+              content = payload[:messages][last_user_idx][:content]
+              payload[:messages][last_user_idx][:content] = wrap_content_for_caching(content)
+            end
+          end
+        else
+          system_content = format_system_content(system_msgs)
+          payload[:system] = system_content if system_content
+        end
+
         tool_defs = format_tools(tools) if tools&.any?
         payload[:tools] = tool_defs if tool_defs
         payload[:temperature] = temperature if temperature
@@ -102,6 +116,8 @@ module Ask
           stop_sequence: body["stop_sequence"],
           input_tokens: usage["input_tokens"],
           output_tokens: usage["output_tokens"],
+          cache_creation_input_tokens: usage["cache_creation_input_tokens"],
+          cache_read_input_tokens: usage["cache_read_input_tokens"],
           thinking: thinking_blocks.map { |b| b["thinking"] || b["text"] }.compact.join("\n"),
           raw: body
         }.compact
@@ -147,10 +163,11 @@ module Ask
 
       def format_message(msg)
         role = (msg[:role] || msg["role"]).to_s
-        content = msg[:content] || msg["content"]
+        raw_content = msg[:content] || msg["content"]
 
         if msg[:tool_calls] || msg["tool_calls"]
           tc = msg[:tool_calls] || msg["tool_calls"]
+          content = raw_content.is_a?(Array) ? raw_content.map { |b| format_anthropic_content_block(b) } : raw_content
           return {
             role:,
             content:,
@@ -166,17 +183,63 @@ module Ask
         end
 
         if msg[:tool_call_id] || msg["tool_call_id"]
+          content = if raw_content.is_a?(Array)
+            raw_content.map { |b| format_anthropic_content_block(b) }
+          else
+            raw_content || ""
+          end
           return {
             role: "user",
             content: [{
               type: "tool_result",
               tool_use_id: msg[:tool_call_id] || msg["tool_call_id"],
-              content: content || ""
+              content: content
             }]
           }
         end
 
-        { role:, content: }.compact
+        # Multi-modal content blocks
+        if raw_content.is_a?(Array)
+          content = raw_content.map { |b| format_anthropic_content_block(b) }
+          return { role:, content: }
+        end
+
+        { role:, content: raw_content }.compact
+      end
+
+      # Transform a generic content block hash into Anthropic's wire format.
+      # https://docs.anthropic.com/en/docs/build-with-claude/vision
+      def format_anthropic_content_block(block)
+        block = block.transform_keys(&:to_sym) if block.respond_to?(:transform_keys)
+        type = block[:type] || block["type"]
+
+        case type
+        when "text"
+          { type: "text", text: block[:text] || block["text"] }
+        when "image"
+          if block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "image/png"
+            { type: "image", source: { type: "base64", media_type: mime, data: block[:base64] || block["base64"] } }
+          elsif block[:url] || block["url"]
+            mime = block[:mime_type] || block["mime_type"] || "image/jpeg"
+            { type: "image", source: { type: "url", url: block[:url] || block["url"] } }
+          elsif block[:file_id] || block["file_id"]
+            # Anthropic doesn't support file_id for images; pass as URL
+            { type: "image", source: { type: "url", url: block[:file_id] || block["file_id"] } }
+          else
+            block
+          end
+        when "audio", "video"
+          # Anthropic doesn't support audio/video content blocks in messages
+          # Fall back to text description
+          { type: "text", text: "[#{type} content not supported by Anthropic]" }
+        when "file"
+          data = block[:data] || block["data"] || ""
+          filename = block[:filename] ? "[#{block[:filename]}] " : ""
+          { type: "text", text: "#{filename}#{data}" }
+        else
+          block
+        end
       end
 
       private
@@ -203,6 +266,32 @@ module Ask
         texts.join("\n")
       end
 
+      def format_system_with_caching(system_msgs, chat_msgs)
+        texts = system_msgs.map { |m| m[:content] || m["content"] }.compact
+        return nil if texts.empty?
+
+        combined = texts.join("\n")
+        [{ type: "text", text: combined, cache_control: { type: "ephemeral" } }]
+      end
+
+      # Wrap the last user message content for caching.
+      # Plain strings become [{ type: "text", text: content, cache_control: { type: "ephemeral" } }].
+      # Already-structured content blocks get cache_control appended.
+      def wrap_content_for_caching(content)
+        case content
+        when Array
+          content.map { |c|
+            if c.is_a?(Hash)
+              c.merge(cache_control: { type: "ephemeral" })
+            else
+              { type: "text", text: c.to_s, cache_control: { type: "ephemeral" } }
+            end
+          }
+        else
+          [{ type: "text", text: content.to_s, cache_control: { type: "ephemeral" } }]
+        end
+      end
+
       def parse_json(str)
         JSON.parse(str)
       rescue JSON::ParserError

data/lib/ask/provider/openai.rb

@@ -31,8 +31,19 @@ module Ask
 
       def chat(messages, model:, tools: nil, temperature: nil, stream: nil, schema: nil, **params, &block)
         msgs = messages.is_a?(Ask::Conversation) ? messages.to_a : messages
-        payload = build_request(msgs, model:, tools:, temperature:, stream:, schema:, **params)
-        stream ? chat_stream(payload, model, &block) : chat_nonstream(payload, model)
+
+        # Separate provider tools from regular tools
+        regular_tools, provider_tools = split_tools(tools)
+
+        if provider_tools.any?
+          # Use the Responses API when provider tools are involved
+          responses_chat(msgs, model:, regular_tools:, provider_tools:,
+                         temperature:, stream:, schema:, **params, &block)
+        else
+          payload = build_request(msgs, model:, tools: regular_tools,
+                                  temperature:, stream:, schema:, **params)
+          stream ? chat_stream(payload, model, &block) : chat_nonstream(payload, model)
+        end
       end
 
       def embed(texts, model:)
@@ -65,7 +76,8 @@ module Ask
           {
             chat: true, streaming: true, tool_calls: true, vision: true,
             thinking: true, structured_output: true, embed: true,
-            transcribe: true, paint: true, moderate: true
+            transcribe: true, paint: true, moderate: true,
+            prompt_caching: true
           }
         end
 
@@ -94,18 +106,19 @@ module Ask
 
         msg = choice["message"]
         usage = body["usage"] || {}
-        Ask::Message.new(
-          role: :assistant,
-          content: msg["content"],
-          tool_calls: parse_tool_calls(msg["tool_calls"]),
-          metadata: {
-            model: body["model"] || model,
-            finish_reason: choice["finish_reason"],
-            input_tokens: usage["prompt_tokens"],
-            output_tokens: usage["completion_tokens"],
-            raw: body
-          }
-        )
+	        Ask::Message.new(
+	          role: :assistant,
+	          content: msg["content"],
+	          tool_calls: parse_tool_calls(msg["tool_calls"]),
+	          metadata: {
+	            model: body["model"] || model,
+	            finish_reason: choice["finish_reason"],
+	            input_tokens: usage["prompt_tokens"],
+	            output_tokens: usage["completion_tokens"],
+	            cached_tokens: usage.dig("prompt_tokens_details", "cached_tokens"),
+	            raw: body
+	          }
+	        )
       end
 
       def parse_stream(raw, stream, model, &block)
@@ -126,7 +139,15 @@ module Ask
         end
       end
 
+      def split_tools(tools)
+        return [[], []] unless tools&.any?
+
+        tools.partition { |t| !t.respond_to?(:provider_tool?) || !t.provider_tool? }
+      end
+
       def format_tools(tools)
+        return [] unless tools&.any?
+
         tools.map do |t|
           {
             type: "function",
@@ -139,9 +160,33 @@ module Ask
         end
       end
 
+      def format_responses_tools(provider_tools)
+        provider_tools.map do |pt|
+          case pt.name
+          when "web_search"
+            { type: "web_search" }.merge(pt.args)
+          when "file_search"
+            { type: "file_search" }.merge(pt.args)
+          when "code_interpreter"
+            { type: "code_interpreter" }.merge(pt.args)
+          else
+            { type: pt.name }.merge(pt.args)
+          end
+        end
+      end
+
       def format_message(msg)
         role = msg[:role] || msg["role"] || :user
-        { role: role.to_s, content: msg[:content] || msg["content"] }.tap do |fm|
+        raw_content = msg[:content] || msg["content"]
+
+        # Multi-modal content blocks — transform to OpenAI's format
+        if raw_content.is_a?(Array)
+          content = raw_content.map { |block| format_openai_content_block(block) }
+        else
+          content = raw_content
+        end
+
+        { role: role.to_s, content: content }.tap do |fm|
           if (tc = msg[:tool_calls] || msg["tool_calls"]) && tc.respond_to?(:any?) && tc.any?
             calls = tc.is_a?(Hash) ? tc.values : tc
             fm[:tool_calls] = calls.map { |t|
@@ -156,6 +201,139 @@ module Ask
         end.compact
       end
 
+      # Use the OpenAI Responses API, which supports provider-executed tools
+      # like web_search, file_search, and code_interpreter.
+      def responses_chat(messages, model:, regular_tools:, provider_tools:,
+                         temperature: nil, stream: nil, schema: nil, **params, &block)
+        payload = {
+          model: model,
+          input: format_responses_input(messages)
+        }
+
+        all_tools = []
+        all_tools.concat(format_tools(regular_tools)) if regular_tools&.any?
+        all_tools.concat(format_responses_tools(provider_tools)) if provider_tools&.any?
+        payload[:tools] = all_tools if all_tools.any?
+        payload[:temperature] = temperature if temperature
+        payload.merge!(params)
+
+        if stream
+          responses_chat_stream(payload, model, provider_tools, &block)
+        else
+          responses_chat_nonstream(payload, model, provider_tools)
+        end
+      end
+
+      def responses_chat_nonstream(payload, model, provider_tools)
+        response = @http.post("responses") { |r| r.body = payload }
+        raise LLM::HTTP.map_error(response.status, response.body, provider: "OpenAI") unless response.success?
+
+        body = response.body
+        output = body["output"] || []
+
+        # Extract text content and provider-executed tool results
+        text_parts = output.select { |o| o["type"] == "message" }
+        content = text_parts.flat_map { |m| (m["content"] || []) }
+                            .select { |c| c["type"] == "output_text" }
+                            .map { |c| c["text"] }
+                            .join
+
+        # Extract provider-executed tool results
+        provider_results = extract_responses_provider_results(output, provider_tools)
+
+        # Extract regular tool calls
+        regular_calls = extract_responses_tool_calls(output)
+
+        usage = body["usage"] || {}
+        Ask::Message.new(
+          role: :assistant,
+          content: content,
+          tool_calls: regular_calls,
+          metadata: {
+            model: body["model"] || model,
+            finish_reason: body.dig("status"),
+            input_tokens: usage["input_tokens"],
+            output_tokens: usage["output_tokens"],
+            provider_results: provider_results,
+            raw: body
+          }
+        )
+      end
+
+      def responses_chat_stream(payload, model, provider_tools, &block)
+        # Streaming with the Responses API — for now, fall back to non-streaming
+        # and return the full result. Full streaming support can be added later.
+        responses_chat_nonstream(payload, model, provider_tools)
+      end
+
+      def format_responses_input(messages)
+        messages.map do |msg|
+          role = msg[:role] || msg["role"] || "user"
+          content = msg[:content] || msg["content"] || ""
+
+          entry = { role: role.to_s }
+          entry[:content] = [{ type: "input_text", text: content.to_s }]
+
+          # Handle tool calls in assistant messages
+          if (tc = msg[:tool_calls] || msg["tool_calls"]) && tc.respond_to?(:any?) && tc.any?
+            calls = tc.is_a?(Hash) ? tc.values : tc
+            entry[:content] = calls.map { |t|
+              id = t.respond_to?(:id) ? t.id : (t[:id] || t["id"])
+              name = t.respond_to?(:name) ? t.name : (t[:name] || t["name"] || t.dig(:function, :name))
+              raw_args = t.respond_to?(:arguments) ? t.arguments : (t[:arguments] || t["arguments"] || t.dig(:function, :arguments))
+              args = raw_args.is_a?(String) ? raw_args : JSON.generate(raw_args)
+              { type: "function_call", id: id, name: name, arguments: args, status: "completed" }
+            }
+          end
+
+          # Handle tool results
+          if (tid = msg[:tool_call_id] || msg["tool_call_id"])
+            entry[:content] = [{ type: "function_call_output", id: tid, output: content.to_s }]
+          end
+
+          entry
+        end
+      end
+
+      def extract_responses_provider_results(output, provider_tools)
+        results = {}
+        provider_tool_names = provider_tools.map(&:name)
+
+        output.each do |item|
+          case item["type"]
+          when "web_search_call"
+            result_item = output.find { |o| o["type"] == "web_search_result" && o["id"] == item["id"] }
+            if result_item
+              results[item["id"]] = {
+                provider_executed: true,
+                tool_name: "web_search",
+                message: result_item.to_s,
+                status: "success"
+              }
+            end
+          when "file_search_call"
+            result_item = output.find { |o| o["type"] == "file_search_result" && o["id"] == item["id"] }
+            if result_item
+              results[item["id"]] = {
+                provider_executed: true,
+                tool_name: "file_search",
+                message: result_item.to_s,
+                status: "success"
+              }
+            end
+          when "function_call"
+            # Regular tool call — handled elsewhere
+          end
+        end
+        results
+      end
+
+      def extract_responses_tool_calls(output)
+        output.select { |o| o["type"] == "function_call" }.map do |fc|
+          { id: fc["id"], type: "function", name: fc["name"], arguments: fc["arguments"] }
+        end
+      end
+
       private
 
       def extract_provider_keys(config)
@@ -193,6 +371,63 @@ module Ask
         messages.map { |msg| format_message(msg) }
       end
 
+      # Transform a generic content block hash into OpenAI's wire format.
+      # https://platform.openai.com/docs/guides/vision
+      def format_openai_content_block(block)
+        block = block.transform_keys(&:to_sym) if block.respond_to?(:transform_keys)
+        type = block[:type] || block["type"]
+
+        case type
+        when "text"
+          { type: "text", text: block[:text] || block["text"] }
+        when "image"
+          if block[:url] || block["url"]
+            { type: "image_url", image_url: { url: block[:url] || block["url"] } }
+          elsif block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "image/png"
+            data = block[:base64] || block["base64"]
+            { type: "image_url", image_url: { url: "data:#{mime};base64,#{data}" } }
+          elsif block[:file_id] || block["file_id"]
+            { type: "image_url", image_url: { url: block[:file_id] || block["file_id"] } }
+          else
+            block
+          end
+        when "audio"
+          if block[:url] || block["url"]
+            { type: "input_audio", input_audio: { data: block[:url] || block["url"], format: detect_audio_format(block) } }
+          elsif block[:base64] || block["base64"]
+            mime = block[:mime_type] || block["mime_type"] || "audio/wav"
+            { type: "input_audio", input_audio: { data: block[:base64] || block["base64"], format: mime.split("/").last } }
+          else
+            block
+          end
+        when "file"
+          # OpenAI doesn't have a generic file content block; send as text
+          data = block[:data] || block["data"] || ""
+          filename = block[:filename] ? "[#{block[:filename]}] " : ""
+          { type: "text", text: "#{filename}#{data}" }
+        when "video"
+          # OpenAI supports video via URLs (same as images)
+          if block[:url] || block["url"]
+            { type: "image_url", image_url: { url: block[:url] || block["url"] } }
+          else
+            block
+          end
+        else
+          block
+        end
+      end
+
+      def detect_audio_format(block)
+        mime = block[:mime_type] || block["mime_type"] || ""
+        case mime
+        when /mpeg|mp3/ then "mp3"
+        when /wav/ then "wav"
+        when /opus/ then "opus"
+        else mime.split("/").last || "wav"
+        end
+      end
+
       def chat_nonstream(payload, model)
         response = @http.post("chat/completions") { |r| r.body = payload }
         raise LLM::HTTP.map_error(response.status, response.body, provider: "OpenAI") unless response.success?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-llm-providers
 version: !ruby/object:Gem::Version
-  version: 0.8.7
+  version: 0.10.0
 platform: ruby
 authors:
 - Kaka Ruto
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   name: ask-auth
   requirement: !ruby/object:Gem::Requirement