llm_gateway 0.3.0 → 0.4.0

data/Rakefile

@@ -10,10 +10,9 @@ require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
 begin
-  require "gem/release"
-
   desc "Release with changelog"
   task :gem_release do
+    require "gem/release"
     # Safety checks: ensure we're on main and up-to-date
     current_branch = `git branch --show-current`.strip
     unless current_branch == "main"

data/docs/migration-guide.md

@@ -0,0 +1,135 @@
+# Dont use LlmGateway::Client 
+
+use the provider pattern instead 
+# Migrating from `chat` to `stream`
+
+The `chat` method will be deprecated. New code should use `stream`.
+
+If your application only needs the final assistant response, call `stream` without a block. You do not need to handle streaming events.
+
+## Basic migration
+
+### Before
+
+```ruby
+result = adapter.chat("Write one short sentence about Ruby.")
+```
+
+### After
+
+```ruby
+result = adapter.stream("Write one short sentence about Ruby.")
+```
+
+`stream` returns the final assembled `AssistantMessage`, so most response-handling code can stay the same.
+
+## Migrating calls with options
+
+Pass the same options to `stream` that you passed to `chat`.
+
+### Before
+
+```ruby
+result = adapter.chat(
+  transcript,
+  tools: tools,
+  system: "You are a helpful assistant.",
+  reasoning: "high"
+)
+```
+
+### After
+
+```ruby
+result = adapter.stream(
+  transcript,
+  tools: tools,
+  system: "You are a helpful assistant.",
+  reasoning: "high"
+)
+```
+
+## Reading the final response
+
+The returned object has the same final assistant message shape your existing `chat` code expects:
+
+```ruby
+result = adapter.stream(transcript)
+
+puts result.role
+puts result.stop_reason
+puts result.usage.inspect
+
+text = result.content
+  .select { |block| block.type == "text" }
+  .map(&:text)
+  .join
+
+puts text
+```
+
+## Tool-call flows
+
+If your existing `chat` flow inspected the final response for tool calls, keep the same pattern after switching to `stream`:
+
+```ruby
+response = adapter.stream(transcript, tools: [weather_tool])
+
+tool_uses = response.content.select { |block| block.type == "tool_use" }
+
+# Execute tools, append tool_result messages to the transcript,
+# then call stream again for the next assistant response.
+```
+
+## Recommended migration steps
+
+1. Replace `adapter.chat(...)` with `adapter.stream(...)`.
+2. Do not pass a block if you only need the final response.
+3. Run tests that verify text extraction, tool-call detection, stop reasons, and usage accounting.
+4. Remove new uses of `chat` from application code before deprecation.
+
+# Update ClassNames
+
+If you are using any of these classes you should use the new names
+
+## Clients
+
+- LlmGateway::Clients::Claude → LlmGateway::Clients::Anthropic
+- LlmGateway::Clients::OpenAi → LlmGateway::Clients::OpenAI
+
+## Adapters: Anthropic side
+
+- LlmGateway::Adapters::Claude::* → LlmGateway::Adapters::Anthropic::*
+    - Client
+    - MessagesAdapter
+    - InputMapper
+    - OutputMapper
+    - StreamMapper
+    - BidirectionalMessageMapper
+    - FileOutputMapper
+
+## Adapters: OpenAI side
+
+- LlmGateway::Adapters::OpenAi::* → LlmGateway::Adapters::OpenAI::*
+    - Client
+    - ChatCompletionsAdapter
+    - ResponsesAdapter
+    - PromptCacheOptionMapper
+    - FileOutputMapper
+    - ChatCompletions
+    - Responses
+
+## Adapters: Groq side
+
+Groq now reuses the OpenAI Chat Completions mapper stack via `ActsLikeOpenAIChatCompletions`.
+The dedicated Groq mapper classes were removed rather than aliased:
+
+- LlmGateway::Adapters::Groq::InputMapper → LlmGateway::Adapters::OpenAI::ChatCompletions::InputMapper
+- LlmGateway::Adapters::Groq::OutputMapper → LlmGateway::Adapters::OpenAI::ChatCompletions::OutputMapper
+- LlmGateway::Adapters::Groq::BidirectionalMessageMapper → LlmGateway::Adapters::OpenAI::ChatCompletions::BidirectionalMessageMapper
+
+Still provider-specific:
+
+- LlmGateway::Clients::Groq
+- LlmGateway::Adapters::Groq::ChatCompletionsAdapter
+- LlmGateway::Adapters::Groq::OptionMapper

data/lib/llm_gateway/adapters/adapter.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require_relative "stream_accumulator"
+require_relative "structs"
+
+module LlmGateway
+  module Adapters
+    class Adapter
+      attr_reader :client
+
+      def initialize(client)
+        @client = client
+      end
+
+      def chat(message, tools: nil, system: nil, **options)
+        normalized_input = map_input({
+          messages: sanitize_messages(normalize_messages(message)),
+          tools: tools,
+          system: normalize_system(system)
+        })
+
+        result = perform_chat(
+          normalized_input[:messages],
+          tools: normalized_input[:tools],
+          system: normalized_input[:system],
+          **map_options(options)
+        )
+
+        map_output(result)
+      end
+
+      def stream(message, tools: nil, system: nil, **options, &block)
+        raise LlmGateway::Errors::MissingMapperForProvider, "No stream_mapper configured" unless stream_mapper
+
+        normalized_input = map_input({
+          messages: sanitize_messages(normalize_messages(message)),
+          tools: tools,
+          system: normalize_system(system)
+        })
+
+        accumulator = ::StreamAccumulator.new
+        mapper = stream_mapper.new
+
+        perform_stream(
+          normalized_input[:messages],
+          tools: normalized_input[:tools],
+          system: normalized_input[:system],
+          **map_options(options)
+        ) do |chunk|
+          event = mapper.map(chunk)
+          accumulator.push(event)
+          block.call(event) if block && event
+        end
+
+        AssistantMessage.new(
+          accumulator.result.merge(
+            provider: LlmGateway::Client.provider_id_from_client(client),
+            api: api_name
+          )
+        )
+      end
+
+      def upload_file(filename:, content:, mime_type: "application/octet-stream", purpose: "assistants")
+        raise LlmGateway::Errors::MissingMapperForProvider, "No file_output_mapper configured" unless file_output_mapper
+
+        upload_params = client.method(:upload_file).parameters
+        supports_purpose = upload_params.any? { |type, name| [ :key, :keyreq ].include?(type) && name == :purpose }
+
+        result = if supports_purpose
+          client.upload_file(filename, content, mime_type, purpose: purpose)
+        else
+          client.upload_file(filename, content, mime_type)
+        end
+
+        file_output_mapper.map(result)
+      end
+
+      def download_file(file_id:)
+        raise LlmGateway::Errors::MissingMapperForProvider, "No file_output_mapper configured" unless file_output_mapper
+
+        result = client.download_file(file_id)
+        file_output_mapper.map(result)
+      end
+
+      private
+
+      def input_mapper
+        raise NotImplementedError, "#{self.class} must implement #input_mapper"
+      end
+
+      def input_sanitizer
+        nil
+      end
+
+      def output_mapper
+        raise NotImplementedError, "#{self.class} must implement #output_mapper"
+      end
+
+      def file_output_mapper
+        nil
+      end
+
+      def option_mapper
+        OptionMapper
+      end
+
+      def map_input(input)
+        input_mapper.map(input)
+      end
+
+      def map_output(output)
+        output_mapper.map(output)
+      end
+
+      def map_options(options)
+        option_mapper.map(options)
+      end
+
+      def perform_chat(messages, tools:, system:, **options)
+        client.chat(messages, tools: tools, system: system, **options)
+      end
+
+      def perform_stream(messages, tools:, system:, **options, &block)
+        client.stream(messages, tools: tools, system: system, **options, &block)
+      end
+
+      def api_name
+        self.class.name.split("::").last.gsub(/Adapter$/, "").downcase
+      end
+
+      def stream_mapper
+        nil
+      end
+
+      def sanitize_messages(messages)
+        return messages unless input_sanitizer
+
+        target_provider = LlmGateway::Client.provider_id_from_client(client)
+        target_api = api_name
+        target_model = client.model_key
+
+        return messages if target_provider.nil? || target_api.nil? || target_model.nil?
+
+        input_sanitizer.sanitize(
+          messages,
+          target_provider: target_provider,
+          target_api: target_api,
+          target_model: target_model
+        )
+      end
+
+      def normalize_system(system)
+        if system.nil?
+          []
+        elsif system.is_a?(String)
+          [ { role: "system", content: system } ]
+        elsif system.is_a?(Array)
+          system
+        else
+          raise ArgumentError, "System parameter must be a string or array, got #{system.class}"
+        end
+      end
+
+      def normalize_messages(message)
+        if message.is_a?(String)
+          [ { role: "user", content: message } ]
+        else
+          message
+        end
+      end
+    end
+  end
+end

data/lib/llm_gateway/adapters/anthropic/acts_like_messages.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module LlmGateway
+  module Adapters
+    module ActsLikeAnthropicMessages
+      private
+
+      def api_name = "messages"
+
+      def input_mapper = Anthropic::InputMapper
+
+      def input_sanitizer = InputMessageSanitizer
+
+      def output_mapper = Anthropic::OutputMapper
+
+      def file_output_mapper = Anthropic::FileOutputMapper
+
+      def option_mapper = AnthropicOptionMapper
+
+      def stream_mapper = Anthropic::StreamMapper
+    end
+  end
+end

data/lib/llm_gateway/adapters/{claude → anthropic}/bidirectional_message_mapper.rb

@@ -2,7 +2,7 @@
 
 module LlmGateway
   module Adapters
-    module Claude
+    module Anthropic
       class BidirectionalMessageMapper
         attr_reader :direction
 
@@ -25,6 +25,8 @@ module LlmGateway
             map_tool_use_content(content)
           when "tool_result"
             map_tool_result_content(content)
+          when "thinking", "reasoning"
+            map_reasoning_content(content)
           else
             content
           end
@@ -33,10 +35,12 @@ module LlmGateway
         private
 
         def map_text_content(content)
-          {
+          result = {
             type: "text",
             text: content[:text]
           }
+          result[:cache_control] = content[:cache_control] if content[:cache_control]
+          result
         end
 
         def map_file_content(content)
@@ -71,12 +75,36 @@ module LlmGateway
         end
 
         def map_tool_result_content(content)
+          mapped_content = content[:content]
+          if mapped_content.is_a?(Array)
+            mapped_content = mapped_content.map do |item|
+              item.is_a?(Hash) ? map_content(item.transform_keys(&:to_sym)) : item
+            end
+          end
+
           {
             type: "tool_result",
             tool_use_id: content[:tool_use_id],
-            content: content[:content]
+            content: mapped_content
           }
         end
+
+        def map_reasoning_content(content)
+          if direction == LlmGateway::DIRECTION_IN
+            result = {
+              type: "thinking",
+              thinking: content[:reasoning]
+            }
+            result[:signature] = content[:signature] unless content[:signature].nil?
+            result
+          else
+            {
+              type: "reasoning",
+              reasoning: content[:thinking] || content[:reasoning],
+              signature: content[:signature]
+            }
+          end
+        end
       end
     end
   end

data/lib/llm_gateway/adapters/{claude → anthropic}/input_mapper.rb

@@ -4,12 +4,11 @@ require_relative "bidirectional_message_mapper"
 
 module LlmGateway
   module Adapters
-    module Claude
+    module Anthropic
       class InputMapper
         def self.map(data)
           {
             messages: map_messages(data[:messages]),
-            response_format: data[:response_format],
             tools: data[:tools],
             system: map_system(data[:system])
           }
@@ -45,7 +44,9 @@ module LlmGateway
             nil
           elsif system.length == 1 && system.first[:role] == "system"
             # If we have a single system message, convert to Claude format
-            [ { type: "text", text: system.first[:content] } ]
+            mapped = { type: "text", text: system.first[:content] }
+            mapped[:cache_control] = system.first[:cache_control] if system.first[:cache_control]
+            [ mapped ]
           else
             # For multiple messages or non-standard format, pass through
             system

data/lib/llm_gateway/adapters/anthropic/messages_adapter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "../adapter"
+require_relative "acts_like_messages"
+require_relative "../anthropic_option_mapper"
+require_relative "../input_message_sanitizer"
+require_relative "input_mapper"
+require_relative "output_mapper"
+require_relative "stream_mapper"
+
+module LlmGateway
+  module Adapters
+    module Anthropic
+      class MessagesAdapter < Adapter
+        include ActsLikeAnthropicMessages
+      end
+    end
+  end
+end

data/lib/llm_gateway/adapters/{claude → anthropic}/output_mapper.rb

@@ -2,7 +2,7 @@
 
 module LlmGateway
   module Adapters
-    module Claude
+    module Anthropic
       class FileOutputMapper
         def self.map(data)
           data.delete(:type) # Didnt see much value in this only option is "file"

data/lib/llm_gateway/adapters/anthropic/stream_mapper.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require_relative "../structs.rb"
+
+module LlmGateway
+  module Adapters
+    module Anthropic
+      class StreamMapper
+        def map(chunk)
+          case chunk[:event]
+          when "message_start"
+            delta = {
+              id: chunk.dig(:data, :message, :id),
+              model: chunk.dig(:data, :message, :model),
+              role: chunk.dig(:data, :message, :role)
+            }
+            usage_increment = chunk.dig(:data, :message, :usage) || {}
+
+            AssistantStreamMessageEvent.new(type: :message_start, usage_increment:, delta:)
+          when "content_block_start"
+            content_index = chunk.dig(:data, :index)
+            delta = chunk.dig(:data, :content_block, :text)
+            current_type = chunk.dig(:data, :content_block, :type)
+            content_block_types[content_index] = current_type
+
+            case current_type
+            when "thinking"
+              AssistantStreamEvent.new(type: :reasoning_start, content_index:, delta:)
+            when "text"
+              AssistantStreamEvent.new(type: :text_start, content_index:, delta:)
+            when "tool_use"
+              id = chunk.dig(:data, :content_block, :id)
+              name = chunk.dig(:data, :content_block, :name)
+              AssistantToolStartEvent.new(type: :tool_start, content_index:, delta:, id:, name:)
+            end
+          when "content_block_delta"
+            content_index = chunk.dig(:data, :index)
+
+            case content_block_types[content_index]
+            when "thinking"
+              delta = chunk.dig(:data, :delta, :thinking)
+              signature = chunk.dig(:data, :delta, :signature)
+              AssistantStreamReasoningEvent.new(type: :reasoning_delta, signature:, delta:, content_index:)
+            when "text"
+              delta = chunk.dig(:data, :delta, :text)
+              AssistantStreamEvent.new(type: :text_delta, content_index:, delta:)
+            when "tool_use"
+              delta = chunk.dig(:data, :delta, :partial_json)
+              AssistantStreamEvent.new(type: :tool_delta, content_index:, delta:)
+            end
+          when "content_block_stop"
+            content_index = chunk.dig(:data, :index)
+            type = case content_block_types[content_index]
+            when "thinking"
+              :reasoning_end
+            when "text"
+              :text_end
+            when "tool_use"
+              :tool_end
+            end
+            AssistantStreamEvent.new(type: type, content_index:, delta: "")
+          when "message_delta"
+            delta = normalize_message_delta(chunk.dig(:data, :delta) || {})
+            usage_increment = chunk.dig(:data, :usage) || {}
+
+            AssistantStreamMessageEvent.new(type: :message_delta, usage_increment:, delta:)
+          when "message_stop"
+            AssistantStreamMessageEvent.new(type: :message_end, usage_increment: {}, delta: {})
+          when "ping"
+            nil
+          when "error"
+            error = chunk.dig(:data, :error) || {}
+            message = error[:message] || "Stream error"
+            code = error[:type]
+
+            if LlmGateway::Errors.context_overflow_message?(message)
+              raise LlmGateway::Errors::PromptTooLong.new(message, code)
+            end
+
+            if code == "overloaded_error"
+              raise LlmGateway::Errors::OverloadError.new(message, code)
+            end
+
+            raise LlmGateway::Errors::APIStatusError.new(message, code)
+          end
+        end
+
+        private
+
+        def content_block_types
+          @content_block_types ||= {}
+        end
+
+        def normalize_message_delta(delta)
+          return delta unless delta[:stop_reason] || delta["stop_reason"]
+
+          stop_reason = delta[:stop_reason] || delta["stop_reason"]
+          normalized_stop_reason = case stop_reason
+          when "end_turn"
+            "stop"
+          else
+            stop_reason
+          end
+
+          delta.merge(stop_reason: normalized_stop_reason)
+        end
+      end
+    end
+  end
+end

data/lib/llm_gateway/adapters/anthropic_option_mapper.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module LlmGateway
+  module Adapters
+    module AnthropicOptionMapper
+      DEFAULT_MAX_TOKENS = 20_480
+      REASONING_EFFORT_BUDGET_TOKENS = {
+        "low" => 1024,
+        "medium" => 5 * 1024,
+        "high" => 10 * 1024,
+        "xhigh" => 20 * 1024
+      }.freeze
+
+      module_function
+
+      def map(options)
+        mapped_options = options.reject { |key, _| %i[reasoning max_completion_tokens response_format prompt_cache_retention cache_key prompt_cache_key].include?(key) }
+        mapped_options[:max_tokens] = options[:max_completion_tokens] || DEFAULT_MAX_TOKENS
+
+        retention = options[:cache_retention]
+        mapped_options[:cache_retention] = retention unless retention.nil?
+
+        response_format = options[:response_format]
+        mapped_options[:output_config] = normalize_output_config(response_format) unless response_format.nil?
+
+        reasoning = options[:reasoning]
+        return mapped_options if reasoning.nil? || reasoning.to_s == "none"
+
+        mapped_options[:thinking] = normalize_reasoning(reasoning)
+        mapped_options
+      end
+
+      def normalize_output_config(response_format)
+        format_type = response_format.is_a?(Hash) ? response_format[:type] || response_format["type"] : response_format
+
+        case format_type.to_s
+        when "json_object", "json_schema"
+          { format: "json_schema" }
+        else
+          { format: "text" }
+        end
+      end
+
+      def normalize_reasoning(reasoning)
+        budget_tokens = REASONING_EFFORT_BUDGET_TOKENS[reasoning.to_s] ||
+          raise(ArgumentError,
+                "Invalid reasoning '#{reasoning}'. Use 'none', 'low', 'medium', 'high', or 'xhigh'.")
+
+        { type: "enabled", budget_tokens: budget_tokens }
+      end
+    end
+  end
+end

data/lib/llm_gateway/adapters/groq/chat_completions_adapter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "../adapter"
+require_relative "../openai/acts_like_chat_completions"
+require_relative "../input_message_sanitizer"
+require_relative "../openai/chat_completions/input_mapper"
+require_relative "option_mapper"
+
+module LlmGateway
+  module Adapters
+    module Groq
+      class ChatCompletionsAdapter < Adapter
+        include ActsLikeOpenAIChatCompletions
+
+        private
+
+        def file_output_mapper = nil
+        def stream_mapper = nil
+        def option_mapper = Groq::OptionMapper
+
+        def map_input(input)
+          groq_safe_input = input.dup
+          groq_safe_input[:messages] = Array(input[:messages]).map do |msg|
+            next msg unless msg.is_a?(Hash) && msg[:content].is_a?(Array)
+
+            rewritten_content = msg[:content].map do |block|
+              next block unless block.is_a?(Hash) && block[:type] == "file"
+
+              {
+                type: "text",
+                text: block[:text] || "[File: #{block[:name]}]"
+              }
+            end
+
+            msg.merge(content: rewritten_content)
+          end
+
+          mapped = super(groq_safe_input)
+          mapped[:system] = Array(mapped[:system]).map do |msg|
+            msg[:role] == "developer" ? msg.merge(role: "system") : msg
+          end
+          mapped
+        end
+      end
+    end
+  end
+end