llm_gateway 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 173ab613e57543956e39d70f4a38fc865bc6b6bac4e8dfe319be9c2928810f77
-  data.tar.gz: 46c761a838aee6c3cebad151467555cba8ab70480e952ab741874c2d8acc13e8
+  metadata.gz: c70b5fdd6e77e60cbe437a5698e078b7aa88fb41dca96311dcf9a58585238476
+  data.tar.gz: cb9dd1f8aa71c33751ddd270acb849d964ed137b6b3fdd6c96e072082bc6e488
 SHA512:
-  metadata.gz: 0f21f7288e4d8d374ea77d96ee3110b08a260a2e06ef6fd6372357b88abb5e936d2cbeae934720a0a5e17ad431bdce7027a3cd68e4fc60460c6cb5d0f02acc0a
-  data.tar.gz: ecf15206364c5ef7d632c0c421294deb1929e508b4287828acbee91e4a4182fb0efd5fb12cca58d6909499b2b33197070bd4c19e232bed8f25fd12f86e2dd604
+  metadata.gz: d21ab094dd5f600731e12541c6ba0a672eb8d0862c732ce2f5f2a415c394024ff87565dee5820e2d1dcec5ee665715d7beacc5eb5fb5a35d518e98b4c3c6c6e1
+  data.tar.gz: 97f4cabb1416beddc261fc849d23823722f9c36afb70843fcb1ef9e748e7f7eeec7f8bb4cfa0f92b25dbc864bf2c27e8751eeb9367f5c6233a906ac79d483219

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [v0.8.1](https://github.com/Hyper-Unearthing/llm_gateway/tree/v0.8.1) (2026-06-27)
+
+[Full Changelog](https://github.com/Hyper-Unearthing/llm_gateway/compare/v0.8.0...v0.8.1)
+
+**Merged pull requests:**
+
+- fix: do not return empty assistant blocks [\#94](https://github.com/Hyper-Unearthing/llm_gateway/pull/94) ([billybonks](https://github.com/billybonks))
+- feat: when codex hits a rate limit pass all the information [\#93](https://github.com/Hyper-Unearthing/llm_gateway/pull/93) ([billybonks](https://github.com/billybonks))
+- Feat/fix proxy client [\#92](https://github.com/Hyper-Unearthing/llm_gateway/pull/92) ([billybonks](https://github.com/billybonks))
+
+## [v0.8.0](https://github.com/Hyper-Unearthing/llm_gateway/tree/v0.8.0) (2026-06-06)
+
+[Full Changelog](https://github.com/Hyper-Unearthing/llm_gateway/compare/v0.7.0...v0.8.0)
+
+**Merged pull requests:**
+
+- feat: create server proxy and corresponding client [\#89](https://github.com/Hyper-Unearthing/llm_gateway/pull/89) ([billybonks](https://github.com/billybonks))
+
 ## [v0.7.0](https://github.com/Hyper-Unearthing/llm_gateway/tree/v0.7.0) (2026-06-03)
 
 [Full Changelog](https://github.com/Hyper-Unearthing/llm_gateway/compare/v0.6.0...v0.7.0)

data/README.md

@@ -916,6 +916,21 @@ puts result.content.select { |b| b.type == "text" }.map(&:text).join
 
 If your app refreshes tokens in the background, rebuild the adapter (or recreate client state) with the newest `access_token` before subsequent calls.
 
+### Codex rate-limit reset metadata
+
+OpenAI Codex usage-limit responses include reset information on `LlmGateway::Errors::RateLimitError`:
+
+```ruby
+begin
+  adapter.stream("Hello from OAuth auth", model: "gpt-5.4")
+rescue LlmGateway::Errors::RateLimitError => e
+  puts e.message                 # "The usage limit has been reached"
+  puts e.reset_after_seconds     # primary reset window, when available
+  puts e.reset_at                # Time for the primary reset, when available
+  puts e.rate_limit_info.inspect # full parsed Codex headers/body metadata
+end
+```
+
 ### Token refresh responsibility
 
 #### Library’s role (llm_gateway)

data/lib/llm_gateway/adapters/adapter.rb

@@ -5,32 +5,38 @@ require_relative "structs"
 module LlmGateway
   module Adapters
     class Adapter
-      attr_reader :client
+      attr_reader :client, :provider_key
 
-      def initialize(client)
+      def initialize(client, provider_key: nil)
         @client = client
+        @provider_key = provider_key
       end
 
-      def stream(message, tools: nil, system: nil, **options, &block)
-        raise LlmGateway::Errors::MissingMapperForProvider, "No stream_mapper configured" unless stream_mapper
-
+      def raw_stream(message, tools: nil, system: nil, **options, &block)
         normalized_input = map_input({
           messages: sanitize_messages(normalize_messages(message), target_model: options[:model]),
           tools: tools,
           system: normalize_system(system)
         })
 
+        perform_stream(
+          normalized_input[:messages],
+          tools: normalized_input[:tools],
+          system: normalized_input[:system],
+          **map_options(options),
+          &block
+        )
+      end
+
+      def stream(message, tools: nil, system: nil, **options, &block)
+        raise LlmGateway::Errors::MissingMapperForProvider, "No stream_mapper configured" unless stream_mapper
+
         mapper = stream_mapper.new(
           provider: LlmGateway::Client.provider_id_from_client(client),
           api: api_name
         )
 
-        perform_stream(
-          normalized_input[:messages],
-          tools: normalized_input[:tools],
-          system: normalized_input[:system],
-          **map_options(options)
-        ) do |chunk|
+        raw_stream(message, tools: tools, system: system, **options) do |chunk|
           mapper.map(chunk, &block)
         end
 
@@ -89,6 +95,18 @@ module LlmGateway
         client.stream(messages, tools: tools, system: system, **options, &block)
       end
 
+      public
+
+      def stream_api_name
+        api_name
+      end
+
+      def stream_mapper_class
+        stream_mapper
+      end
+
+      private
+
       def api_name
         self.class.name.split("::").last.gsub(/Adapter$/, "").downcase
       end

data/lib/llm_gateway/adapters/normalized_stream_accumulator.rb

@@ -330,7 +330,9 @@ module LlmGateway
       end
 
       def serialized_blocks
-        blocks.compact.map do |content_block|
+        blocks.compact.filter_map do |content_block|
+          next if blank_text_block?(content_block)
+
           if [ "tool_use", "server_tool_use" ].include?(content_block[:type])
             next content_block.merge(input: parse_tool_input(content_block[:input]).deep_symbolize_keys)
           end
@@ -348,6 +350,10 @@ module LlmGateway
         end
       end
 
+      def blank_text_block?(content_block)
+        content_block[:type] == "text" && content_block[:text].to_s.blank?
+      end
+
       def parse_tool_input(input)
         return {} if input.blank?
 

data/lib/llm_gateway/agents/harness.rb

@@ -128,6 +128,11 @@ module LlmGateway
       end
 
       def enqueue_or_run_message(message, queue, &block)
+        if session_manager.idle?
+          drain_queue(:steer)
+          drain_queue(:next_turn)
+          drain_queue(:follow_up)
+        end
         prepared_input = LlmGateway::Utils.deep_symbolize_keys(message)
         result = session_manager.start_or_enqueue_user_message(prepared_input, queue: queue) do
           compact_if_needed
@@ -135,6 +140,8 @@ module LlmGateway
         return if result == session_manager.class::MESSAGE_QUEUED
 
         begin
+
+
           continue(&block)
 
           loop do

data/lib/llm_gateway/agents/in_memory_session_manager.rb

@@ -45,6 +45,10 @@ module LlmGateway
         @state == :busy
       end
 
+      def idle?
+        @state == :idle
+      end
+
       def validate_queue!(queue)
         queue = queue.to_sym
         raise ArgumentError, "Invalid queue mode: #{queue}" unless QUEUES.include?(queue)

data/lib/llm_gateway/clients/openai.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "time"
+
 require_relative "../base_client"
 
 module LlmGateway
@@ -179,6 +181,87 @@ module LlmGateway
         }
       end
 
+      def rate_limit_error_options(response, error)
+        info = rate_limit_info(response, error)
+        return {} if info.empty?
+
+        {
+          reset_at: info[:primary_reset_at] || info[:reset_at],
+          reset_after_seconds: info[:primary_reset_after_seconds] || info[:reset_after_seconds],
+          rate_limit_info: info
+        }
+      end
+
+      def rate_limit_info(response, error)
+        headers = response_headers(response)
+        codex_headers = headers.select { |key, _value| key.start_with?("x-codex-") }
+        info = {}
+
+        info[:provider] = "openai_codex" if codex_headers.any?
+        info[:error_type] = error["type"]
+        info[:plan_type] = headers["x-codex-plan-type"] || error["plan_type"]
+        info[:active_limit] = headers["x-codex-active-limit"]
+        info[:primary_used_percent] = integer_header(headers, "x-codex-primary-used-percent")
+        info[:secondary_used_percent] = integer_header(headers, "x-codex-secondary-used-percent")
+        info[:primary_window_minutes] = integer_header(headers, "x-codex-primary-window-minutes")
+        info[:secondary_window_minutes] = integer_header(headers, "x-codex-secondary-window-minutes")
+        info[:primary_over_secondary_limit_percent] = integer_header(headers, "x-codex-primary-over-secondary-limit-percent")
+        info[:primary_reset_after_seconds] = integer_header(headers, "x-codex-primary-reset-after-seconds")
+        info[:secondary_reset_after_seconds] = integer_header(headers, "x-codex-secondary-reset-after-seconds")
+        info[:primary_reset_at] = epoch_time(headers["x-codex-primary-reset-at"])
+        info[:secondary_reset_at] = epoch_time(headers["x-codex-secondary-reset-at"])
+        info[:credits_has_credits] = boolean_header(headers, "x-codex-credits-has-credits")
+        info[:credits_balance] = integer_header(headers, "x-codex-credits-balance")
+        info[:credits_unlimited] = boolean_header(headers, "x-codex-credits-unlimited")
+        info[:reset_after_seconds] = integer_value(error["resets_in_seconds"]) || retry_after_seconds(headers["retry-after"])
+        info[:reset_at] = epoch_time(error["resets_at"]) || retry_after_time(headers["retry-after"])
+
+        info.compact
+      end
+
+      def response_headers(response)
+        headers = {}
+        response.each_header { |key, value| headers[key.downcase] = value }
+        headers
+      end
+
+      def integer_header(headers, key)
+        integer_value(headers[key])
+      end
+
+      def integer_value(value)
+        return nil if value.nil?
+
+        Integer(value)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def boolean_header(headers, key)
+        case headers[key].to_s.downcase
+        when "true" then true
+        when "false" then false
+        end
+      end
+
+      def epoch_time(value)
+        integer = integer_value(value)
+        Time.at(integer) if integer
+      end
+
+      def retry_after_seconds(value)
+        integer_value(value)
+      end
+
+      def retry_after_time(value)
+        seconds = retry_after_seconds(value)
+        return Time.now + seconds if seconds
+
+        Time.httpdate(value)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
       def handle_client_specific_errors(response, error)
         # OpenAI uses 'code' instead of 'type' for error codes
         error_code = error["code"]
@@ -190,7 +273,7 @@ module LlmGateway
 
         case response.code.to_i
         when 429
-          raise Errors::RateLimitError.new(error_message, error_code)
+          raise Errors::RateLimitError.new(error_message, error_code, **rate_limit_error_options(response, error))
         when 503
           raise Errors::OverloadError.new(error_message, error_code)
         end

data/lib/llm_gateway/errors.rb

@@ -19,7 +19,16 @@ module LlmGateway
     class NotFoundError < ClientError; end
     class ConflictError < ClientError; end
     class UnprocessableEntityError < ClientError; end
-    class RateLimitError < ClientError; end
+    class RateLimitError < ClientError
+      attr_reader :reset_at, :reset_after_seconds, :rate_limit_info
+
+      def initialize(message = nil, code = nil, reset_at: nil, reset_after_seconds: nil, rate_limit_info: {})
+        @reset_at = reset_at
+        @reset_after_seconds = reset_after_seconds
+        @rate_limit_info = rate_limit_info
+        super(message, code)
+      end
+    end
     class InternalServerError < ClientError; end
     class APIStatusError < ClientError; end
     class APITimeoutError < ClientError; end

data/lib/llm_gateway/proxy/adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module LlmGateway
+  module Proxy
+    class Adapter
+      attr_reader :client, :provider_key
+
+      def initialize(client, provider_key: nil)
+        @client = client
+        @provider_key = provider_key
+      end
+
+      def stream(message, tools: nil, system: nil, **options, &block)
+        target_adapter = LlmGateway.build_provider(client.target_config.merge(provider: client.target_provider))
+        mapper_class = target_adapter.stream_mapper_class
+        raise LlmGateway::Errors::MissingMapperForProvider, "No stream_mapper configured" unless mapper_class
+
+        mapper = mapper_class.new(
+          provider: LlmGateway::Client.provider_id_from_client(target_adapter.client),
+          api: target_adapter.stream_api_name
+        )
+
+        client.stream(normalize_messages(message), tools: tools, system: normalize_system(system), **options) do |chunk|
+          mapper.map(chunk, &block)
+        end
+
+        mapper.result
+      end
+
+      private
+
+      def normalize_system(system)
+        if system.nil?
+          []
+        elsif system.is_a?(String)
+          [ { role: "system", content: system } ]
+        elsif system.is_a?(Array)
+          system
+        else
+          raise ArgumentError, "System parameter must be a string or array, got #{system.class}"
+        end
+      end
+
+      def normalize_messages(message)
+        message.is_a?(String) ? [ { role: "user", content: message } ] : message
+      end
+    end
+  end
+end

data/lib/llm_gateway/proxy/client.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module LlmGateway
+  module Proxy
+    class Client
+      attr_reader :url, :target_provider, :target_config, :path
+
+      def initialize(url:, target_provider:, target_config: {}, api_key: nil, path: "/agent/llm_proxy", **_options)
+        @url = url.to_s.sub(%r{/+\z}, "")
+        @target_provider = target_provider.to_s
+        @target_config = (target_config || {}).transform_keys(&:to_sym)
+        @api_key = api_key
+        @path = path.to_s.start_with?("/") ? path.to_s : "/#{path}"
+      end
+
+      def stream(messages, tools: nil, system: nil, **options, &block)
+        uri = URI("#{url}#{path}")
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.read_timeout = 480
+        http.open_timeout = 10
+
+        request = Net::HTTP::Post.new(uri)
+        request["content-type"] = "application/json"
+        request["accept"] = "text/event-stream"
+        request["accept-encoding"] = "identity"
+        request["authorization"] = "Bearer #{@api_key}" if @api_key
+        request.body = {
+          provider: target_provider,
+          config: target_config,
+          messages: messages,
+          system: system,
+          tools: tools,
+          options: options
+        }.to_json
+
+        http.request(request) do |response|
+          unless response.code.to_i == 200
+            body = +""
+            response.read_body { |chunk| body << chunk }
+            raise Errors::APIStatusError.new("Proxy request failed with status #{response.code}: #{body}", nil)
+          end
+
+          parse_sse_stream(response, &block)
+        end
+      end
+
+      private
+
+      def parse_sse_stream(response)
+        buffer = +""
+        response.read_body do |chunk|
+          buffer << chunk
+          while (idx = buffer.index("\n\n"))
+            raw_event = buffer.slice!(0, idx + 2)
+            event_type = nil
+            data_lines = []
+
+            raw_event.each_line do |line|
+              line = line.chomp
+              event_type = line.sub(/^event:\s*/, "") if line.start_with?("event:")
+              data_lines << line.sub(/^data:\s*/, "") if line.start_with?("data:")
+            end
+
+            next if data_lines.empty?
+
+            data_str = data_lines.join("\n")
+            next if data_str == "[DONE]"
+
+            data = begin
+              JSON.parse(data_str).deep_symbolize_keys
+            rescue JSON::ParserError
+              { raw: data_str }
+            end
+            yield({ event: event_type, data: data })
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm_gateway/proxy/server.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmGateway
+  module Proxy
+    class Server
+      PATH = "/agent/llm_proxy"
+
+      def call(env)
+        return not_found unless env["REQUEST_METHOD"] == "POST" && env["PATH_INFO"] == PATH
+
+        request = JSON.parse(env["rack.input"].read).deep_symbolize_keys
+        options = request[:options] || {}
+        options = options.merge(model: request[:model]) if request.key?(:model)
+        adapter = build_adapter(request)
+
+        body = Enumerator.new do |yielder|
+          adapter.raw_stream(
+            request[:messages],
+            system: request[:system],
+            tools: request[:tools],
+            **options
+          ) do |chunk|
+            yielder << encode_sse(chunk)
+          end
+        end
+
+        [ 200, { "content-type" => "text/event-stream", "cache-control" => "no-cache" }, body ]
+      rescue KeyError, JSON::ParserError, ArgumentError => e
+        json_error(400, e.message)
+      rescue Errors::UnsupportedProvider => e
+        json_error(404, e.message)
+      rescue StandardError => e
+        json_error(500, e.message)
+      end
+
+      private
+
+      def build_adapter(request)
+        provider = request.fetch(:provider)
+        config = (request[:config] || {}).merge(provider: provider)
+
+        LlmGateway.build_provider(config)
+      end
+
+      def encode_sse(chunk)
+        event = chunk[:event]
+        data = chunk[:data]
+        out = +""
+        out << "event: #{event}\n" if event
+        JSON.generate(data).each_line { |line| out << "data: #{line.chomp}\n" }
+        out << "\n"
+      end
+
+      def json_error(status, message)
+        [ status, { "content-type" => "application/json" }, [ { error: message }.to_json ] ]
+      end
+
+      def not_found
+        [ 404, { "content-type" => "application/json" }, [ { error: "Not found" }.to_json ] ]
+      end
+    end
+  end
+end

data/lib/llm_gateway/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmGateway
-  VERSION = "0.7.0"
+  VERSION = "0.8.1"
 end

data/lib/llm_gateway.rb

@@ -48,6 +48,9 @@ require_relative "llm_gateway/adapters/groq/chat_completions_adapter"
 
 # Load provider registry
 require_relative "llm_gateway/provider_registry"
+require_relative "llm_gateway/proxy/client"
+require_relative "llm_gateway/proxy/adapter"
+require_relative "llm_gateway/proxy/server"
 
 module LlmGateway
   class Error < StandardError; end
@@ -110,7 +113,7 @@ module LlmGateway
     entry = ProviderRegistry.resolve(provider_name)
 
     client = entry[:client].new(**config)
-    entry[:adapter].new(client)
+    entry[:adapter].new(client, provider_key: provider_name)
   end
 
   def self.configure(configs)
@@ -160,4 +163,8 @@ module LlmGateway
   ProviderRegistry.register("openai_codex",
     client: Clients::OpenAI,
     adapter: Adapters::OpenAICodex::ResponsesAdapter)
+
+  ProviderRegistry.register("proxy",
+    client: Proxy::Client,
+    adapter: Proxy::Adapter)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llm_gateway
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - billybonks
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-06-03 00:00:00.000000000 Z
+date: 2026-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-struct
@@ -97,6 +97,9 @@ files:
 - lib/llm_gateway/errors.rb
 - lib/llm_gateway/prompt.rb
 - lib/llm_gateway/provider_registry.rb
+- lib/llm_gateway/proxy/adapter.rb
+- lib/llm_gateway/proxy/client.rb
+- lib/llm_gateway/proxy/server.rb
 - lib/llm_gateway/tool.rb
 - lib/llm_gateway/utils.rb
 - lib/llm_gateway/version.rb