llm.rb 8.0.0 → 9.0.0

data/lib/llm/transport/http.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+class LLM::Transport
+  ##
+  # The {LLM::Transport::HTTP LLM::Transport::HTTP} transport is the
+  # built-in adapter for Ruby's {Net::HTTP Net::HTTP}. It manages
+  # transient HTTP connections, tracks active requests by owner, and
+  # interrupts in-flight requests when needed.
+  #
+  # @api private
+  class HTTP < self
+    INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+    Request = Struct.new(:client, keyword_init: true)
+
+    ##
+    # @param [String] host
+    # @param [Integer] port
+    # @param [Integer] timeout
+    # @param [Boolean] ssl
+    # @return [LLM::Transport::HTTP]
+    def initialize(host:, port:, timeout:, ssl:)
+      @host = host
+      @port = port
+      @timeout = timeout
+      @ssl = ssl
+      @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
+      @monitor = Monitor.new
+    end
+
+    ##
+    # Returns the current request owner.
+    # @return [Object]
+    def request_owner
+      return Fiber.current unless defined?(::Async)
+      Async::Task.current? ? Async::Task.current : Fiber.current
+    end
+
+    ##
+    # @return [Array<Class<Exception>>]
+    def interrupt_errors
+      [*INTERRUPT_ERRORS, *optional_interrupt_errors]
+    end
+
+    ##
+    # Interrupt an active request, if any.
+    # @param [Fiber] owner
+    # @return [nil]
+    def interrupt!(owner)
+      req = request_for(owner) or return
+      lock { (@interrupts ||= {})[owner] = true }
+      close_socket(req.client)
+      req.client.finish if req.client.active?
+      owner.stop if owner.respond_to?(:stop)
+    rescue *interrupt_errors
+      nil
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Fiber] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      lock { @interrupts&.delete(owner) }
+    end
+
+    ##
+    # Performs a request on the current HTTP transport.
+    # @param [Net::HTTPRequest] request
+    # @param [Fiber] owner
+    # @param [LLM::Object, nil] stream
+    # @yieldparam [LLM::Transport::Response] response
+    # @return [Object]
+    def request(request, owner:, stream: nil, &b)
+      client = client()
+      set_request(Request.new(client:), owner)
+      perform_request(client, request, stream, &b)
+    ensure
+      clear_request(owner)
+    end
+
+    ##
+    # @return [String]
+    def inspect
+      "#<#{self.class.name}:0x#{object_id.to_s(16)}>"
+    end
+
+    private
+
+    attr_reader :host, :port, :timeout, :ssl, :base_uri
+
+    def client
+      client = Net::HTTP.new(host, port)
+      client.read_timeout = timeout
+      client.use_ssl = ssl
+      client
+    end
+
+    def close_socket(http)
+      socket = http&.instance_variable_get(:@socket) or return
+      socket = socket.io if socket.respond_to?(:io)
+      socket.close
+    rescue *interrupt_errors
+      nil
+    end
+
+    def request_for(owner)
+      lock do
+        @requests ||= {}
+        @requests[owner]
+      end
+    end
+
+    def set_request(req, owner)
+      lock do
+        @requests ||= {}
+        @requests[owner] = req
+      end
+    end
+
+    def clear_request(owner)
+      lock { @requests&.delete(owner) }
+    end
+
+    def lock(&)
+      @monitor.synchronize(&)
+    end
+
+    def optional_interrupt_errors
+      defined?(::Async::Stop) ? [Async::Stop] : []
+    end
+  end
+end

data/lib/llm/transport/persistent_http.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+class LLM::Transport
+  ##
+  # The {LLM::Transport::PersistentHTTP LLM::Transport::PersistentHTTP}
+  # transport is the built-in adapter for
+  # [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent).
+  # It manages pooled HTTP connections, tracks active requests by owner,
+  # and interrupts in-flight requests when needed.
+  #
+  # @api private
+  class PersistentHTTP < self
+    INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+    Request = Struct.new(:client, :connection, keyword_init: true)
+    @registry = {}
+    @monitor = Monitor.new
+
+    ##
+    # Returns the process-wide connection pool registry.
+    # @return [Hash]
+    def self.registry
+      @registry
+    end
+
+    def self.lock(&)
+      @monitor.synchronize(&)
+    end
+
+    ##
+    # @param [String] host
+    # @param [Integer] port
+    # @param [Integer] timeout
+    # @param [Boolean] ssl
+    # @return [LLM::Transport::PersistentHTTP]
+    def initialize(host:, port:, timeout:, ssl:)
+      @host = host
+      @port = port
+      @timeout = timeout
+      @ssl = ssl
+      @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
+      @monitor = Monitor.new
+    end
+
+    ##
+    # Returns the current request owner.
+    # @return [Object]
+    def request_owner
+      return Fiber.current unless defined?(::Async)
+      Async::Task.current? ? Async::Task.current : Fiber.current
+    end
+
+    ##
+    # @return [Array<Class<Exception>>]
+    def interrupt_errors
+      [*INTERRUPT_ERRORS, *optional_interrupt_errors]
+    end
+
+    ##
+    # Interrupt an active request, if any.
+    # @param [Fiber] owner
+    # @return [nil]
+    def interrupt!(owner)
+      req = request_for(owner) or return
+      lock { (@interrupts ||= {})[owner] = true }
+      close_socket(req.connection&.http)
+      req.client.finish(req.connection)
+      owner.stop if owner.respond_to?(:stop)
+    rescue *interrupt_errors
+      nil
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Fiber] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      lock { @interrupts&.delete(owner) }
+    end
+
+    ##
+    # Performs a request on the current HTTP transport.
+    # @param [Net::HTTPRequest] request
+    # @param [Fiber] owner
+    # @param [LLM::Object, nil] stream
+    # @yieldparam [LLM::Transport::Response] response
+    # @return [Object]
+    def request(request, owner:, stream: nil, &b)
+      client.connection_for(URI.join(base_uri, request.path)) do |connection|
+        set_request(Request.new(client:, connection:), owner)
+        perform_request(connection.http, request, stream, &b)
+      end
+    ensure
+      clear_request(owner)
+    end
+
+    private
+
+    attr_reader :host, :port, :timeout, :ssl, :base_uri
+
+    def client
+      self.class.lock do
+        if self.class.registry[client_id]
+          self.class.registry[client_id]
+        else
+          LLM.require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
+          client = Net::HTTP::Persistent.new(name: self.class.name)
+          client.read_timeout = timeout
+          client.open_timeout = timeout
+          self.class.registry[client_id] = client
+        end
+      end
+    end
+
+    def client_id
+      "#{host}:#{port}:#{timeout}:#{ssl}"
+    end
+
+    def close_socket(http)
+      socket = http&.instance_variable_get(:@socket) or return
+      socket = socket.io if socket.respond_to?(:io)
+      socket.close
+    rescue *interrupt_errors
+      nil
+    end
+
+    def request_for(owner)
+      lock do
+        @requests ||= {}
+        @requests[owner]
+      end
+    end
+
+    def set_request(req, owner)
+      lock do
+        @requests ||= {}
+        @requests[owner] = req
+      end
+    end
+
+    def clear_request(owner)
+      lock { @requests&.delete(owner) }
+    end
+
+    def lock(&)
+      @monitor.synchronize(&)
+    end
+
+    def optional_interrupt_errors
+      defined?(::Async::Stop) ? [Async::Stop] : []
+    end
+  end
+end

data/lib/llm/transport/response/http.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+class LLM::Transport::Response
+  ##
+  # {LLM::Transport::Response::HTTP LLM::Transport::Response::HTTP}
+  # adapts a {Net::HTTPResponse Net::HTTPResponse} to the
+  # {LLM::Transport::Response LLM::Transport::Response} interface.
+  #
+  # This is the default wrapper for responses produced by the built-in
+  # {LLM::Transport::HTTP LLM::Transport::HTTP} transport.
+  class HTTP < self
+    ##
+    # @return [Net::HTTPResponse]
+    attr_reader :res
+
+    ##
+    # @param [Net::HTTPResponse] res
+    # @return [LLM::Transport::Response::HTTP]
+    def initialize(res)
+      @res = res
+    end
+
+    ##
+    # @return [String]
+    def code
+      @res.code
+    end
+
+    ##
+    # @return [Object]
+    def body
+      @res.body
+    end
+
+    ##
+    # @param [Object] value
+    # @return [Object]
+    def body=(value)
+      @res.body = value
+    end
+
+    ##
+    # @param [String] key
+    # @return [String, nil]
+    def [](key)
+      @res[key]
+    end
+
+    ##
+    # @param [Object, nil] dest
+    # @yieldparam [String] chunk
+    # @return [void]
+    def read_body(dest = nil, &block)
+      if dest && block
+        @res.read_body(dest) { block.call(_1) }
+      elsif dest
+        @res.read_body(dest)
+      elsif block
+        @res.read_body { block.call(_1) }
+      else
+        @res.read_body
+      end
+    end
+
+    ##
+    # @return [Boolean]
+    def success?
+      Net::HTTPSuccess === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def ok?
+      Net::HTTPOK === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def bad_request?
+      Net::HTTPBadRequest === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def unauthorized?
+      Net::HTTPUnauthorized === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def forbidden?
+      Net::HTTPForbidden === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def not_found?
+      Net::HTTPNotFound === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def rate_limited?
+      Net::HTTPTooManyRequests === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def server_error?
+      Net::HTTPServerError === @res
+    end
+  end
+end

data/lib/llm/transport/response.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+class LLM::Transport
+  ##
+  # {LLM::Transport::Response LLM::Transport::Response} defines the
+  # normalized HTTP response interface expected by transports and
+  # provider error handlers.
+  #
+  # Custom transports can execute requests through any underlying HTTP
+  # client, then adapt that client's native response object to this
+  # interface.
+  #
+  # This keeps the transport boundary focused on one contract:
+  # providers, execution, and error handlers only need a response
+  # object that implements
+  # {LLM::Transport::Response LLM::Transport::Response}, regardless of
+  # how the request was actually performed.
+  class Response
+    require_relative "response/http"
+
+    ##
+    # @param [Object] res
+    # @return [LLM::Transport::Response]
+    def self.from(res)
+      return res if LLM::Transport::Response === res
+      return HTTP.new(res) if Net::HTTPResponse === res
+      res
+    end
+
+    ##
+    # @return [String]
+    def code
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Object]
+    def body
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [Object] value
+    # @return [Object]
+    def body=(value)
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [String] key
+    # @return [String, nil]
+    def [](key)
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [Object, nil] dest
+    # @yieldparam [String] chunk
+    # @return [void]
+    def read_body(dest = nil, &)
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def success?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def ok?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def bad_request?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def unauthorized?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def forbidden?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def not_found?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def rate_limited?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def server_error?
+      raise NotImplementedError
+    end
+  end
+end

data/lib/llm/{provider/transport/http → transport}/stream_decoder.rb

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
-module LLM::Provider::Transport
+class LLM::Transport
   ##
-  # @private
-  class HTTP::StreamDecoder
+  # {LLM::Transport::StreamDecoder LLM::Transport::StreamDecoder}
+  # incrementally decodes streamed HTTP response bodies into parser
+  # events.
+  #
+  # @api private
+  class StreamDecoder
     ##
     # @return [Object]
     attr_reader :parser
 
     ##
     # @param [#parse!, #body] parser
-    # @return [LLM::Provider::Transport::HTTP::StreamDecoder]
+    # @return [LLM::Transport::StreamDecoder]
     def initialize(parser)
       @buffer = +""
       @cursor = 0

data/lib/llm/transport.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # The {LLM::Transport LLM::Transport} class defines the execution
+  # interface used by {LLM::Provider}.
+  #
+  # Custom transports can subclass this class and override {#request} to
+  # execute provider requests without changing request adapters or
+  # response adapters.
+  #
+  # Providers currently construct {Net::HTTPRequest Net::HTTPRequest}
+  # objects before delegating to a transport. Custom transports are
+  # therefore expected to execute those requests directly, or transform
+  # them into backend-specific request objects before execution.
+  #
+  # Only {#request} is required. The remaining methods are optional hooks
+  # for features such as interruption, request ownership, or persistence,
+  # and only need to be implemented when the underlying adapter can
+  # support them.
+  #
+  # Returned responses should implement the
+  # {LLM::Transport::Response LLM::Transport::Response} interface. In
+  # practice this can mean adapting another client's response object so
+  # existing provider execution, response adapters, and error handlers
+  # can rely on one normalized response contract instead of
+  # transport-specific classes.
+  class Transport
+    require_relative "transport/response"
+    require_relative "transport/stream_decoder"
+    require_relative "transport/http"
+    require_relative "transport/persistent_http"
+    require_relative "transport/execution"
+
+    ##
+    # Returns the built-in Net::HTTP transport class.
+    # @return [Class]
+    def self.net_http
+      HTTP
+    end
+
+    ##
+    # Returns the built-in Net::HTTP::Persistent transport class.
+    # @return [Class]
+    def self.net_http_persistent
+      PersistentHTTP
+    end
+
+    ##
+    # Performs a request through the transport.
+    # @param [Net::HTTPRequest] request
+    # @param [Object] owner
+    # @param [LLM::Object, nil] stream
+    # @yieldparam [LLM::Transport::Response] response
+    # @return [Object]
+    def request(request, owner:, stream: nil, &)
+      raise NotImplementedError
+    end
+
+    ##
+    # Returns the current request owner.
+    # @return [Object]
+    def request_owner
+      return Fiber.current unless defined?(::Async)
+      Async::Task.current? ? Async::Task.current : Fiber.current
+    end
+
+    ##
+    # Returns the exception classes that indicate an interrupted request.
+    # @return [Array<Class<Exception>>]
+    def interrupt_errors
+      []
+    end
+
+    ##
+    # Interrupt an active request, if any.
+    # @param [Object] owner
+    # @return [nil]
+    def interrupt!(owner)
+      raise NotImplementedError
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Object] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      nil
+    end
+
+    ##
+    # @note
+    #  Custom transports may be able to reuse this helper when they
+    #  operate on Net::HTTPRequest objects, or implement their own
+    #  request body preparation path instead.
+    # @param [Net::HTTPRequest] request
+    # @param [IO] io
+    # @return [void]
+    def set_body_stream(request, io)
+      request.body_stream = io
+      request["transfer-encoding"] = "chunked" unless request["content-length"]
+    end
+
+    private
+
+    ##
+    # @api private
+    # @note
+    #  Custom transports may be able to reuse this helper when they
+    #  execute requests through a Net::HTTP-compatible client, or
+    #  implement their own request execution path instead.
+    def perform_request(client, request, stream, &b)
+      if stream
+        client.request(request) do |raw|
+          res = LLM::Transport::Response.from(raw)
+          if res.success?
+            parser = stream.decoder.new(stream.parser.new(stream.streamer))
+            res.read_body(parser)
+            body = parser.body
+            res.body = (Hash === body || Array === body) ? LLM::Object.from(body) : body
+          else
+            body = +""
+            res.read_body { body << _1 }
+            res.body = body
+          end
+        ensure
+          parser&.free
+        end
+      elsif b
+        client.request(request) do |raw|
+          res = LLM::Transport::Response.from(raw)
+          res.success? ? b.call(res) : res
+        end
+      else
+        LLM::Transport::Response.from(client.request(request))
+      end
+    end
+  end
+end

data/lib/llm/usage.rb

@@ -4,13 +4,22 @@
 # The {LLM::Usage LLM::Usage} class represents token usage for
 # a given conversation or completion. As a conversation grows,
 # so does the number of tokens used. This class helps track
-# the number of input, output, reasoning and overall token count.
-# It can also help track usage of the context window (which may
-# vary by model).
-class LLM::Usage < Struct.new(:input_tokens, :output_tokens, :reasoning_tokens, :total_tokens, keyword_init: true)
+# the number of input, output, reasoning, cache, and overall
+# token count. It can also help track usage of the context
+# window (which may vary by model).
+class LLM::Usage < Struct.new(
+  :input_tokens, :output_tokens, :reasoning_tokens,
+  :input_audio_tokens, :output_audio_tokens, :input_image_tokens,
+  :cache_read_tokens, :cache_write_tokens, :total_tokens, keyword_init: true
+)
   ##
   # @return [String]
   def to_json(...)
-    LLM.json.dump({input_tokens:, output_tokens:, reasoning_tokens:, total_tokens:})
+    LLM.json.dump({
+      input_tokens:, output_tokens:,
+      reasoning_tokens:,
+      input_audio_tokens:, output_audio_tokens:, input_image_tokens:,
+      cache_read_tokens:, cache_write_tokens:, total_tokens:
+    })
   end
 end

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "8.0.0"
+  VERSION = "9.0.0"
 end