llm.rb 8.1.0 → 9.0.0

data/lib/llm/transport/persistent_http.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+class LLM::Transport
+  ##
+  # The {LLM::Transport::PersistentHTTP LLM::Transport::PersistentHTTP}
+  # transport is the built-in adapter for
+  # [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent).
+  # It manages pooled HTTP connections, tracks active requests by owner,
+  # and interrupts in-flight requests when needed.
+  #
+  # @api private
+  class PersistentHTTP < self
+    INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+    Request = Struct.new(:client, :connection, keyword_init: true)
+    @registry = {}
+    @monitor = Monitor.new
+
+    ##
+    # Returns the process-wide connection pool registry.
+    # @return [Hash]
+    def self.registry
+      @registry
+    end
+
+    def self.lock(&)
+      @monitor.synchronize(&)
+    end
+
+    ##
+    # @param [String] host
+    # @param [Integer] port
+    # @param [Integer] timeout
+    # @param [Boolean] ssl
+    # @return [LLM::Transport::PersistentHTTP]
+    def initialize(host:, port:, timeout:, ssl:)
+      @host = host
+      @port = port
+      @timeout = timeout
+      @ssl = ssl
+      @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
+      @monitor = Monitor.new
+    end
+
+    ##
+    # Returns the current request owner.
+    # @return [Object]
+    def request_owner
+      return Fiber.current unless defined?(::Async)
+      Async::Task.current? ? Async::Task.current : Fiber.current
+    end
+
+    ##
+    # @return [Array<Class<Exception>>]
+    def interrupt_errors
+      [*INTERRUPT_ERRORS, *optional_interrupt_errors]
+    end
+
+    ##
+    # Interrupt an active request, if any.
+    # @param [Fiber] owner
+    # @return [nil]
+    def interrupt!(owner)
+      req = request_for(owner) or return
+      lock { (@interrupts ||= {})[owner] = true }
+      close_socket(req.connection&.http)
+      req.client.finish(req.connection)
+      owner.stop if owner.respond_to?(:stop)
+    rescue *interrupt_errors
+      nil
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Fiber] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      lock { @interrupts&.delete(owner) }
+    end
+
+    ##
+    # Performs a request on the current HTTP transport.
+    # @param [Net::HTTPRequest] request
+    # @param [Fiber] owner
+    # @param [LLM::Object, nil] stream
+    # @yieldparam [LLM::Transport::Response] response
+    # @return [Object]
+    def request(request, owner:, stream: nil, &b)
+      client.connection_for(URI.join(base_uri, request.path)) do |connection|
+        set_request(Request.new(client:, connection:), owner)
+        perform_request(connection.http, request, stream, &b)
+      end
+    ensure
+      clear_request(owner)
+    end
+
+    private
+
+    attr_reader :host, :port, :timeout, :ssl, :base_uri
+
+    def client
+      self.class.lock do
+        if self.class.registry[client_id]
+          self.class.registry[client_id]
+        else
+          LLM.require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
+          client = Net::HTTP::Persistent.new(name: self.class.name)
+          client.read_timeout = timeout
+          client.open_timeout = timeout
+          self.class.registry[client_id] = client
+        end
+      end
+    end
+
+    def client_id
+      "#{host}:#{port}:#{timeout}:#{ssl}"
+    end
+
+    def close_socket(http)
+      socket = http&.instance_variable_get(:@socket) or return
+      socket = socket.io if socket.respond_to?(:io)
+      socket.close
+    rescue *interrupt_errors
+      nil
+    end
+
+    def request_for(owner)
+      lock do
+        @requests ||= {}
+        @requests[owner]
+      end
+    end
+
+    def set_request(req, owner)
+      lock do
+        @requests ||= {}
+        @requests[owner] = req
+      end
+    end
+
+    def clear_request(owner)
+      lock { @requests&.delete(owner) }
+    end
+
+    def lock(&)
+      @monitor.synchronize(&)
+    end
+
+    def optional_interrupt_errors
+      defined?(::Async::Stop) ? [Async::Stop] : []
+    end
+  end
+end

data/lib/llm/transport/response/http.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+class LLM::Transport::Response
+  ##
+  # {LLM::Transport::Response::HTTP LLM::Transport::Response::HTTP}
+  # adapts a {Net::HTTPResponse Net::HTTPResponse} to the
+  # {LLM::Transport::Response LLM::Transport::Response} interface.
+  #
+  # This is the default wrapper for responses produced by the built-in
+  # {LLM::Transport::HTTP LLM::Transport::HTTP} transport.
+  class HTTP < self
+    ##
+    # @return [Net::HTTPResponse]
+    attr_reader :res
+
+    ##
+    # @param [Net::HTTPResponse] res
+    # @return [LLM::Transport::Response::HTTP]
+    def initialize(res)
+      @res = res
+    end
+
+    ##
+    # @return [String]
+    def code
+      @res.code
+    end
+
+    ##
+    # @return [Object]
+    def body
+      @res.body
+    end
+
+    ##
+    # @param [Object] value
+    # @return [Object]
+    def body=(value)
+      @res.body = value
+    end
+
+    ##
+    # @param [String] key
+    # @return [String, nil]
+    def [](key)
+      @res[key]
+    end
+
+    ##
+    # @param [Object, nil] dest
+    # @yieldparam [String] chunk
+    # @return [void]
+    def read_body(dest = nil, &block)
+      if dest && block
+        @res.read_body(dest) { block.call(_1) }
+      elsif dest
+        @res.read_body(dest)
+      elsif block
+        @res.read_body { block.call(_1) }
+      else
+        @res.read_body
+      end
+    end
+
+    ##
+    # @return [Boolean]
+    def success?
+      Net::HTTPSuccess === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def ok?
+      Net::HTTPOK === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def bad_request?
+      Net::HTTPBadRequest === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def unauthorized?
+      Net::HTTPUnauthorized === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def forbidden?
+      Net::HTTPForbidden === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def not_found?
+      Net::HTTPNotFound === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def rate_limited?
+      Net::HTTPTooManyRequests === @res
+    end
+
+    ##
+    # @return [Boolean]
+    def server_error?
+      Net::HTTPServerError === @res
+    end
+  end
+end

data/lib/llm/transport/response.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+class LLM::Transport
+  ##
+  # {LLM::Transport::Response LLM::Transport::Response} defines the
+  # normalized HTTP response interface expected by transports and
+  # provider error handlers.
+  #
+  # Custom transports can execute requests through any underlying HTTP
+  # client, then adapt that client's native response object to this
+  # interface.
+  #
+  # This keeps the transport boundary focused on one contract:
+  # providers, execution, and error handlers only need a response
+  # object that implements
+  # {LLM::Transport::Response LLM::Transport::Response}, regardless of
+  # how the request was actually performed.
+  class Response
+    require_relative "response/http"
+
+    ##
+    # @param [Object] res
+    # @return [LLM::Transport::Response]
+    def self.from(res)
+      return res if LLM::Transport::Response === res
+      return HTTP.new(res) if Net::HTTPResponse === res
+      res
+    end
+
+    ##
+    # @return [String]
+    def code
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Object]
+    def body
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [Object] value
+    # @return [Object]
+    def body=(value)
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [String] key
+    # @return [String, nil]
+    def [](key)
+      raise NotImplementedError
+    end
+
+    ##
+    # @param [Object, nil] dest
+    # @yieldparam [String] chunk
+    # @return [void]
+    def read_body(dest = nil, &)
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def success?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def ok?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def bad_request?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def unauthorized?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def forbidden?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def not_found?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def rate_limited?
+      raise NotImplementedError
+    end
+
+    ##
+    # @return [Boolean]
+    def server_error?
+      raise NotImplementedError
+    end
+  end
+end

data/lib/llm/{provider/transport/http → transport}/stream_decoder.rb

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
-module LLM::Provider::Transport
+class LLM::Transport
   ##
-  # @private
-  class HTTP::StreamDecoder
+  # {LLM::Transport::StreamDecoder LLM::Transport::StreamDecoder}
+  # incrementally decodes streamed HTTP response bodies into parser
+  # events.
+  #
+  # @api private
+  class StreamDecoder
     ##
     # @return [Object]
     attr_reader :parser
 
     ##
     # @param [#parse!, #body] parser
-    # @return [LLM::Provider::Transport::HTTP::StreamDecoder]
+    # @return [LLM::Transport::StreamDecoder]
     def initialize(parser)
       @buffer = +""
       @cursor = 0

data/lib/llm/transport.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # The {LLM::Transport LLM::Transport} class defines the execution
+  # interface used by {LLM::Provider}.
+  #
+  # Custom transports can subclass this class and override {#request} to
+  # execute provider requests without changing request adapters or
+  # response adapters.
+  #
+  # Providers currently construct {Net::HTTPRequest Net::HTTPRequest}
+  # objects before delegating to a transport. Custom transports are
+  # therefore expected to execute those requests directly, or transform
+  # them into backend-specific request objects before execution.
+  #
+  # Only {#request} is required. The remaining methods are optional hooks
+  # for features such as interruption, request ownership, or persistence,
+  # and only need to be implemented when the underlying adapter can
+  # support them.
+  #
+  # Returned responses should implement the
+  # {LLM::Transport::Response LLM::Transport::Response} interface. In
+  # practice this can mean adapting another client's response object so
+  # existing provider execution, response adapters, and error handlers
+  # can rely on one normalized response contract instead of
+  # transport-specific classes.
+  class Transport
+    require_relative "transport/response"
+    require_relative "transport/stream_decoder"
+    require_relative "transport/http"
+    require_relative "transport/persistent_http"
+    require_relative "transport/execution"
+
+    ##
+    # Returns the built-in Net::HTTP transport class.
+    # @return [Class]
+    def self.net_http
+      HTTP
+    end
+
+    ##
+    # Returns the built-in Net::HTTP::Persistent transport class.
+    # @return [Class]
+    def self.net_http_persistent
+      PersistentHTTP
+    end
+
+    ##
+    # Performs a request through the transport.
+    # @param [Net::HTTPRequest] request
+    # @param [Object] owner
+    # @param [LLM::Object, nil] stream
+    # @yieldparam [LLM::Transport::Response] response
+    # @return [Object]
+    def request(request, owner:, stream: nil, &)
+      raise NotImplementedError
+    end
+
+    ##
+    # Returns the current request owner.
+    # @return [Object]
+    def request_owner
+      return Fiber.current unless defined?(::Async)
+      Async::Task.current? ? Async::Task.current : Fiber.current
+    end
+
+    ##
+    # Returns the exception classes that indicate an interrupted request.
+    # @return [Array<Class<Exception>>]
+    def interrupt_errors
+      []
+    end
+
+    ##
+    # Interrupt an active request, if any.
+    # @param [Object] owner
+    # @return [nil]
+    def interrupt!(owner)
+      raise NotImplementedError
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Object] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      nil
+    end
+
+    ##
+    # @note
+    #  Custom transports may be able to reuse this helper when they
+    #  operate on Net::HTTPRequest objects, or implement their own
+    #  request body preparation path instead.
+    # @param [Net::HTTPRequest] request
+    # @param [IO] io
+    # @return [void]
+    def set_body_stream(request, io)
+      request.body_stream = io
+      request["transfer-encoding"] = "chunked" unless request["content-length"]
+    end
+
+    private
+
+    ##
+    # @api private
+    # @note
+    #  Custom transports may be able to reuse this helper when they
+    #  execute requests through a Net::HTTP-compatible client, or
+    #  implement their own request execution path instead.
+    def perform_request(client, request, stream, &b)
+      if stream
+        client.request(request) do |raw|
+          res = LLM::Transport::Response.from(raw)
+          if res.success?
+            parser = stream.decoder.new(stream.parser.new(stream.streamer))
+            res.read_body(parser)
+            body = parser.body
+            res.body = (Hash === body || Array === body) ? LLM::Object.from(body) : body
+          else
+            body = +""
+            res.read_body { body << _1 }
+            res.body = body
+          end
+        ensure
+          parser&.free
+        end
+      elsif b
+        client.request(request) do |raw|
+          res = LLM::Transport::Response.from(raw)
+          res.success? ? b.call(res) : res
+        end
+      else
+        LLM::Transport::Response.from(client.request(request))
+      end
+    end
+  end
+end

data/lib/llm/usage.rb

@@ -4,13 +4,22 @@
 # The {LLM::Usage LLM::Usage} class represents token usage for
 # a given conversation or completion. As a conversation grows,
 # so does the number of tokens used. This class helps track
-# the number of input, output, reasoning and overall token count.
-# It can also help track usage of the context window (which may
-# vary by model).
-class LLM::Usage < Struct.new(:input_tokens, :output_tokens, :reasoning_tokens, :total_tokens, keyword_init: true)
+# the number of input, output, reasoning, cache, and overall
+# token count. It can also help track usage of the context
+# window (which may vary by model).
+class LLM::Usage < Struct.new(
+  :input_tokens, :output_tokens, :reasoning_tokens,
+  :input_audio_tokens, :output_audio_tokens, :input_image_tokens,
+  :cache_read_tokens, :cache_write_tokens, :total_tokens, keyword_init: true
+)
   ##
   # @return [String]
   def to_json(...)
-    LLM.json.dump({input_tokens:, output_tokens:, reasoning_tokens:, total_tokens:})
+    LLM.json.dump({
+      input_tokens:, output_tokens:,
+      reasoning_tokens:,
+      input_audio_tokens:, output_audio_tokens:, input_image_tokens:,
+      cache_read_tokens:, cache_write_tokens:, total_tokens:
+    })
   end
 end

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "8.1.0"
+  VERSION = "9.0.0"
 end

data/lib/llm.rb

@@ -14,8 +14,8 @@ module LLM
   require_relative "llm/object"
   require_relative "llm/model"
   require_relative "llm/version"
-  require_relative "llm/utils"
   require_relative "llm/message"
+  require_relative "llm/transport"
   require_relative "llm/response"
   require_relative "llm/mime"
   require_relative "llm/multipart"
@@ -37,22 +37,12 @@ module LLM
 
   ##
   # Thread-safe monitors for different contexts
-  @monitors = {require: Monitor.new, clients: Monitor.new, inherited: Monitor.new, registry: Monitor.new, mcp: Monitor.new}
+  @monitors = {require: Monitor.new, inherited: Monitor.new, registry: Monitor.new, mcp: Monitor.new}
 
   ##
   # Model registry
   @registry = {}
 
-  ##
-  # Shared HTTP clients used by providers.
-  @clients = {}
-
-  ##
-  # @api private
-  def self.clients
-    @clients
-  end
-
   ##
   # Requires an optional runtime dependency
   # @raise [LLM::DependencyError]

data/llm.gemspec

@@ -8,22 +8,8 @@ Gem::Specification.new do |spec|
   spec.authors = ["Antar Azri", "0x1eef", "Christos Maris", "Rodrigo Serrano"]
   spec.email = ["azantar@proton.me", "0x1eef@hardenedbsd.org"]
 
-  spec.summary = "Lightweight runtime for building capable AI systems in Ruby."
-
-  spec.description = <<~DESCRIPTION
-  llm.rb is a lightweight runtime for building capable AI systems in Ruby.
-  It is not just an API wrapper. llm.rb gives you one runtime for providers,
-  contexts, agents, tools, MCP servers, streaming, schemas, files, and
-  persisted state, so real systems can be built out of one coherent
-  execution model instead of a pile of adapters. It stays close to Ruby, runs
-  on the standard library by default, loads optional pieces only when needed,
-  includes built-in ActiveRecord support through acts_as_llm and
-  acts_as_agent, includes built-in Sequel support through plugin :llm,
-  and is designed for engineers who want control over long-lived,
-  tool-capable, stateful AI workflows instead of just request/response
-  helpers.
-  DESCRIPTION
-
+  spec.summary = "Ruby's most capable AI runtime"
+  spec.description = spec.summary
   spec.license = "0BSD"
   spec.required_ruby_version = ">= 3.3.0"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm.rb
 version: !ruby/object:Gem::Version
-  version: 8.1.0
+  version: 9.0.0
 platform: ruby
 authors:
 - Antar Azri
@@ -264,18 +264,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
-description: |
-  llm.rb is a lightweight runtime for building capable AI systems in Ruby.
-  It is not just an API wrapper. llm.rb gives you one runtime for providers,
-  contexts, agents, tools, MCP servers, streaming, schemas, files, and
-  persisted state, so real systems can be built out of one coherent
-  execution model instead of a pile of adapters. It stays close to Ruby, runs
-  on the standard library by default, loads optional pieces only when needed,
-  includes built-in ActiveRecord support through acts_as_llm and
-  acts_as_agent, includes built-in Sequel support through plugin :llm,
-  and is designed for engineers who want control over long-lived,
-  tool-capable, stateful AI workflows instead of just request/response
-  helpers.
+description: Ruby's most capable AI runtime
 email:
 - azantar@proton.me
 - 0x1eef@hardenedbsd.org
@@ -315,6 +304,7 @@ files:
 - lib/llm/file.rb
 - lib/llm/function.rb
 - lib/llm/function/array.rb
+- lib/llm/function/call_group.rb
 - lib/llm/function/fiber_group.rb
 - lib/llm/function/fork.rb
 - lib/llm/function/fork/job.rb
@@ -352,10 +342,6 @@ files:
 - lib/llm/pipe.rb
 - lib/llm/prompt.rb
 - lib/llm/provider.rb
-- lib/llm/provider/transport/http.rb
-- lib/llm/provider/transport/http/execution.rb
-- lib/llm/provider/transport/http/interruptible.rb
-- lib/llm/provider/transport/http/stream_decoder.rb
 - lib/llm/providers/anthropic.rb
 - lib/llm/providers/anthropic/error_handler.rb
 - lib/llm/providers/anthropic/files.rb
@@ -473,8 +459,14 @@ files:
 - lib/llm/tracer/logger.rb
 - lib/llm/tracer/null.rb
 - lib/llm/tracer/telemetry.rb
+- lib/llm/transport.rb
+- lib/llm/transport/execution.rb
+- lib/llm/transport/http.rb
+- lib/llm/transport/persistent_http.rb
+- lib/llm/transport/response.rb
+- lib/llm/transport/response/http.rb
+- lib/llm/transport/stream_decoder.rb
 - lib/llm/usage.rb
-- lib/llm/utils.rb
 - lib/llm/version.rb
 - lib/sequel/plugins/agent.rb
 - lib/sequel/plugins/llm.rb
@@ -503,5 +495,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.3
 specification_version: 4
-summary: Lightweight runtime for building capable AI systems in Ruby.
+summary: Ruby's most capable AI runtime
 test_files: []