llm.rb 8.1.0 → 9.0.0

data/lib/llm/mcp.rb

@@ -24,14 +24,6 @@ class LLM::MCP
 
   include RPC
 
-  @clients = {}
-
-  ##
-  # @api private
-  def self.clients
-    @clients
-  end
-
   ##
   # Builds an MCP client that uses the stdio transport.
   # @param [LLM::Provider, nil] llm
@@ -69,6 +61,9 @@ class LLM::MCP
   #  The URL for the MCP HTTP endpoint
   # @option http [Hash] :headers
   #  Extra headers for requests
+  # @option http [LLM::Transport, Class] :transport
+  #  Optional override with any {LLM::Transport} instance or subclass,
+  #  similar to {LLM::Provider}
   # @param [Integer] timeout
   #  The maximum amount of time to wait when reading from an MCP process
   # @return [LLM::MCP] A new MCP instance
@@ -82,8 +77,9 @@ class LLM::MCP
       @transport = Transport::Stdio.new(command:)
     elsif http
       persistent = http.delete(:persistent)
-      @transport = Transport::HTTP.new(**http, timeout:)
-      @transport.persistent if persistent
+      transport = http.delete(:transport)
+      transport ||= LLM::Transport::PersistentHTTP if persistent
+      @transport = Transport::HTTP.new(**http, timeout:, transport:)
     else
       raise ArgumentError, "stdio or http is required"
     end
@@ -121,19 +117,6 @@ class LLM::MCP
     stop
   end
 
-  ##
-  # Configures an HTTP MCP transport to use a persistent connection pool
-  # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-  # @example
-  #   mcp = LLM::MCP.http(url: "https://example.com/mcp", persistent: true)
-  #   # do something with 'mcp'
-  # @return [LLM::MCP]
-  def persist!
-    transport.persist!
-    self
-  end
-  alias_method :persistent, :persist!
-
   ##
   # Returns the tools provided by the MCP process.
   # @return [Array<Class<LLM::Tool>>]

data/lib/llm/provider.rb

@@ -6,10 +6,7 @@
 #
 # @abstract
 class LLM::Provider
-  require "net/http"
-  require_relative "provider/transport/http"
-  require_relative "provider/transport/http/execution"
-  include Transport::HTTP::Execution
+  include LLM::Transport::Execution
 
   ##
   # @param [String, nil] key
@@ -27,7 +24,9 @@ class LLM::Provider
   # @param [Boolean] persistent
   #  Whether to use a persistent connection.
   #  Requires the net-http-persistent gem.
-  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, base_path: "", persistent: false)
+  # @param [LLM::Transport, Class, nil] transport
+  #  Optional override with any {LLM::Transport} instance or subclass.
+  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, base_path: "", persistent: false, transport: nil)
     @key = key
     @host = host
     @port = port
@@ -36,7 +35,7 @@ class LLM::Provider
     @base_path = normalize_base_path(base_path)
     @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
     @headers = {"User-Agent" => "llm.rb v#{LLM::VERSION}"}
-    @transport = Transport::HTTP.new(host:, port:, timeout:, ssl:, persistent:)
+    @transport = resolve_transport(transport, persistent:)
     @monitor = Monitor.new
   end
 
@@ -316,19 +315,6 @@ class LLM::Provider
     end
   end
 
-  ##
-  # This method configures a provider to use a persistent connection pool
-  # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-  # @example
-  #   llm = LLM.openai(key: ENV["KEY"]).persistent
-  #   # do something with 'llm'
-  # @return [LLM::Provider]
-  def persist!
-    transport.persist!
-    self
-  end
-  alias_method :persistent, :persist!
-
   ##
   # Interrupt the active request, if any.
   # @param [Fiber] owner
@@ -403,7 +389,7 @@ class LLM::Provider
   # @return [Class]
   #  Returns the class responsible for decoding streamed response bodies
   def stream_decoder
-    LLM::Provider::Transport::HTTP::StreamDecoder
+    LLM::Transport::StreamDecoder
   end
 
   ##
@@ -431,6 +417,23 @@ class LLM::Provider
     @monitor.synchronize(&)
   end
 
+  ##
+  # @api private
+  def default_transport(persistent:)
+    transport_class = persistent ? LLM::Transport::PersistentHTTP : LLM::Transport::HTTP
+    transport_class.new(host:, port:, timeout:, ssl:)
+  end
+
+  ##
+  # @api private
+  def resolve_transport(transport, persistent:)
+    return default_transport(persistent:) if transport.nil?
+    if Class === transport && transport <= LLM::Transport
+      return transport.new(host:, port:, timeout:, ssl:)
+    end
+    transport
+  end
+
   ##
   # @api private
   def thread

data/lib/llm/providers/anthropic/error_handler.rb

@@ -5,7 +5,7 @@ class LLM::Anthropic
   # @private
   class ErrorHandler
     ##
-    # @return [Net::HTTPResponse]
+    # @return [LLM::Transport::Response]
     #  Non-2XX response from the server
     attr_reader :res
 
@@ -19,13 +19,13 @@ class LLM::Anthropic
     #  The tracer
     # @param [Object, nil] span
     #  The span
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     #  The response from the server
     # @return [LLM::Anthropic::ErrorHandler]
     def initialize(tracer, span, res)
       @tracer = tracer
       @span = span
-      @res = res
+      @res = LLM::Transport::Response.from(res)
     end
 
     ##
@@ -43,12 +43,11 @@ class LLM::Anthropic
     ##
     # @return [LLM::Error]
     def error
-      case res
-      when Net::HTTPServerError
+      if res.server_error?
         LLM::ServerError.new("Server error").tap { _1.response = res }
-      when Net::HTTPUnauthorized
+      elsif res.unauthorized?
         LLM::UnauthorizedError.new("Authentication error").tap { _1.response = res }
-      when Net::HTTPTooManyRequests
+      elsif res.rate_limited?
         LLM::RateLimitError.new("Too many requests").tap { _1.response = res }
       else
         LLM::Error.new("Unexpected response").tap { _1.response = res }

data/lib/llm/providers/anthropic/files.rb

@@ -58,7 +58,7 @@ class LLM::Anthropic
       multi = LLM::Multipart.new(params.merge!(file: LLM.File(file)))
       req = Net::HTTP::Post.new("/v1/files", headers)
       req["content-type"] = multi.content_type
-      set_body_stream(req, multi.body)
+      transport.set_body_stream(req, multi.body)
       res, span, tracer = execute(request: req, operation: "request")
       res = ResponseAdapter.adapt(res, type: :file)
       tracer.on_request_finish(operation: "request", res:, span:)
@@ -159,7 +159,7 @@ class LLM::Anthropic
       @provider.instance_variable_get(:@key)
     end
 
-    [:headers, :execute, :set_body_stream].each do |m|
+    [:headers, :execute, :transport].each do |m|
       define_method(m) { |*args, **kwargs, &b| @provider.send(m, *args, **kwargs, &b) }
     end
   end

data/lib/llm/providers/anthropic/response_adapter/completion.rb

@@ -27,6 +27,36 @@ module LLM::Anthropic::ResponseAdapter
       0
     end
 
+    ##
+    # (see LLM::Contract::Completion#input_audio_tokens)
+    def input_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#output_audio_tokens)
+    def output_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#input_image_tokens)
+    def input_image_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_read_tokens)
+    def cache_read_tokens
+      body.usage&.cache_read_input_tokens || 0
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_write_tokens)
+    def cache_write_tokens
+      body.usage&.cache_creation_input_tokens || 0
+    end
+
     ##
     # (see LLM::Contract::Completion#total_tokens)
     def total_tokens

data/lib/llm/providers/anthropic.rb

@@ -161,7 +161,7 @@ module LLM
       payload = adapt(messages)
       body = LLM.json.dump(payload.merge!(params))
       req = Net::HTTP::Post.new("/v1/messages", headers)
-      set_body_stream(req, StringIO.new(body))
+      transport.set_body_stream(req, StringIO.new(body))
       req
     end
 

data/lib/llm/providers/bedrock/error_handler.rb

@@ -11,7 +11,7 @@ class LLM::Bedrock
   # @api private
   class ErrorHandler
     ##
-    # @return [Net::HTTPResponse]
+    # @return [LLM::Transport::Response]
     attr_reader :res
 
     ##
@@ -21,12 +21,12 @@ class LLM::Bedrock
     ##
     # @param [LLM::Tracer] tracer
     # @param [Object, nil] span
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     # @return [LLM::Bedrock::ErrorHandler]
     def initialize(tracer, span, res)
       @tracer = tracer
       @span = span
-      @res = res
+      @res = LLM::Transport::Response.from(res)
     end
 
     ##
@@ -44,16 +44,15 @@ class LLM::Bedrock
     # @return [LLM::Error]
     def error
       message = extract_message
-      case res
-      when Net::HTTPServerError
+      if res.server_error?
         LLM::ServerError.new(message).tap { _1.response = res }
-      when Net::HTTPUnauthorized
+      elsif res.unauthorized?
         LLM::UnauthorizedError.new(message).tap { _1.response = res }
-      when Net::HTTPForbidden
+      elsif res.forbidden?
         LLM::UnauthorizedError.new(message).tap { _1.response = res }
-      when Net::HTTPTooManyRequests
+      elsif res.rate_limited?
         LLM::RateLimitError.new(message).tap { _1.response = res }
-      when Net::HTTPNotFound
+      elsif res.not_found?
         LLM::Error.new("Bedrock model not found: #{message}").tap { _1.response = res }
       else
         LLM::Error.new(message).tap { _1.response = res }

data/lib/llm/providers/bedrock/models.rb

@@ -8,8 +8,9 @@ class LLM::Bedrock
   #
   # Unlike the Converse API (which lives on `bedrock-runtime.<region>.amazonaws.com`),
   # the models endpoint lives on the control plane at
-  # `bedrock.<region>.amazonaws.com`. This class manages its own HTTP
-  # connection since the provider's transport is pinned to the runtime host.
+  # `bedrock.<region>.amazonaws.com`. This class builds a matching
+  # transport for the control-plane host from the provider's current
+  # transport class.
   #
   # @example
   #   llm = LLM.bedrock(
@@ -39,19 +40,18 @@ class LLM::Bedrock
     # @return [LLM::Response]
     def all(**params)
       host = credentials.host
-      handle_response http(host).request(build_request(host, params))
+      req = build_request(host, params)
+      res = build_transport(host).request(req, owner: self)
+      handle_response(res)
     end
 
     private
 
     ##
     # @param [String] host
-    # @return [Net::HTTP]
-    def http(host)
-      http = Net::HTTP.new(host, 443)
-      http.use_ssl = true
-      http.read_timeout = timeout
-      http
+    # @return [LLM::Transport]
+    def build_transport(host)
+      transport.class.new(host:, port: 443, timeout:, ssl: true)
     end
 
     ##
@@ -68,12 +68,12 @@ class LLM::Bedrock
     end
 
     ##
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     # @return [LLM::Response]
     # @raise [LLM::Error]
     def handle_response(res)
-      case res
-      when Net::HTTPSuccess
+      res = LLM::Transport::Response.from(res)
+      if res.success?
         res.body = LLM::Object.from(LLM.json.load(res.body || "{}"))
         LLM::Bedrock::ResponseAdapter.adapt(res, type: :models)
       else
@@ -102,7 +102,7 @@ class LLM::Bedrock
       end
     end
 
-    [:timeout, :tracer].each do |m|
+    [:timeout, :tracer, :transport].each do |m|
       define_method(m) { @provider.send(m) }
     end
   end

data/lib/llm/providers/bedrock/response_adapter/completion.rb

@@ -56,6 +56,36 @@ module LLM::Bedrock::ResponseAdapter
       0
     end
 
+    ##
+    # (see LLM::Contract::Completion#input_audio_tokens)
+    def input_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#output_audio_tokens)
+    def output_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#input_image_tokens)
+    def input_image_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_read_tokens)
+    def cache_read_tokens
+      0
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_write_tokens)
+    def cache_write_tokens
+      0
+    end
+
     ##
     # (see LLM::Contract::Completion#total_tokens)
     def total_tokens

data/lib/llm/providers/bedrock.rb

@@ -218,7 +218,7 @@ module LLM
       path = stream ? "/model/#{model_id}/converse-stream" \
                     : "/model/#{model_id}/converse"
       req = Net::HTTP::Post.new(path, headers)
-      set_body_stream(req, StringIO.new(body))
+      transport.set_body_stream(req, StringIO.new(body))
       [req, messages, body]
     end
 

data/lib/llm/providers/google/error_handler.rb

@@ -5,7 +5,7 @@ class LLM::Google
   # @private
   class ErrorHandler
     ##
-    # @return [Net::HTTPResponse]
+    # @return [LLM::Transport::Response]
     #  Non-2XX response from the server
     attr_reader :res
 
@@ -19,13 +19,13 @@ class LLM::Google
     #  The tracer
     # @param [Object, nil] span
     #  The span
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     #  The response from the server
     # @return [LLM::Google::ErrorHandler]
     def initialize(tracer, span, res)
       @tracer = tracer
       @span = span
-      @res = res
+      @res = LLM::Transport::Response.from(res)
     end
 
     ##
@@ -49,17 +49,16 @@ class LLM::Google
     ##
     # @return [LLM::Error]
     def error
-      case res
-      when Net::HTTPServerError
+      if res.server_error?
         LLM::ServerError.new("Server error").tap { _1.response = res }
-      when Net::HTTPBadRequest
+      elsif res.bad_request?
         reason = body.dig("error", "details", 0, "reason")
         if reason == "API_KEY_INVALID"
           LLM::UnauthorizedError.new("Authentication error").tap { _1.response = res }
         else
           LLM::Error.new("Unexpected response").tap { _1.response = res }
         end
-      when Net::HTTPTooManyRequests
+      elsif res.rate_limited?
         LLM::RateLimitError.new("Too many requests").tap { _1.response = res }
       else
         LLM::Error.new("Unexpected response").tap { _1.response = res }

data/lib/llm/providers/google/files.rb

@@ -69,7 +69,7 @@ class LLM::Google
       req["X-Goog-Upload-Offset"] = 0
       req["X-Goog-Upload-Command"] = "upload, finalize"
       file.with_io do |io|
-        set_body_stream(req, io)
+        transport.set_body_stream(req, io)
         res, span, tracer = execute(request: req, operation: "request")
         res = ResponseAdapter.adapt(res, type: :file)
         tracer.on_request_finish(operation: "request", res:, span:)
@@ -127,8 +127,6 @@ class LLM::Google
 
     private
 
-    include LLM::Utils
-
     def request_upload_url(file:)
       req = Net::HTTP::Post.new("/upload/v1beta/files?key=#{key}", headers)
       req["X-Goog-Upload-Protocol"] = "resumable"
@@ -146,7 +144,7 @@ class LLM::Google
       @provider.instance_variable_get(:@key)
     end
 
-    [:headers, :execute, :set_body_stream].each do |m|
+    [:headers, :execute, :transport].each do |m|
       define_method(m) { |*args, **kwargs, &b| @provider.send(m, *args, **kwargs, &b) }
     end
   end

data/lib/llm/providers/google/images.rb

@@ -91,7 +91,7 @@ class LLM::Google
       @provider.instance_variable_get(:@key)
     end
 
-    [:headers, :execute, :set_body_stream].each do |m|
+    [:headers, :execute].each do |m|
       define_method(m) { |*args, **kwargs, &b| @provider.send(m, *args, **kwargs, &b) }
     end
   end

data/lib/llm/providers/google/models.rb

@@ -17,8 +17,6 @@ class LLM::Google
   #     print "id: ", model.id, "\n"
   #   end
   class Models
-    include LLM::Utils
-
     ##
     # Returns a new Models object
     # @param provider [LLM::Provider]

data/lib/llm/providers/google/response_adapter/completion.rb

@@ -33,6 +33,36 @@ module LLM::Google::ResponseAdapter
       body.usageMetadata.thoughtsTokenCount || 0
     end
 
+    ##
+    # (see LLM::Contract::Completion#input_audio_tokens)
+    def input_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#output_audio_tokens)
+    def output_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#input_image_tokens)
+    def input_image_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_read_tokens)
+    def cache_read_tokens
+      0
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_write_tokens)
+    def cache_write_tokens
+      0
+    end
+
     ##
     # (see LLM::Contract::Completion#total_tokens)
     def total_tokens

data/lib/llm/providers/google.rb

@@ -208,7 +208,7 @@ module LLM
       req  = Net::HTTP::Post.new(path, headers)
       messages = build_complete_messages(prompt, params, role)
       body = LLM.json.dump({contents: adapt(messages)}.merge!(params))
-      set_body_stream(req, StringIO.new(body))
+      transport.set_body_stream(req, StringIO.new(body))
       req
     end
 

data/lib/llm/providers/ollama/error_handler.rb

@@ -5,7 +5,7 @@ class LLM::Ollama
   # @private
   class ErrorHandler
     ##
-    # @return [Net::HTTPResponse]
+    # @return [LLM::Transport::Response]
     #  Non-2XX response from the server
     attr_reader :res
 
@@ -19,13 +19,13 @@ class LLM::Ollama
     #  The tracer
     # @param [Object, nil] span
     #  The span
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     #  The response from the server
     # @return [LLM::Ollama::ErrorHandler]
     def initialize(tracer, span, res)
       @tracer = tracer
       @span = span
-      @res = res
+      @res = LLM::Transport::Response.from(res)
     end
 
     ##
@@ -43,12 +43,11 @@ class LLM::Ollama
     ##
     # @return [LLM::Error]
     def error
-      case res
-      when Net::HTTPServerError
+      if res.server_error?
         LLM::ServerError.new("Server error").tap { _1.response = res }
-      when Net::HTTPUnauthorized
+      elsif res.unauthorized?
         LLM::UnauthorizedError.new("Authentication error").tap { _1.response = res }
-      when Net::HTTPTooManyRequests
+      elsif res.rate_limited?
         LLM::RateLimitError.new("Too many requests").tap { _1.response = res }
       else
         LLM::Error.new("Unexpected response").tap { _1.response = res }

data/lib/llm/providers/ollama/models.rb

@@ -17,8 +17,6 @@ class LLM::Ollama
   #     print "id: ", model.id, "\n"
   #   end
   class Models
-    include LLM::Utils
-
     ##
     # Returns a new Models object
     # @param provider [LLM::Provider]

data/lib/llm/providers/ollama/response_adapter/completion.rb

@@ -27,6 +27,36 @@ module LLM::Ollama::ResponseAdapter
       0
     end
 
+    ##
+    # (see LLM::Contract::Completion#input_audio_tokens)
+    def input_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#output_audio_tokens)
+    def output_audio_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#input_image_tokens)
+    def input_image_tokens
+      super
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_read_tokens)
+    def cache_read_tokens
+      0
+    end
+
+    ##
+    # (see LLM::Contract::Completion#cache_write_tokens)
+    def cache_write_tokens
+      0
+    end
+
     ##
     # (see LLM::Contract::Completion#total_tokens)
     def total_tokens

data/lib/llm/providers/ollama.rb

@@ -130,7 +130,7 @@ module LLM
       messages = build_complete_messages(prompt, params, role)
       body = LLM.json.dump({messages: [adapt(messages)].flatten}.merge!(params))
       req = Net::HTTP::Post.new("/api/chat", headers)
-      set_body_stream(req, StringIO.new(body))
+      transport.set_body_stream(req, StringIO.new(body))
       req
     end
 

data/lib/llm/providers/openai/audio.rb

@@ -57,7 +57,7 @@ class LLM::OpenAI
       multi = LLM::Multipart.new(params.merge!(file: LLM.File(file), model:))
       req = Net::HTTP::Post.new(path("/audio/transcriptions"), headers)
       req["content-type"] = multi.content_type
-      set_body_stream(req, multi.body)
+      transport.set_body_stream(req, multi.body)
       res, span, tracer = execute(request: req, operation: "request")
       res = LLM::Response.new(res)
       tracer.on_request_finish(operation: "request", model:, res:, span:)
@@ -81,7 +81,7 @@ class LLM::OpenAI
       multi = LLM::Multipart.new(params.merge!(file: LLM.File(file), model:))
       req = Net::HTTP::Post.new(path("/audio/translations"), headers)
       req["content-type"] = multi.content_type
-      set_body_stream(req, multi.body)
+      transport.set_body_stream(req, multi.body)
       res, span, tracer = execute(request: req, operation: "request")
       res = LLM::Response.new(res)
       tracer.on_request_finish(operation: "request", model:, res:, span:)
@@ -90,7 +90,7 @@ class LLM::OpenAI
 
     private
 
-    [:path, :headers, :execute, :set_body_stream].each do |m|
+    [:path, :headers, :execute, :transport].each do |m|
       define_method(m) { |*args, **kwargs, &b| @provider.send(m, *args, **kwargs, &b) }
     end
   end