llm.rb 8.1.0 → 10.0.0

data/lib/llm/cost.rb

@@ -2,19 +2,96 @@
 
 ##
 # The {LLM::Cost LLM::Cost} class represents an approximate
-# cost breakdown for a provider request. It stores the input
-# and output costs separately and can return the total.
+# cost breakdown for a provider request. It stores input,
+# output, input audio, output audio, input image, cache read, cache write,
+# and reasoning costs separately and can return the total.
 #
 # @attr [Float] input_costs
 #   Returns the input cost
 # @attr [Float] output_costs
 #   Returns the output cost
-class LLM::Cost < Struct.new(:input_costs, :output_costs)
+# @attr [Float, nil] input_audio_costs
+#   Returns the input audio cost, or nil when no input audio tokens
+#   were used
+# @attr [Float, nil] output_audio_costs
+#   Returns the output audio cost, or nil when no output audio tokens
+#   were used
+# @attr [Float, nil] input_image_costs
+#   Returns the input image cost, or nil when no input image tokens
+#   were used
+# @attr [Float, nil] cache_read_costs
+#   Returns the cache read cost, or nil when no cache tokens
+#   were used
+# @attr [Float, nil] cache_write_costs
+#   Returns the cache write cost, or nil when no cache creation
+#   tokens were used
+# @attr [Float, nil] reasoning_costs
+#   Returns the reasoning cost, or nil when no reasoning tokens
+#   were used
+class LLM::Cost < Struct.new(
+  :input_costs, :output_costs,
+  :input_audio_costs, :output_audio_costs,
+  :cache_read_costs, :cache_write_costs,
+  :input_image_costs, :reasoning_costs,
+  keyword_init: true
+)
+  ##
+  # Build a cost breakdown from token usage and model pricing.
+  # @param [LLM::Context]
+  #  Context used to resolve provider, model, and token usage
+  # @return [LLM::Cost]
+  def self.from(ctx)
+    pricing = LLM.registry_for(ctx.llm).cost(model: ctx.model)
+    new(
+      input_costs: price(pricing.input, ctx.usage.input_tokens),
+      output_costs: price(pricing.output, ctx.usage.output_tokens),
+      input_audio_costs: price(pricing.input_audio, ctx.usage.input_audio_tokens),
+      output_audio_costs: price(pricing.output_audio, ctx.usage.output_audio_tokens),
+      input_image_costs: price(pricing.input, ctx.usage.input_image_tokens),
+      cache_read_costs: price(pricing.cache_read, ctx.usage.cache_read_tokens),
+      cache_write_costs: price(pricing.cache_write, ctx.usage.cache_write_tokens),
+      reasoning_costs: price(pricing.output, ctx.usage.reasoning_tokens)
+    )
+  rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
+    new
+  end
+
+  ##
+  # @api private
+  def self.price(rate, tokens)
+    return if tokens.nil? || tokens.to_i.zero?
+    return if rate.nil? || rate.to_f.zero?
+    ((rate.to_f / 1_000_000.0) * tokens.to_i).round(12)
+  end
+  private_class_method :price
+
   ##
   # @return [Float]
   #  Returns the total cost
   def total
-    input_costs + output_costs
+    [
+      input_costs, output_costs,
+      input_audio_costs, output_audio_costs,
+      cache_read_costs, cache_write_costs,
+      input_image_costs, reasoning_costs
+    ].compact.sum.round(12)
+  end
+
+  ##
+  # @return [Hash]
+  #  Returns a hash with the non-nil cost components and the total
+  def to_h
+    {
+      input: input_costs,
+      output: output_costs,
+      input_audio: input_audio_costs,
+      output_audio: output_audio_costs,
+      input_image: input_image_costs,
+      cache_read: cache_read_costs,
+      cache_write: cache_write_costs,
+      reasoning: reasoning_costs,
+      total: total
+    }.compact
   end
 
   ##

data/lib/llm/error.rb

@@ -5,7 +5,7 @@ module LLM
   # The superclass of all LLM errors
   class Error < RuntimeError
     ##
-    # @return [Net::HTTPResponse, nil]
+    # @return [LLM::Transport::Response, nil]
     #  Returns the response associated with an error, or nil
     attr_accessor :response
 

data/lib/llm/function/array.rb

@@ -18,21 +18,23 @@ class LLM::Function
 
     ##
     # Calls all functions in a collection concurrently.
-    # This method returns an {LLM::Function::ThreadGroup},
-    # {LLM::Function::TaskGroup}, or {LLM::Function::FiberGroup}
-    # that can be waited on to access the return values.
+    # This method returns an execution group that can be
+    # waited on to access the return values.
     #
     # @param [Symbol] strategy
     #   Controls concurrency strategy:
+    #   - `:call`: Call functions sequentially without spawning
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)
     #   - `:fork`: Use forked child processes
     #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #
-    # @return [LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
+    # @return [LLM::Function::CallGroup, LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
     def spawn(strategy)
       case strategy
+      when :call
+        CallGroup.new(self)
       when :task
         TaskGroup.new(map { |fn| fn.spawn(:task) })
       when :thread
@@ -44,7 +46,7 @@ class LLM::Function
       when :ractor
         Ractor::Group.new(map { |fn| fn.spawn(:ractor) })
       else
-        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, :fork, or :ractor"
+        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :call, :thread, :task, :fiber, :fork, or :ractor"
       end
     end
 
@@ -54,6 +56,7 @@ class LLM::Function
     #
     # @param [Symbol] strategy
     #   Controls concurrency strategy:
+    #   - `:call`: Call each function sequentially through a call group
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)

data/lib/llm/function/call_group.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::CallGroup} class wraps an array of
+  # {LLM::Function} objects for sequential execution.
+  #
+  # It provides the same basic interface as the concurrent group
+  # wrappers so callers can flow through `spawn(strategy).wait`
+  # uniformly, even when the selected strategy is direct calls.
+  class CallGroup
+    ##
+    # @param [Array<LLM::Function>] functions
+    # @return [LLM::Function::CallGroup]
+    def initialize(functions)
+      @functions = functions
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      false
+    end
+
+    ##
+    # @return [nil]
+    def interrupt!
+      nil
+    end
+    alias_method :cancel!, :interrupt!
+
+    ##
+    # @return [Array<LLM::Function::Return>]
+    def wait
+      @functions.map(&:call)
+    end
+    alias_method :value, :wait
+  end
+end

data/lib/llm/function/call_task.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::CallTask} class wraps a single direct function call
+  # behind the same task-like interface used by spawned concurrency modes.
+  class CallTask
+    ##
+    # @return [LLM::Function]
+    attr_reader :function
+
+    ##
+    # @param [LLM::Function] function
+    # @return [LLM::Function::CallTask]
+    def initialize(function)
+      @function = function
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      false
+    end
+
+    ##
+    # @return [nil]
+    def interrupt!
+      function.interrupt!
+      nil
+    end
+    alias_method :cancel!, :interrupt!
+
+    ##
+    # @return [LLM::Function::Return]
+    def wait
+      function.call
+    end
+    alias_method :value, :wait
+
+    ##
+    # @return [Class]
+    def group_class
+      LLM::Function::TaskGroup
+    end
+  end
+end

data/lib/llm/function/fork/task.rb

@@ -63,6 +63,12 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      LLM::Function::Fork::Group
+    end
+
     private
 
     def reap

data/lib/llm/function/ractor/task.rb

@@ -57,6 +57,12 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      LLM::Function::Ractor::Group
+    end
+
     private
 
     def build_task

data/lib/llm/function/task.rb

@@ -53,6 +53,16 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      case task
+      when Thread then LLM::Function::ThreadGroup
+      when Fiber then LLM::Function::FiberGroup
+      else LLM::Function::TaskGroup
+      end
+    end
+
     private
 
     def scheduler

data/lib/llm/function.rb

@@ -32,6 +32,8 @@ class LLM::Function
   require_relative "function/registry"
   require_relative "function/tracing"
   require_relative "function/array"
+  require_relative "function/call_group"
+  require_relative "function/call_task"
   require_relative "function/task"
   require_relative "function/thread_group"
   require_relative "function/fiber_group"
@@ -209,6 +211,7 @@ class LLM::Function
   #
   # @param [Symbol] strategy
   #   Controls concurrency strategy:
+  #   - `:call`: Call the function sequentially without spawning
   #   - `:thread`: Use threads
   #   - `:task`: Use async tasks (requires async gem)
   #   - `:fork`: Use a forked child process (requires xchan.rb support)
@@ -220,6 +223,8 @@ class LLM::Function
   #   Returns a task whose `#value` is an {LLM::Function::Return}.
   def spawn(strategy)
     task = case strategy
+    when :call
+      CallTask.new(self)
     when :task
       LLM.require "async" unless defined?(::Async)
       Async { call! }
@@ -240,7 +245,7 @@ class LLM::Function
       span = @tracer&.on_tool_start(id:, name:, arguments:, model:)
       Ractor::Task.new(@runner, id, name, arguments, tracer: @tracer, span:).spawn
     else
-      raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, :fork, or :ractor"
+      raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :call, :thread, :task, :fiber, :fork, or :ractor"
     end
     Task.new(task, self)
   ensure
@@ -294,6 +299,28 @@ class LLM::Function
     !@called && !@cancelled
   end
 
+  ##
+  # Returns an in-band error for an unresolved function call.
+  # @return [LLM::Function::Return]
+  def unavailable
+    Return.new(id, name, {
+      error: true,
+      type: LLM::NoSuchToolError.name,
+      message: "tool not found"
+    })
+  end
+
+  ##
+  # Returns an in-band error for a tool loop rate limit.
+  # @return [LLM::Function::Return]
+  def rate_limit
+    LLM::Function::Return.new(id, name, {
+      error: true,
+      type: LLM::ToolLoopError.name,
+      message: "tool loop rate limit reached"
+    })
+  end
+
   ##
   # @return [Hash]
   def adapt(provider)

data/lib/llm/mcp/transport/http.rb

@@ -16,12 +16,13 @@ module LLM::MCP::Transport
     #  Extra headers to send with requests
     # @param [Integer, nil] timeout
     #  The timeout in seconds. Defaults to nil
+    # @param [LLM::Transport, Class, nil] transport
+    #  Optional override with any {LLM::Transport} instance or subclass
     # @return [LLM::MCP::Transport::HTTP]
-    def initialize(url:, headers: {}, timeout: nil)
+    def initialize(url:, headers: {}, timeout: nil, transport: nil)
       @uri = URI.parse(url)
-      @use_ssl = @uri.scheme == "https"
       @headers = headers
-      @timeout = timeout
+      @transport = resolve_transport(transport, timeout:)
       @queue = []
       @monitor = Monitor.new
       @running = false
@@ -61,21 +62,11 @@ module LLM::MCP::Transport
     # @return [void]
     def write(message)
       raise LLM::MCP::Error, "MCP transport is not running" unless running?
-      req = Net::HTTP::Post.new(uri.path, headers.merge("content-type" => "application/json"))
+      req = Net::HTTP::Post.new(uri.request_uri, headers.merge("content-type" => "application/json"))
       req.body = LLM.json.dump(message)
-      if persistent_client.nil?
-        http = Net::HTTP.start(uri.host, uri.port, use_ssl:, open_timeout: timeout, read_timeout: timeout)
-        args = [req]
-      else
-        http = persistent_client
-        args = [uri, req]
-      end
-      http.request(*args) do |res|
-        unless Net::HTTPSuccess === res
-          raise LLM::MCP::Error, "MCP transport write failed with HTTP #{res.code}"
-        end
-        read(res)
-      end
+      res = transport.request(req, owner: self) { consume(_1) }
+      res = LLM::Transport::Response.from(res)
+      raise LLM::MCP::Error, "MCP transport write failed with HTTP #{res.code}" unless res.success?
     end
 
     ##
@@ -100,30 +91,27 @@ module LLM::MCP::Transport
       @running
     end
 
-    ##
-    # Configures the transport to use a persistent HTTP connection pool
-    # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-    # @example
-    #   mcp = LLM::MCP.http(url: "https://example.com/mcp", persistent: true)
-    #   # do something with 'mcp'
-    # @return [LLM::MCP::Transport::HTTP]
-    def persist!
-      LLM.lock(:mcp) do
-        LLM.require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
-        unless LLM::MCP.clients.key?(key)
-          http = Net::HTTP::Persistent.new(name: self.class.name)
-          http.read_timeout = timeout
-          http.open_timeout = timeout
-          LLM::MCP.clients[key] ||= http
-        end
-      end
-      self
+    private
+
+    attr_reader :uri, :headers, :transport
+
+    def consume(res)
+      res = LLM::Transport::Response.from(res)
+      read(res)
+      res
     end
-    alias_method :persistent, :persist!
 
-    private
+    def resolve_transport(transport, timeout:)
+      return default_transport(timeout:) if transport.nil?
+      if Class === transport && transport <= LLM::Transport
+        return transport.new(host: uri.host, port: uri.port, timeout:, ssl: uri.scheme == "https")
+      end
+      transport
+    end
 
-    attr_reader :uri, :use_ssl, :headers, :timeout
+    def default_transport(timeout:)
+      LLM::Transport::HTTP.new(host: uri.host, port: uri.port, timeout:, ssl: uri.scheme == "https")
+    end
 
     def read(res)
       if res["content-type"].to_s.include?("text/event-stream")
@@ -142,14 +130,6 @@ module LLM::MCP::Transport
       lock { @queue << message }
     end
 
-    def persistent_client
-      LLM::MCP.clients[key]
-    end
-
-    def key
-      "#{uri.scheme}:#{uri.host}:#{uri.port}:#{timeout}"
-    end
-
     def lock(&)
       @monitor.synchronize(&)
     end

data/lib/llm/mcp/transport/stdio.rb

@@ -78,14 +78,6 @@ module LLM::MCP::Transport
       command.wait
     end
 
-    ##
-    # This method is a no-op for stdio transports
-    # @return [LLM::MCP::Transport::Stdio]
-    def persist!
-      self
-    end
-    alias_method :persistent, :persist!
-
     private
 
     attr_reader :command, :stdin, :stdout, :stderr

data/lib/llm/mcp.rb

@@ -24,14 +24,6 @@ class LLM::MCP
 
   include RPC
 
-  @clients = {}
-
-  ##
-  # @api private
-  def self.clients
-    @clients
-  end
-
   ##
   # Builds an MCP client that uses the stdio transport.
   # @param [LLM::Provider, nil] llm
@@ -69,6 +61,9 @@ class LLM::MCP
   #  The URL for the MCP HTTP endpoint
   # @option http [Hash] :headers
   #  Extra headers for requests
+  # @option http [LLM::Transport, Class] :transport
+  #  Optional override with any {LLM::Transport} instance or subclass,
+  #  similar to {LLM::Provider}
   # @param [Integer] timeout
   #  The maximum amount of time to wait when reading from an MCP process
   # @return [LLM::MCP] A new MCP instance
@@ -82,8 +77,9 @@ class LLM::MCP
       @transport = Transport::Stdio.new(command:)
     elsif http
       persistent = http.delete(:persistent)
-      @transport = Transport::HTTP.new(**http, timeout:)
-      @transport.persistent if persistent
+      transport = http.delete(:transport)
+      transport ||= LLM::Transport::PersistentHTTP if persistent
+      @transport = Transport::HTTP.new(**http, timeout:, transport:)
     else
       raise ArgumentError, "stdio or http is required"
     end
@@ -121,19 +117,6 @@ class LLM::MCP
     stop
   end
 
-  ##
-  # Configures an HTTP MCP transport to use a persistent connection pool
-  # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-  # @example
-  #   mcp = LLM::MCP.http(url: "https://example.com/mcp", persistent: true)
-  #   # do something with 'mcp'
-  # @return [LLM::MCP]
-  def persist!
-    transport.persist!
-    self
-  end
-  alias_method :persistent, :persist!
-
   ##
   # Returns the tools provided by the MCP process.
   # @return [Array<Class<LLM::Tool>>]

data/lib/llm/provider.rb

@@ -6,10 +6,7 @@
 #
 # @abstract
 class LLM::Provider
-  require "net/http"
-  require_relative "provider/transport/http"
-  require_relative "provider/transport/http/execution"
-  include Transport::HTTP::Execution
+  include LLM::Transport::Execution
 
   ##
   # @param [String, nil] key
@@ -27,7 +24,9 @@ class LLM::Provider
   # @param [Boolean] persistent
   #  Whether to use a persistent connection.
   #  Requires the net-http-persistent gem.
-  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, base_path: "", persistent: false)
+  # @param [LLM::Transport, Class, nil] transport
+  #  Optional override with any {LLM::Transport} instance or subclass.
+  def initialize(key:, host:, port: 443, timeout: 60, ssl: true, base_path: "", persistent: false, transport: nil)
     @key = key
     @host = host
     @port = port
@@ -36,7 +35,7 @@ class LLM::Provider
     @base_path = normalize_base_path(base_path)
     @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
     @headers = {"User-Agent" => "llm.rb v#{LLM::VERSION}"}
-    @transport = Transport::HTTP.new(host:, port:, timeout:, ssl:, persistent:)
+    @transport = resolve_transport(transport, persistent:)
     @monitor = Monitor.new
   end
 
@@ -316,19 +315,6 @@ class LLM::Provider
     end
   end
 
-  ##
-  # This method configures a provider to use a persistent connection pool
-  # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-  # @example
-  #   llm = LLM.openai(key: ENV["KEY"]).persistent
-  #   # do something with 'llm'
-  # @return [LLM::Provider]
-  def persist!
-    transport.persist!
-    self
-  end
-  alias_method :persistent, :persist!
-
   ##
   # Interrupt the active request, if any.
   # @param [Fiber] owner
@@ -353,6 +339,13 @@ class LLM::Provider
     LLM::Stream === stream || stream.respond_to?(:<<)
   end
 
+  ##
+  # @return [Boolean]
+  #  Returns true when an API key is configured
+  def key?
+    @key != nil && @key.to_s.strip.size > 0
+  end
+
   private
 
   def path(suffix)
@@ -403,7 +396,7 @@ class LLM::Provider
   # @return [Class]
   #  Returns the class responsible for decoding streamed response bodies
   def stream_decoder
-    LLM::Provider::Transport::HTTP::StreamDecoder
+    LLM::Transport::StreamDecoder
   end
 
   ##
@@ -431,6 +424,23 @@ class LLM::Provider
     @monitor.synchronize(&)
   end
 
+  ##
+  # @api private
+  def default_transport(persistent:)
+    transport_class = persistent ? LLM::Transport::PersistentHTTP : LLM::Transport::HTTP
+    transport_class.new(host:, port:, timeout:, ssl:)
+  end
+
+  ##
+  # @api private
+  def resolve_transport(transport, persistent:)
+    return default_transport(persistent:) if transport.nil?
+    if Class === transport && transport <= LLM::Transport
+      return transport.new(host:, port:, timeout:, ssl:)
+    end
+    transport
+  end
+
   ##
   # @api private
   def thread

data/lib/llm/providers/anthropic/error_handler.rb

@@ -5,7 +5,7 @@ class LLM::Anthropic
   # @private
   class ErrorHandler
     ##
-    # @return [Net::HTTPResponse]
+    # @return [LLM::Transport::Response]
     #  Non-2XX response from the server
     attr_reader :res
 
@@ -19,13 +19,13 @@ class LLM::Anthropic
     #  The tracer
     # @param [Object, nil] span
     #  The span
-    # @param [Net::HTTPResponse] res
+    # @param [LLM::Transport::Response, Net::HTTPResponse] res
     #  The response from the server
     # @return [LLM::Anthropic::ErrorHandler]
     def initialize(tracer, span, res)
       @tracer = tracer
       @span = span
-      @res = res
+      @res = LLM::Transport::Response.from(res)
     end
 
     ##
@@ -43,12 +43,11 @@ class LLM::Anthropic
     ##
     # @return [LLM::Error]
     def error
-      case res
-      when Net::HTTPServerError
+      if res.server_error?
         LLM::ServerError.new("Server error").tap { _1.response = res }
-      when Net::HTTPUnauthorized
+      elsif res.unauthorized?
         LLM::UnauthorizedError.new("Authentication error").tap { _1.response = res }
-      when Net::HTTPTooManyRequests
+      elsif res.rate_limited?
         LLM::RateLimitError.new("Too many requests").tap { _1.response = res }
       else
         LLM::Error.new("Unexpected response").tap { _1.response = res }

data/lib/llm/providers/anthropic/files.rb

@@ -58,7 +58,7 @@ class LLM::Anthropic
       multi = LLM::Multipart.new(params.merge!(file: LLM.File(file)))
       req = Net::HTTP::Post.new("/v1/files", headers)
       req["content-type"] = multi.content_type
-      set_body_stream(req, multi.body)
+      transport.set_body_stream(req, multi.body)
       res, span, tracer = execute(request: req, operation: "request")
       res = ResponseAdapter.adapt(res, type: :file)
       tracer.on_request_finish(operation: "request", res:, span:)
@@ -159,7 +159,7 @@ class LLM::Anthropic
       @provider.instance_variable_get(:@key)
     end
 
-    [:headers, :execute, :set_body_stream].each do |m|
+    [:headers, :execute, :transport].each do |m|
       define_method(m) { |*args, **kwargs, &b| @provider.send(m, *args, **kwargs, &b) }
     end
   end