llm.rb 8.1.0 → 9.0.0

data/lib/llm/active_record/acts_as_llm.rb

@@ -75,14 +75,6 @@ module LLM::ActiveRecord
         ctx.wait(...)
       end
 
-      ##
-      # Calls into the stored context.
-      # @see LLM::Context#call
-      # @return [Object]
-      def call(...)
-        ctx.call(...)
-      end
-
       ##
       # @see LLM::Context#mode
       # @return [Symbol]
@@ -112,6 +104,13 @@ module LLM::ActiveRecord
         ctx.functions
       end
 
+      ##
+      # @see LLM::Context#functions?
+      # @return [Boolean]
+      def functions?
+        ctx.functions?
+      end
+
       ##
       # @see LLM::Context#returns
       # @return [Array<LLM::Function::Return>]

data/lib/llm/agent.rb

@@ -23,8 +23,7 @@ module LLM
   #   advisory tool errors back through the model and keeps the loop in-band.
   #   Set `tool_attempts: nil` to disable that advisory behavior.
   # * Tool loop execution can be configured with `concurrency :call`,
-  #   `:thread`, `:task`, `:fiber`, `:ractor`, or a list of queued task
-  #   types such as `[:thread, :ractor]`.
+  #   `:thread`, `:task`, `:fiber`, or `:ractor`.
   #
   # @example
   #   class SystemAdmin < LLM::Agent
@@ -110,9 +109,8 @@ module LLM
     #  - `:fork`: forked child processes
     #  - `:ractor`: concurrent Ruby ractors for class-based tools; MCP tools are not supported,
     #    and this mode is especially useful for CPU-bound tool work
-    #  - `[:thread, :ractor]`: the possible concurrency strategies to wait on, in the
-    #    given order. This is useful for mixed tool sets or when work may have been
-    #    spawned with more than one concurrency strategy.
+    #  Usually pass a single strategy. Arrays are only for advanced mixed-work
+    #  cases and are not needed for normal queued stream tool loops.
     # @return [Symbol, Array<Symbol>, nil]
     def self.concurrency(concurrency = nil)
       return @concurrency if concurrency.nil?
@@ -139,6 +137,26 @@ module LLM
       @tracer = block || tracer
     end
 
+    ##
+    # Set or get the default stream.
+    #
+    # When a block is provided, it is stored and evaluated lazily against the
+    # agent instance during initialization so it can build a fresh stream for
+    # each agent.
+    #
+    # @example
+    #   class Agent < LLM::Agent
+    #     stream { MyStream.new }
+    #   end
+    #
+    # @param [Object, Proc, nil] stream
+    # @yieldreturn [Object, nil]
+    # @return [Object, Proc, nil]
+    def self.stream(stream = nil, &block)
+      return @stream if stream.nil? && !block
+      @stream = block || stream
+    end
+
     ##
     # @param [LLM::Provider] provider
     #  A provider
@@ -157,7 +175,9 @@ module LLM
       @concurrency = params.delete(:concurrency) || self.class.concurrency
       @llm = llm
       tracer = params.key?(:tracer) ? params.delete(:tracer) : self.class.tracer
+      stream = params.key?(:stream) ? params.delete(:stream) : self.class.stream
       @tracer = resolve_option(tracer) unless tracer.nil?
+      params[:stream] = resolve_option(stream) unless stream.nil?
       @ctx = LLM::Context.new(llm, defaults.merge({guard: true}).merge(params))
     end
 
@@ -222,13 +242,6 @@ module LLM
       @ctx.returns
     end
 
-    ##
-    # @see LLM::Context#call
-    # @return [Object]
-    def call(...)
-      @tracer ? @llm.with_tracer(@tracer) { @ctx.call(...) } : @ctx.call(...)
-    end
-
     ##
     # @see LLM::Context#wait
     # @return [Array<LLM::Function::Return>]
@@ -293,6 +306,13 @@ module LLM
       @tracer || @ctx.tracer
     end
 
+    ##
+    # @return [LLM::Stream, #<<, nil]
+    #  Returns a stream object, or nil
+    def stream
+      @ctx.stream
+    end
+
     ##
     # Returns the model an Agent is actively using
     # @return [String]
@@ -397,7 +417,7 @@ module LLM
     # @return [Array<LLM::Function::Return>]
     def call_functions
       case concurrency || :call
-      when :call then call(:functions)
+      when :call then wait(:call)
       when :thread, :task, :fiber, :fork, :ractor, Array then wait(concurrency)
       else raise ArgumentError, "Unknown concurrency: #{concurrency.inspect}. " \
                                 "Expected :call, :thread, :task, :fiber, :fork, :ractor, " \
@@ -413,13 +433,13 @@ module LLM
         stream.extra[:concurrency] = concurrency if LLM::Stream === stream
         res = @ctx.public_send(method, apply_instructions(prompt), params)
         loop do
-          break if @ctx.functions.empty?
+          break unless @ctx.functions?
           if max
             max.times do
-              break if @ctx.functions.empty?
+              break unless @ctx.functions?
               res = @ctx.public_send(method, call_functions, params)
             end
-            break if @ctx.functions.empty?
+            break unless @ctx.functions?
             res = @ctx.public_send(method, @ctx.functions.map { rate_limit(_1) }, params)
           else
             res = @ctx.public_send(method, call_functions, params)

data/lib/llm/context.rb

@@ -44,6 +44,11 @@ module LLM
       input_tokens: 0,
       output_tokens: 0,
       reasoning_tokens: 0,
+      input_audio_tokens: 0,
+      output_audio_tokens: 0,
+      input_image_tokens: 0,
+      cache_read_tokens: 0,
+      cache_write_tokens: 0,
       total_tokens: 0
     )
     private_constant :ZERO_USAGE
@@ -257,18 +262,13 @@ module LLM
     end
 
     ##
-    # Calls a named collection of work through the context.
-    #
-    # This currently supports `:functions`, forwarding to `functions.call`.
-    #
-    # @param [Symbol] target
-    #  The work collection to call
-    # @return [Array<LLM::Function::Return>]
-    def call(target)
-      case target
-      when :functions then guarded_returns || functions.call
-      else raise ArgumentError, "Unknown target: #{target.inspect}. Expected :functions"
-      end
+    # Returns whether there is pending tool work in this context.
+    # This prefers queued streamed tool work when present, and otherwise
+    # falls back to unresolved functions derived from the message history.
+    # @return [Boolean]
+    def functions?
+      pending = queue
+      (pending && !pending.empty?) || functions.any?
     end
 
     ##
@@ -307,14 +307,15 @@ module LLM
     # the context's pending functions directly.
     #
     # @param [Symbol, Array<Symbol>] strategy
-    #  The concurrency strategy to use, or the possible concurrency strategies to
-    #  wait on. For example, `[:thread, :ractor]` waits for any queued thread or
-    #  ractor work, in that order.
+    #  If the stream queue already has tool work, `wait` will drain it
+    #  without using this argument.
+    #  Otherwise, this controls how pending functions are resolved directly.
+    #  Use `:call` for sequential execution without spawning.
     # @return [Array<LLM::Function::Return>]
     def wait(strategy)
       if LLM::Stream === stream && !stream.queue.empty?
         @queue = stream.queue
-        @queue.wait(strategy)
+        @queue.wait
       else
         return guarded_returns if guarded_returns
         @queue = functions.spawn(strategy)
@@ -350,6 +351,11 @@ module LLM
           input_tokens: usage.input_tokens || 0,
           output_tokens: usage.output_tokens || 0,
           reasoning_tokens: usage.reasoning_tokens || 0,
+          input_audio_tokens: usage.input_audio_tokens || 0,
+          output_audio_tokens: usage.output_audio_tokens || 0,
+          input_image_tokens: usage.input_image_tokens || 0,
+          cache_read_tokens: usage.cache_read_tokens || 0,
+          cache_write_tokens: usage.cache_write_tokens || 0,
           total_tokens: usage.total_tokens || 0
         )
       else
@@ -414,6 +420,13 @@ module LLM
       @llm.tracer
     end
 
+    ##
+    # @return [LLM::Stream, #<<, nil]
+    #  Returns a stream object, or nil
+    def stream
+      @stream || @params[:stream]
+    end
+
     ##
     # Returns the model a Context is actively using
     # @return [String]
@@ -458,12 +471,7 @@ module LLM
     #  Returns an _approximate_ cost for a given context
     #  based on both the provider, and model
     def cost
-      cost = LLM.registry_for(llm).cost(model:)
-      input_cost = (cost.input.to_f / 1_000_000.0) * usage.input_tokens
-      output_cost = (cost.output.to_f / 1_000_000.0) * usage.output_tokens
-      LLM::Cost.new(input_cost, output_cost)
-    rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
-      LLM::Cost.new(0, 0)
+      LLM::Cost.from(self)
     end
 
     ##
@@ -499,10 +507,6 @@ module LLM
       stream.queue if LLM::Stream === stream
     end
 
-    def stream
-      @stream || @params[:stream]
-    end
-
     def load_skills(skills)
       [*skills].map { LLM::Skill.load(_1).to_tool(self) }
     end

data/lib/llm/contract/completion.rb

@@ -36,6 +36,46 @@ module LLM::Contract
       raise NotImplementedError, "#{self.class} does not implement '#{__method__}'"
     end
 
+    ##
+    # @return [Integer]
+    #  Returns the number of input audio tokens, or 0 when the
+    #  provider does not report input audio usage
+    def input_audio_tokens
+      0
+    end
+
+    ##
+    # @return [Integer]
+    #  Returns the number of output audio tokens, or 0 when the
+    #  provider does not report output audio usage
+    def output_audio_tokens
+      0
+    end
+
+    ##
+    # @return [Integer]
+    #  Returns the number of input image tokens, or 0 when the
+    #  provider does not report input image usage
+    def input_image_tokens
+      0
+    end
+
+    ##
+    # @return [Integer]
+    #  Returns the number of cached input tokens, or 0 when the
+    #  provider does not report cache usage
+    def cache_read_tokens
+      0
+    end
+
+    ##
+    # @return [Integer]
+    #  Returns the number of cache creation input tokens, or 0 when the
+    #  provider does not report cache creation usage
+    def cache_write_tokens
+      0
+    end
+
     ##
     # @return [Integer]
     #  Returns the total number of tokens
@@ -72,6 +112,11 @@ module LLM::Contract
         input_tokens:,
         output_tokens:,
         reasoning_tokens:,
+        input_audio_tokens:,
+        output_audio_tokens:,
+        input_image_tokens:,
+        cache_read_tokens:,
+        cache_write_tokens:,
         total_tokens:
       )
     end

data/lib/llm/cost.rb

@@ -2,19 +2,96 @@
 
 ##
 # The {LLM::Cost LLM::Cost} class represents an approximate
-# cost breakdown for a provider request. It stores the input
-# and output costs separately and can return the total.
+# cost breakdown for a provider request. It stores input,
+# output, input audio, output audio, input image, cache read, cache write,
+# and reasoning costs separately and can return the total.
 #
 # @attr [Float] input_costs
 #   Returns the input cost
 # @attr [Float] output_costs
 #   Returns the output cost
-class LLM::Cost < Struct.new(:input_costs, :output_costs)
+# @attr [Float, nil] input_audio_costs
+#   Returns the input audio cost, or nil when no input audio tokens
+#   were used
+# @attr [Float, nil] output_audio_costs
+#   Returns the output audio cost, or nil when no output audio tokens
+#   were used
+# @attr [Float, nil] input_image_costs
+#   Returns the input image cost, or nil when no input image tokens
+#   were used
+# @attr [Float, nil] cache_read_costs
+#   Returns the cache read cost, or nil when no cache tokens
+#   were used
+# @attr [Float, nil] cache_write_costs
+#   Returns the cache write cost, or nil when no cache creation
+#   tokens were used
+# @attr [Float, nil] reasoning_costs
+#   Returns the reasoning cost, or nil when no reasoning tokens
+#   were used
+class LLM::Cost < Struct.new(
+  :input_costs, :output_costs,
+  :input_audio_costs, :output_audio_costs,
+  :cache_read_costs, :cache_write_costs,
+  :input_image_costs, :reasoning_costs,
+  keyword_init: true
+)
+  ##
+  # Build a cost breakdown from token usage and model pricing.
+  # @param [LLM::Context]
+  #  Context used to resolve provider, model, and token usage
+  # @return [LLM::Cost]
+  def self.from(ctx)
+    pricing = LLM.registry_for(ctx.llm).cost(model: ctx.model)
+    new(
+      input_costs: price(pricing.input, ctx.usage.input_tokens),
+      output_costs: price(pricing.output, ctx.usage.output_tokens),
+      input_audio_costs: price(pricing.input_audio, ctx.usage.input_audio_tokens),
+      output_audio_costs: price(pricing.output_audio, ctx.usage.output_audio_tokens),
+      input_image_costs: price(pricing.input, ctx.usage.input_image_tokens),
+      cache_read_costs: price(pricing.cache_read, ctx.usage.cache_read_tokens),
+      cache_write_costs: price(pricing.cache_write, ctx.usage.cache_write_tokens),
+      reasoning_costs: price(pricing.output, ctx.usage.reasoning_tokens)
+    )
+  rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
+    new
+  end
+
+  ##
+  # @api private
+  def self.price(rate, tokens)
+    return if tokens.nil? || tokens.to_i.zero?
+    return if rate.nil? || rate.to_f.zero?
+    ((rate.to_f / 1_000_000.0) * tokens.to_i).round(12)
+  end
+  private_class_method :price
+
   ##
   # @return [Float]
   #  Returns the total cost
   def total
-    input_costs + output_costs
+    [
+      input_costs, output_costs,
+      input_audio_costs, output_audio_costs,
+      cache_read_costs, cache_write_costs,
+      input_image_costs, reasoning_costs
+    ].compact.sum.round(12)
+  end
+
+  ##
+  # @return [Hash]
+  #  Returns a hash with the non-nil cost components and the total
+  def to_h
+    {
+      input: input_costs,
+      output: output_costs,
+      input_audio: input_audio_costs,
+      output_audio: output_audio_costs,
+      input_image: input_image_costs,
+      cache_read: cache_read_costs,
+      cache_write: cache_write_costs,
+      reasoning: reasoning_costs,
+      total: total
+    }.compact
   end
 
   ##

data/lib/llm/error.rb

@@ -5,7 +5,7 @@ module LLM
   # The superclass of all LLM errors
   class Error < RuntimeError
     ##
-    # @return [Net::HTTPResponse, nil]
+    # @return [LLM::Transport::Response, nil]
     #  Returns the response associated with an error, or nil
     attr_accessor :response
 

data/lib/llm/function/array.rb

@@ -18,21 +18,23 @@ class LLM::Function
 
     ##
     # Calls all functions in a collection concurrently.
-    # This method returns an {LLM::Function::ThreadGroup},
-    # {LLM::Function::TaskGroup}, or {LLM::Function::FiberGroup}
-    # that can be waited on to access the return values.
+    # This method returns an execution group that can be
+    # waited on to access the return values.
     #
     # @param [Symbol] strategy
     #   Controls concurrency strategy:
+    #   - `:call`: Call functions sequentially without spawning
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)
     #   - `:fork`: Use forked child processes
     #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #
-    # @return [LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
+    # @return [LLM::Function::CallGroup, LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
     def spawn(strategy)
       case strategy
+      when :call
+        CallGroup.new(self)
       when :task
         TaskGroup.new(map { |fn| fn.spawn(:task) })
       when :thread
@@ -44,7 +46,7 @@ class LLM::Function
       when :ractor
         Ractor::Group.new(map { |fn| fn.spawn(:ractor) })
       else
-        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, :fork, or :ractor"
+        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :call, :thread, :task, :fiber, :fork, or :ractor"
       end
     end
 
@@ -54,6 +56,7 @@ class LLM::Function
     #
     # @param [Symbol] strategy
     #   Controls concurrency strategy:
+    #   - `:call`: Call each function sequentially through a call group
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)

data/lib/llm/function/call_group.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::CallGroup} class wraps an array of
+  # {LLM::Function} objects for sequential execution.
+  #
+  # It provides the same basic interface as the concurrent group
+  # wrappers so callers can flow through `spawn(strategy).wait`
+  # uniformly, even when the selected strategy is direct calls.
+  class CallGroup
+    ##
+    # @param [Array<LLM::Function>] functions
+    # @return [LLM::Function::CallGroup]
+    def initialize(functions)
+      @functions = functions
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      false
+    end
+
+    ##
+    # @return [nil]
+    def interrupt!
+      nil
+    end
+    alias_method :cancel!, :interrupt!
+
+    ##
+    # @return [Array<LLM::Function::Return>]
+    def wait
+      @functions.map(&:call)
+    end
+    alias_method :value, :wait
+  end
+end

data/lib/llm/function/fork/task.rb

@@ -63,6 +63,12 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      LLM::Function::Fork::Group
+    end
+
     private
 
     def reap

data/lib/llm/function/ractor/task.rb

@@ -57,6 +57,12 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      LLM::Function::Ractor::Group
+    end
+
     private
 
     def build_task

data/lib/llm/function/task.rb

@@ -53,6 +53,16 @@ class LLM::Function
     end
     alias_method :value, :wait
 
+    ##
+    # @return [Class]
+    def group_class
+      case task
+      when Thread then LLM::Function::ThreadGroup
+      when Fiber then LLM::Function::FiberGroup
+      else LLM::Function::TaskGroup
+      end
+    end
+
     private
 
     def scheduler

data/lib/llm/function.rb

@@ -32,6 +32,7 @@ class LLM::Function
   require_relative "function/registry"
   require_relative "function/tracing"
   require_relative "function/array"
+  require_relative "function/call_group"
   require_relative "function/task"
   require_relative "function/thread_group"
   require_relative "function/fiber_group"

data/lib/llm/mcp/transport/http.rb

@@ -16,12 +16,13 @@ module LLM::MCP::Transport
     #  Extra headers to send with requests
     # @param [Integer, nil] timeout
     #  The timeout in seconds. Defaults to nil
+    # @param [LLM::Transport, Class, nil] transport
+    #  Optional override with any {LLM::Transport} instance or subclass
     # @return [LLM::MCP::Transport::HTTP]
-    def initialize(url:, headers: {}, timeout: nil)
+    def initialize(url:, headers: {}, timeout: nil, transport: nil)
       @uri = URI.parse(url)
-      @use_ssl = @uri.scheme == "https"
       @headers = headers
-      @timeout = timeout
+      @transport = resolve_transport(transport, timeout:)
       @queue = []
       @monitor = Monitor.new
       @running = false
@@ -61,21 +62,11 @@ module LLM::MCP::Transport
     # @return [void]
     def write(message)
       raise LLM::MCP::Error, "MCP transport is not running" unless running?
-      req = Net::HTTP::Post.new(uri.path, headers.merge("content-type" => "application/json"))
+      req = Net::HTTP::Post.new(uri.request_uri, headers.merge("content-type" => "application/json"))
       req.body = LLM.json.dump(message)
-      if persistent_client.nil?
-        http = Net::HTTP.start(uri.host, uri.port, use_ssl:, open_timeout: timeout, read_timeout: timeout)
-        args = [req]
-      else
-        http = persistent_client
-        args = [uri, req]
-      end
-      http.request(*args) do |res|
-        unless Net::HTTPSuccess === res
-          raise LLM::MCP::Error, "MCP transport write failed with HTTP #{res.code}"
-        end
-        read(res)
-      end
+      res = transport.request(req, owner: self) { consume(_1) }
+      res = LLM::Transport::Response.from(res)
+      raise LLM::MCP::Error, "MCP transport write failed with HTTP #{res.code}" unless res.success?
     end
 
     ##
@@ -100,30 +91,27 @@ module LLM::MCP::Transport
       @running
     end
 
-    ##
-    # Configures the transport to use a persistent HTTP connection pool
-    # via the optional dependency [Net::HTTP::Persistent](https://github.com/drbrain/net-http-persistent)
-    # @example
-    #   mcp = LLM::MCP.http(url: "https://example.com/mcp", persistent: true)
-    #   # do something with 'mcp'
-    # @return [LLM::MCP::Transport::HTTP]
-    def persist!
-      LLM.lock(:mcp) do
-        LLM.require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
-        unless LLM::MCP.clients.key?(key)
-          http = Net::HTTP::Persistent.new(name: self.class.name)
-          http.read_timeout = timeout
-          http.open_timeout = timeout
-          LLM::MCP.clients[key] ||= http
-        end
-      end
-      self
+    private
+
+    attr_reader :uri, :headers, :transport
+
+    def consume(res)
+      res = LLM::Transport::Response.from(res)
+      read(res)
+      res
     end
-    alias_method :persistent, :persist!
 
-    private
+    def resolve_transport(transport, timeout:)
+      return default_transport(timeout:) if transport.nil?
+      if Class === transport && transport <= LLM::Transport
+        return transport.new(host: uri.host, port: uri.port, timeout:, ssl: uri.scheme == "https")
+      end
+      transport
+    end
 
-    attr_reader :uri, :use_ssl, :headers, :timeout
+    def default_transport(timeout:)
+      LLM::Transport::HTTP.new(host: uri.host, port: uri.port, timeout:, ssl: uri.scheme == "https")
+    end
 
     def read(res)
       if res["content-type"].to_s.include?("text/event-stream")
@@ -142,14 +130,6 @@ module LLM::MCP::Transport
       lock { @queue << message }
     end
 
-    def persistent_client
-      LLM::MCP.clients[key]
-    end
-
-    def key
-      "#{uri.scheme}:#{uri.host}:#{uri.port}:#{timeout}"
-    end
-
     def lock(&)
       @monitor.synchronize(&)
     end

data/lib/llm/mcp/transport/stdio.rb

@@ -78,14 +78,6 @@ module LLM::MCP::Transport
       command.wait
     end
 
-    ##
-    # This method is a no-op for stdio transports
-    # @return [LLM::MCP::Transport::Stdio]
-    def persist!
-      self
-    end
-    alias_method :persistent, :persist!
-
     private
 
     attr_reader :command, :stdin, :stdout, :stderr