llm.rb 4.16.1 → 4.18.0

data/lib/llm/agent.rb

@@ -6,10 +6,18 @@ module LLM
   # reusable, preconfigured assistants with defaults for model,
   # tools, schema, and instructions.
   #
+  # It wraps the same stateful runtime surface as
+  # {LLM::Context LLM::Context}: message history, usage, persistence,
+  # streaming parameters, and provider-backed requests still flow through
+  # an underlying context. The defining behavior of an agent is that it
+  # automatically resolves pending tool calls for you during `talk` and
+  # `respond`, instead of leaving tool loops to the caller.
+  #
   # **Notes:**
   # * Instructions are injected only on the first request.
-  # * An agent will automatically execute tool calls (unlike {LLM::Context LLM::Context}).
-  # * The idea originally came from RubyLLM and was adapted to llm.rb.
+  # * An agent automatically executes tool loops (unlike {LLM::Context LLM::Context}).
+  # * Tool loop execution can be configured with `concurrency :call`,
+  #   `:thread`, `:task`, `:fiber`, or `:ractor`.
   #
   # @example
   #   class SystemAdmin < LLM::Agent
@@ -72,6 +80,23 @@ module LLM
       @instructions = instructions
     end
 
+    ##
+    # Set or get the tool execution concurrency.
+    #
+    # @param [Symbol, nil] concurrency
+    #  Controls how pending tool loops are executed:
+    #  - `:call`: sequential calls
+    #  - `:thread`: concurrent threads
+    #  - `:task`: concurrent async tasks
+    #  - `:fiber`: concurrent raw fibers
+    #  - `:ractor`: concurrent Ruby ractors for class-based tools; MCP tools are not supported,
+    #    and this mode is especially useful for CPU-bound tool work
+    # @return [Symbol, nil]
+    def self.concurrency(concurrency = nil)
+      return @concurrency if concurrency.nil?
+      @concurrency = concurrency
+    end
+
     ##
     # @param [LLM::Provider] provider
     #  A provider
@@ -82,8 +107,10 @@ module LLM
     # @option params [String] :model Defaults to the provider's default model
     # @option params [Array<LLM::Function>, nil] :tools Defaults to nil
     # @option params [#to_json, nil] :schema Defaults to nil
+    # @option params [Symbol, nil] :concurrency Defaults to the agent class concurrency
     def initialize(llm, params = {})
       defaults = {model: self.class.model, tools: self.class.tools, schema: self.class.schema}.compact
+      @concurrency = params.delete(:concurrency) || self.class.concurrency
       @llm = llm
       @ctx = LLM::Context.new(llm, defaults.merge(params))
     end
@@ -94,7 +121,7 @@ module LLM
     #
     # @param prompt (see LLM::Provider#complete)
     # @param [Hash] params The params passed to the provider, including optional :stream, :tools, :schema etc.
-    # @option params [Integer] :max_tool_rounds The maxinum number of tool call iterations (default 10)
+    # @option params [Integer] :tool_attempts The maxinum number of tool call iterations (default 10)
     # @return [LLM::Response] Returns the LLM's response for this turn.
     # @example
     #   llm = LLM.openai(key: ENV["KEY"])
@@ -102,13 +129,13 @@ module LLM
     #   response = agent.talk("Hello, what is your name?")
     #   puts response.choices[0].content
     def talk(prompt, params = {})
-      i, max = 0, Integer(params.delete(:max_tool_rounds) || 10)
+      max = Integer(params.delete(:tool_attempts) || 10)
       res = @ctx.talk(apply_instructions(prompt), params)
-      until @ctx.functions.empty?
-        raise LLM::ToolLoopError, "pending tool calls remain" if i >= max
-        res = @ctx.talk @ctx.functions.map(&:call), params
-        i += 1
+      max.times do
+        break if @ctx.functions.empty?
+        res = @ctx.talk(call_functions, params)
       end
+      raise LLM::ToolLoopError, "pending tool calls remain" unless @ctx.functions.empty?
       res
     end
     alias_method :chat, :talk
@@ -120,7 +147,7 @@ module LLM
     # @note Not all LLM providers support this API
     # @param prompt (see LLM::Provider#complete)
     # @param [Hash] params The params passed to the provider, including optional :stream, :tools, :schema etc.
-    # @option params [Integer] :max_tool_rounds The maxinum number of tool call iterations (default 10)
+    # @option params [Integer] :tool_attempts The maxinum number of tool call iterations (default 10)
     # @return [LLM::Response] Returns the LLM's response for this turn.
     # @example
     #   llm = LLM.openai(key: ENV["KEY"])
@@ -128,13 +155,13 @@ module LLM
     #   res = agent.respond("What is the capital of France?")
     #   puts res.output_text
     def respond(prompt, params = {})
-      i, max = 0, Integer(params.delete(:max_tool_rounds) || 10)
+      max = Integer(params.delete(:tool_attempts) || 10)
       res = @ctx.respond(apply_instructions(prompt), params)
-      until @ctx.functions.empty?
-        raise LLM::ToolLoopError, "pending tool calls remain" if i >= max
-        res = @ctx.respond @ctx.functions.map(&:call), params
-        i += 1
+      max.times do
+        break if @ctx.functions.empty?
+        res = @ctx.respond(call_functions, params)
       end
+      raise LLM::ToolLoopError, "pending tool calls remain" unless @ctx.functions.empty?
       res
     end
 
@@ -150,12 +177,41 @@ module LLM
       @ctx.functions
     end
 
+    ##
+    # @see LLM::Context#returns
+    # @return [Array<LLM::Function::Return>]
+    def returns
+      @ctx.returns
+    end
+
+    ##
+    # @see LLM::Context#call
+    # @return [Object]
+    def call(...)
+      @ctx.call(...)
+    end
+
+    ##
+    # @see LLM::Context#wait
+    # @return [Array<LLM::Function::Return>]
+    def wait(...)
+      @ctx.wait(...)
+    end
+
     ##
     # @return [LLM::Object]
     def usage
       @ctx.usage
     end
 
+    ##
+    # Interrupt the active request, if any.
+    # @return [nil]
+    def interrupt!
+      @ctx.interrupt!
+    end
+    alias_method :cancel!, :interrupt!
+
     ##
     # @param (see LLM::Context#prompt)
     # @return (see LLM::Context#prompt)
@@ -206,6 +262,53 @@ module LLM
       @ctx.model
     end
 
+    ##
+    # @return [Symbol]
+    def mode
+      @ctx.mode
+    end
+
+    ##
+    # Returns the configured tool execution concurrency.
+    # @return [Symbol, nil]
+    def concurrency
+      @concurrency
+    end
+
+    ##
+    # @see LLM::Context#cost
+    # @return [LLM::Cost]
+    def cost
+      @ctx.cost
+    end
+
+    ##
+    # @see LLM::Context#context_window
+    # @return [Integer]
+    def context_window
+      @ctx.context_window
+    end
+
+    ##
+    # @see LLM::Context#to_h
+    # @return [Hash]
+    def to_h
+      @ctx.to_h
+    end
+
+    ##
+    # @return [String]
+    def to_json(...)
+      to_h.to_json(...)
+    end
+
+    ##
+    # @return [String]
+    def inspect
+      "#<#{self.class.name}:0x#{object_id.to_s(16)} " \
+      "@llm=#{@llm.class}, @mode=#{mode.inspect}, @messages=#{messages.inspect}>"
+    end
+
     ##
     # @param (see LLM::Context#serialize)
     # @return (see LLM::Context#serialize)
@@ -230,14 +333,24 @@ module LLM
       instr = self.class.instructions
       return new_prompt unless instr
       if LLM::Prompt === new_prompt
-        @ctx.messages.empty? ? new_prompt.system(instr) : nil
+        new_prompt.system(instr) if @ctx.messages.empty?
         new_prompt
       else
         prompt do
-          @ctx.messages.empty? ? _1.system(instr) : nil
+          _1.system(instr) if @ctx.messages.empty?
           _1.user(new_prompt)
         end
       end
     end
+
+    ##
+    # @return [Array<LLM::Function::Return>]
+    def call_functions
+      case concurrency || :call
+      when :call then call(:functions)
+      when :thread, :task, :fiber, :ractor then wait(concurrency)
+      else raise ArgumentError, "Unknown concurrency: #{concurrency.inspect}. Expected :call, :thread, :task, :fiber, or :ractor"
+      end
+    end
   end
 end

data/lib/llm/context.rb

@@ -86,6 +86,7 @@ module LLM
       return respond(prompt, params) if mode == :responses
       params = params.merge(messages: @messages.to_a)
       params = @params.merge(params)
+      bind!(params[:stream], params[:model])
       res = @llm.complete(prompt, params)
       role = params[:role] || @llm.user_role
       role = @llm.tool_role if params[:role].nil? && [*prompt].grep(LLM::Function::Return).any?
@@ -110,6 +111,7 @@ module LLM
     #   puts res.output_text
     def respond(prompt, params = {})
       params = @params.merge(params)
+      bind!(params[:stream], params[:model])
       res_id = params[:store] == false ? nil : @messages.find(&:assistant?)&.response&.response_id
       params = params.merge(previous_response_id: res_id, input: @messages.to_a).compact
       res = @llm.responses.create(prompt, params)
@@ -356,6 +358,14 @@ module LLM
     rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
       0
     end
+
+    private
+
+    def bind!(stream, model)
+      return unless LLM::Stream === stream
+      stream.extra[:tracer] = tracer
+      stream.extra[:model] = model
+    end
   end
 
   # Backward-compatible alias

data/lib/llm/function/array.rb

@@ -27,8 +27,9 @@ class LLM::Function
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use raw fibers
+    #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #
-    # @return [LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup]
+    # @return [LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
     def spawn(strategy)
       case strategy
       when :task
@@ -37,8 +38,10 @@ class LLM::Function
         ThreadGroup.new(map { |fn| fn.spawn(:thread) })
       when :fiber
         FiberGroup.new(map { |fn| fn.spawn(:fiber) })
+      when :ractor
+        Ractor::Group.new(map { |fn| fn.spawn(:ractor) })
       else
-        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, or :fiber"
+        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, or :ractor"
       end
     end
 
@@ -51,6 +54,7 @@ class LLM::Function
     #   - `:thread`: Use threads
     #   - `:task`: Use async tasks (requires async gem)
     #   - `:fiber`: Use raw fibers
+    #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #
     # @return [Array<LLM::Function::Return>]
     #  Returns values to be reported back to the LLM.

data/lib/llm/function/ractor/job.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::Ractor::Job} class manages execution and mailbox
+  # coordination for a single ractor-backed function call.
+  class Ractor::Job
+    ##
+    # @param [::Ractor] mailbox
+    # @param [Class] runner_class
+    # @param [String, nil] id
+    # @param [String] name
+    # @param [Hash, Array, nil] arguments
+    # @return [LLM::Function::Ractor::Job]
+    def initialize(mailbox, runner_class, id, name, arguments)
+      @mailbox = mailbox
+      @runner_class = runner_class
+      @id = id
+      @name = name
+      @arguments = arguments
+    end
+
+    ##
+    # @return [void]
+    def call
+      spawn
+      wait
+    end
+
+    private
+
+    def wait
+      done = false
+      result = nil
+      waiters = []
+      loop do
+        case ::Ractor.receive
+        in [:done, *result]
+          done = true
+          waiters.each { _1.send(result) }
+          waiters.clear
+        in [:alive?, reply]
+          reply.send(!done)
+        in [:wait, reply]
+          done ? reply.send(result) : waiters << reply
+        end
+      end
+    end
+
+    def spawn
+      ::Ractor.new(@mailbox, @runner_class, @id, @name, @arguments) do |mailbox, runner_class, id, name, arguments|
+        kwargs = Hash === arguments ? arguments.transform_keys(&:to_sym) : arguments
+        mailbox.send([:done, id, name, runner_class.new.call(**kwargs)])
+      rescue => ex
+        mailbox.send([:done, id, name, {error: true, type: ex.class.name, message: ex.message}])
+      end
+    end
+  end
+end

data/lib/llm/function/ractor/mailbox.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::Ractor::Mailbox} class manages the mailbox protocol
+  # for a single ractor-backed function call.
+  class Ractor::Mailbox
+    ##
+    # @return [::Ractor]
+    attr_reader :task
+
+    ##
+    # @param [::Ractor] task
+    # @return [LLM::Function::Ractor::Mailbox]
+    def initialize(task)
+      @task = task
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      request(:alive?)
+    end
+
+    ##
+    # @return [Array]
+    def wait
+      request(:wait)
+    end
+
+    private
+
+    def request(type)
+      reply = ::Ractor.new { ::Ractor.receive }
+      task.send([type, reply])
+      reply.respond_to?(:take) ? reply.take : ::Ractor.select(reply).last
+    end
+  end
+end

data/lib/llm/function/ractor/task.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::Ractor::Task} class wraps a ractor-backed function
+  # call and delegates mailbox coordination to
+  # {LLM::Function::Ractor::Mailbox}.
+  class Ractor::Task
+    ##
+    # @return [LLM::Function::Ractor::Mailbox]
+    attr_reader :mailbox
+
+    ##
+    # @param [Class] runner_class
+    # @param [String, nil] id
+    # @param [String] name
+    # @param [Hash, Array, nil] arguments
+    # @return [LLM::Function::Ractor::Task]
+    def initialize(runner_class, id, name, arguments)
+      @mailbox = Ractor::Mailbox.new(build_task(runner_class, id, name, arguments))
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      mailbox.alive?
+    end
+
+    ##
+    # @return [LLM::Function::Return]
+    def wait
+      id, name, value = mailbox.wait
+      Return.new(id, name, value)
+    end
+    alias_method :value, :wait
+
+    private
+
+    def build_task(runner_class, id, name, arguments)
+      ::Ractor.new(runner_class, id, name, arguments) do |runner_class, id, name, arguments|
+        LLM::Function::Ractor::Job.new(::Ractor.current, runner_class, id, name, arguments).call
+      end
+    end
+  end
+end

data/lib/llm/function/ractor.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  module Ractor
+    require_relative "ractor/mailbox"
+    require_relative "ractor/job"
+    require_relative "ractor/task"
+  end
+end

data/lib/llm/function/ractor_group.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class LLM::Function
+  ##
+  # The {LLM::Function::Ractor::Group} class wraps an array of
+  # {LLM::Function::Ractor::Task} objects that are running
+  # {LLM::Function} calls concurrently.
+  class Ractor::Group
+    ##
+    # @param [Array<LLM::Function::Task>] tasks
+    # @return [LLM::Function::Ractor::Group]
+    def initialize(tasks)
+      @tasks = tasks
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      @tasks.any?(&:alive?)
+    end
+
+    ##
+    # @return [Array<LLM::Function::Return>]
+    def wait
+      @tasks.map(&:wait)
+    end
+    alias_method :value, :wait
+  end
+end

data/lib/llm/function/task.rb

@@ -14,7 +14,7 @@ class LLM::Function
     attr_reader :function
 
     ##
-    # @param [Thread, Fiber, Async::Task] task
+    # @param [Thread, Fiber, Async::Task, Ractor, LLM::Function::Ractor::Task] task
     # @param [LLM::Function, nil] function
     # @return [LLM::Function::Task]
     def initialize(task, function = nil)
@@ -25,7 +25,8 @@ class LLM::Function
     ##
     # @return [Boolean]
     def alive?
-      task.alive?
+      return task.alive? if task.respond_to?(:alive?)
+      false
     end
 
     ##

data/lib/llm/function.rb

@@ -36,6 +36,8 @@ class LLM::Function
   require_relative "function/thread_group"
   require_relative "function/fiber_group"
   require_relative "function/task_group"
+  require_relative "function/ractor"
+  require_relative "function/ractor_group"
 
   extend LLM::Function::Registry
   prepend LLM::Function::Tracing
@@ -174,6 +176,7 @@ class LLM::Function
   #   - `:thread`: Use threads
   #   - `:task`: Use async tasks (requires async gem)
   #   - `:fiber`: Use raw fibers
+  #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
   #
   # @return [LLM::Function::Task]
   #   Returns a task whose `#value` is an {LLM::Function::Return}.
@@ -181,17 +184,20 @@ class LLM::Function
     task = case strategy
     when :task
       require "async" unless defined?(::Async)
-      Async { call_function }
+      Async { call }
     when :thread
-      Thread.new { call_function }
+      Thread.new { call }
     when :fiber
       Fiber.new do
-        call_function
+        call
       ensure
         Fiber.yield
       end.tap(&:resume)
+    when :ractor
+      raise ArgumentError, "Ractor concurrency only supports class-based tools" unless Class === @runner
+      Ractor::Task.new(@runner, id, name, arguments)
     else
-      raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, or :fiber"
+      raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, or :ractor"
     end
     Task.new(task, self)
   ensure

data/lib/llm/provider.rb

@@ -271,34 +271,48 @@ class LLM::Provider
 
   ##
   # @return [LLM::Tracer]
-  #  Returns a fiber-local tracer
+  #  Returns the current scoped tracer override or provider default tracer
   def tracer
-    weakmap[self] || LLM::Tracer::Null.new(self)
+    weakmap[self] || @tracer || LLM::Tracer::Null.new(self)
   end
 
   ##
-  # Set a fiber-local tracer
+  # Set the provider's default tracer
+  # This tracer is shared by the provider instance and becomes the fallback
+  # whenever no scoped override is active.
   # @example
   #   llm = LLM.openai(key: ENV["KEY"])
-  #   Thread.new do
-  #     llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log/1.txt")
-  #   end
-  #   Thread.new do
-  #     llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log/2.txt")
-  #   end
-  #   # ...
+  #   llm.tracer = LLM::Tracer::Logger.new(llm, path: "/path/to/log.txt")
   # @param [LLM::Tracer] tracer
   #  A tracer
   # @return [void]
   def tracer=(tracer)
-    if tracer.nil?
-      if weakmap.respond_to?(:delete)
-        weakmap.delete(self)
-      else
-        weakmap[self] = nil
-      end
+    @tracer = tracer
+  end
+
+  ##
+  # Override the tracer for the current fiber while the block runs.
+  # This is useful when you want per-request or per-turn tracing without
+  # replacing the provider's default tracer.
+  # @example
+  #   llm.with_tracer(LLM::Tracer::Logger.new(llm, io: $stdout)) do
+  #     llm.complete("hello", model: "gpt-5.4-mini")
+  #   end
+  # @param [LLM::Tracer] tracer
+  # @yield
+  # @return [Object]
+  def with_tracer(tracer)
+    had_override = weakmap.key?(self)
+    previous = weakmap[self]
+    weakmap[self] = tracer
+    yield
+  ensure
+    if had_override
+      weakmap[self] = previous
+    elsif weakmap.respond_to?(:delete)
+      weakmap.delete(self)
     else
-      weakmap[self] = tracer
+      weakmap[self] = nil
     end
   end
 

data/lib/llm/providers/anthropic/stream_parser.rb

@@ -109,6 +109,8 @@ class LLM::Anthropic
       fn = (registered || LLM::Function.new(tool["name"])).dup.tap do |fn|
         fn.id = tool["id"]
         fn.arguments = LLM::Anthropic.parse_tool_input(tool["input"])
+        fn.tracer = @stream.extra[:tracer]
+        fn.model = @stream.extra[:model]
       end
       [fn, (registered ? nil : @stream.tool_not_found(fn))]
     end

data/lib/llm/providers/google/stream_parser.rb

@@ -157,6 +157,8 @@ class LLM::Google
       fn = (registered || LLM::Function.new(call["name"])).dup.tap do |fn|
         fn.id = LLM::Google.tool_id(part:, cindex:, pindex:)
         fn.arguments = call["args"]
+        fn.tracer = @stream.extra[:tracer]
+        fn.model = @stream.extra[:model]
       end
       [fn, (registered ? nil : @stream.tool_not_found(fn))]
     end

data/lib/llm/providers/openai/responses/stream_parser.rb

@@ -273,6 +273,8 @@ class LLM::OpenAI
       fn = (registered || LLM::Function.new(tool["name"])).dup.tap do |fn|
         fn.id = tool["call_id"]
         fn.arguments = arguments
+        fn.tracer = @stream.extra[:tracer]
+        fn.model = @stream.extra[:model]
       end
       [fn, (registered ? nil : @stream.tool_not_found(fn))]
     end

data/lib/llm/providers/openai/stream_parser.rb

@@ -189,6 +189,8 @@ class LLM::OpenAI
       fn = (registered || LLM::Function.new(function["name"])).dup.tap do |fn|
         fn.id = tool["id"]
         fn.arguments = arguments
+        fn.tracer = @stream.extra[:tracer]
+        fn.model = @stream.extra[:model]
       end
       [fn, (registered ? nil : @stream.tool_not_found(fn))]
     end