llm.rb 12.6.0 → 13.0.0

data/lib/llm/compactor/truncate.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class LLM::Compactor
+  ##
+  # An {LLM::Compactor::Truncate LLM::Compactor::Truncate}
+  # drops the oldest messages when the conversation grows
+  # beyond a configured size, keeping only the N most recent
+  # messages.
+  #
+  # No LLM call is made but this strategy is purely lossy. It
+  # also fast - no network required and operates purely on
+  # memory.
+  class Truncate < self
+    ##
+    # @param [String, Integer] keep
+    #  The last (approx) n number of messages to keep.
+    #  This parameter can also be a percentage: eg "80%"
+    #  to keep 80% of the most recent messages.
+    # @return [Array<LLM::Message>, nil]
+    def call(keep: 64)
+      keep = parse(keep)
+      if keep <= 0 || keep > messages.reject(&:system?).size
+        nil
+      else
+        stream.on_compaction(self)
+        kept = take(messages, keep)
+        messages.replace([messages.select(&:system?).first, *kept].compact)
+        ctx.compacted = true
+        stream.on_compaction_finish(self)
+        kept
+      end
+    end
+
+    private
+
+    ##
+    # @param [String, Integer] input
+    #  The given input
+    # @return [Integer]
+    #  Returns the number of messages to keep
+    def parse(input)
+      if String === input
+        if input.end_with?("%")
+          count = ctx.messages.reject(&:system?).size
+          (count * (Float(input[0..-2]) / 100)).round
+        else
+          Integer(input)
+        end
+      else
+        Integer(input)
+      end
+    end
+
+    def take(messages, limit)
+      subset, in_tool_call = [], false
+      messages.reverse_each.with_index(1) do |m, index|
+        # We travel backwards - so we see a
+        # tool return before we see a tool
+        # call.
+        #
+        # When we see a tool return, our next
+        # task is to find where it was called
+        # from, and we will even override the
+        # limit to do this.
+        #
+        # Otherwise, the conversation will become
+        # corrupted and any attempt to use it will
+        # be an API-level error.
+        in_tool_call = m.tool_return?
+        if index >= limit
+          subset.unshift(m)
+          in_tool_call ? next : break
+        else
+          subset.unshift(m)
+        end
+      end
+      subset
+    end
+  end
+end

data/lib/llm/compactor.rb

@@ -1,135 +1,53 @@
 # frozen_string_literal: true
 
-##
-# {LLM::Compactor LLM::Compactor} summarizes older context messages into a
-# smaller replacement message when a context grows too large.
-#
-# This work is directly inspired by the compaction approach developed by
-# General Intelligence Systems.
-#
-# The compactor can also use a different model from the main context by
-# setting `model:` in the compactor config. Compaction thresholds are opt-in:
-# provide `message_threshold:` and/or `token_threshold:` to enable policy-
-# driven compaction. `token_threshold:` accepts either an integer token count
-# or a percentage string like `"90%"`, which resolves against the current
-# model context window.
-class LLM::Compactor
-  DEFAULTS = {
-    retention_window: 8,
-    model: nil
-  }.freeze
-
-  ##
-  # @return [Hash]
-  attr_reader :config
-
-  ##
-  # @param [LLM::Context] ctx
-  # @param [Hash] config
-  # @option config [Integer, String, nil] :token_threshold
-  #  Enables token-based compaction. Integer values are treated as a fixed
-  #  token count. Percentage strings like `"90%"` are resolved against
-  #  {LLM::Context#context_window}; if the context window is unknown, the
-  #  percentage threshold is treated as disabled.
-  # @option config [Integer, nil] :message_threshold
-  #  Enables message-count-based compaction.
-  # @option config [Integer] :retention_window
-  # @option config [String, nil] :model
-  #  The model to use for the summarization request. Defaults to the current
-  #  context model.
-  def initialize(ctx, config = {})
-    @ctx = ctx
-    @config = DEFAULTS.merge(config)
-  end
-
+module LLM
   ##
-  # Returns true when the context should be compacted.
+  # {LLM::Compactor LLM::Compactor} is the superclass for context compaction
+  # strategies in llm.rb.
   #
-  # When `token_threshold:` is a percentage string such as `"90%"`, the
-  # threshold is resolved against the current context window and compared to
-  # the current total token usage.
-  # @param [Object] prompt
-  #  The next prompt or turn input
-  # @return [Boolean]
-  def compactable?(prompt = nil)
-    return false if ctx.functions.any? || [*prompt].grep(LLM::Function::Return).any?
-    messages = ctx.messages.reject(&:system?)
-    return true if config[:message_threshold] && messages.size > config[:message_threshold]
-    return true if token_threshold and ctx.usage.total_tokens > token_threshold
-    false
-  end
-  alias_method :compact?, :compactable?
-
-  ##
-  # Summarize older messages and replace them with a compact summary.
-  # @param [Object] prompt
-  #  The next prompt or turn input
-  # @return [LLM::Message, nil]
-  def compact!(prompt = nil)
-    return nil if ctx.functions.any? || [*prompt].grep(LLM::Function::Return).any?
-    messages = ctx.messages.reject(&:system?)
-    retention_window = [config[:retention_window], messages.size].min
-    return nil unless messages.size > retention_window
-    stream = ctx.params[:stream]
-    stream.on_compaction(ctx, self)
-    recent = retained_messages
-    older = messages[0...(messages.size - recent.size)]
-    summary = LLM::Message.new(ctx.llm.user_role, "[Previous conversation summary]\n\n#{summarize(older)}", {compaction: true})
-    ctx.messages.replace([*ctx.messages.take_while(&:system?), summary, *recent])
-    ctx.compacted = true
-    stream.on_compaction_finish(ctx, self)
-    summary
-  end
-
-  private
-
-  attr_reader :ctx
-
-  def retained_messages
-    messages = ctx.messages.reject(&:system?)
-    retention_window = [config[:retention_window], messages.size].min
-    start = [messages.size - retention_window, 0].max
-    start -= 1 while start > 0 && messages[start].tool_return?
-    messages[start..] || []
-  end
-
-  def token_threshold
-    @token_threshold ||= begin
-      threshold = config[:token_threshold]
-      return threshold unless threshold.to_s.end_with?("%")
-      return if ctx.context_window <= 0
-      (ctx.context_window * threshold.delete_suffix("%").to_f / 100).floor
+  # A compactor is bound to a context and decides whether and how to compact
+  # the conversation history when {#call} is invoked. Each subclass
+  # implements a different strategy: {LLM::Compactor::Truncate} drops the
+  # oldest messages, and {LLM::Compactor::Null} is a no-op (the default).
+  #
+  # The compactor does not have a separate `compact?` predicate. It inspects
+  # the context internally and returns `nil` when nothing needs to happen.
+  # Callers invoke {#call} unconditionally.
+  class Compactor
+    require_relative "compactor/truncate"
+    require_relative "compactor/null"
+
+    ##
+    # @return [LLM::Context]
+    attr_reader :ctx
+
+    ##
+    # @param ctx [LLM::Context, LLM::Agent]
+    # @return [LLM::Compactor]
+    def initialize(ctx)
+      @ctx = LLM::Agent === ctx ? ctx.instance_variable_get(:@ctx) : ctx
     end
-  end
 
-  def summarize(messages)
-    model = config[:model] || ctx.params[:model] || ctx.llm.default_model
-    ctx.llm.complete(summary_prompt(messages), model:).content
-  end
-
-  def summary_prompt(messages)
-    <<~PROMPT
-      Summarize this conversation history for context continuity.
-      The summary will replace these messages in the context window.
+    ##
+    # @abstract
+    # @param opts [Hash] Per-call options
+    # @return [Object, nil]
+    def call(**opts)
+      raise NotImplementedError
+    end
 
-      Focus on:
-      - What the user asked for
-      - Important facts and decisions
-      - Tool calls and outcomes that still matter
-      - What should happen next
+    private
 
-      Conversation:
-      #{serialize(messages)}
-    PROMPT
-  end
+    ##
+    # @return [LLM::Stream]
+    def stream
+      @ctx.params[:stream]
+    end
 
-  def serialize(messages)
-    messages.map do |message|
-      content = case message.content
-      when Array then message.content.map(&:inspect).join(", ")
-      else message.content.to_s
-      end
-      "#{message.role}: #{content.empty? ? "(empty)" : content}"
-    end.join("\n---\n")
+    ##
+    # @return [LLM::Buffer]
+    def messages
+      @ctx.messages
+    end
   end
 end

data/lib/llm/context.rb

@@ -2,8 +2,10 @@
 
 module LLM
   ##
-  # {LLM::Context LLM::Context} is the stateful execution boundary in
-  # llm.rb.
+  # {LLM::Context LLM::Context} is the low-level stateful execution
+  # boundary in llm.rb. Most users should start with {LLM::Agent}, which
+  # wraps Context and manages tool loops automatically. Use Context
+  # directly when you need manual control over tool execution.
   #
   # It holds the evolving runtime state for an LLM workflow:
   # conversation history, tool calls and returns, schema and streaming
@@ -22,19 +24,16 @@ module LLM
   #   #!/usr/bin/env ruby
   #   require "llm"
   #
-  #   llm = LLM.openai(key: ENV["KEY"])
-  #   ctx = LLM::Context.new(llm)
-  #
-  #   prompt = LLM::Prompt.new(llm) do
-  #     system "Be concise and show your reasoning briefly."
-  #     user "If a train goes 60 mph for 1.5 hours, how far does it travel?"
-  #     user "Now double the speed for the same time."
-  #   end
-  #
-  #   ctx.talk(prompt)
+  #   llm = LLM.deepseek(key: ENV["KEY"])
+  #   ctx = LLM::Context.new(llm, stream: $stdout)
+  #   ctx.talk "If a train goes 60 mph for 1.5 hours, how far does it travel?"
   #   ctx.messages.each { |m| puts "[#{m.role}] #{m.content}" }
+  #
+  # @see LLM::Agent The recommended high-level interface
+  # @see LLM::Buffer Message history (ctx.messages)
+  # @see LLM::Message Individual messages in the conversation
+  # @see LLM::Response Response returned by each turn
   class Context
-    require_relative "compactor"
     require_relative "context/serializer"
     require_relative "context/deserializer"
     include Serializer
@@ -79,12 +78,16 @@ module LLM
     #   Defaults to `:responses` for OpenAI, otherwise it defaults
     #   to `:completions`.
     # @option params [String] :model Defaults to the provider's default model
+    # @option params [Class<LLM::Compactor>, nil] :compactor
+    #   A compactor class to use for context compaction. Defaults to
+    #   {LLM::Compactor::Null}.
+    # @option params [Hash] :compactor_options
+    #   Options passed to the compactor's `call` method. Defaults to `{}`.
     # @option params [Array<LLM::Function>, nil] :tools Defaults to nil
     # @option params [Array<String>, nil] :skills Defaults to nil
     def initialize(llm, params = {})
       @llm = llm
       @mode = params.delete(:mode) || (llm.name == :openai ? :responses : :completions)
-      @compactor = params.delete(:compactor)
       @guard = params.delete(:guard)
       @transformer = params.delete(:transformer)
       tools = [*params.delete(:tools), *load_skills(params.delete(:skills))]
@@ -94,6 +97,10 @@ module LLM
       @messages = LLM::Buffer.new(llm)
       extra = @params.slice(:model, :tools).merge!(ctx: self, tracer:)
       @params[:stream] = LLM::Stream.try(@params[:stream], extra:)
+      @compactor = {
+        klass: params.delete(:compactor) || LLM::Compactor::Null,
+        options: params.delete(:compactor_options) || {}
+      }
     end
 
     ##
@@ -105,20 +112,9 @@ module LLM
 
     ##
     # Returns a context compactor
-    # This feature is inspired by the compaction approach developed by
-    # General Intelligence Systems.
     # @return [LLM::Compactor]
     def compactor
-      @compactor = LLM::Compactor.new(self, @compactor || {}) unless LLM::Compactor === @compactor
-      @compactor
-    end
-
-    ##
-    # Sets a context compactor or compactor config
-    # @param [LLM::Compactor, Hash, nil] compactor
-    # @return [LLM::Compactor, Hash, nil]
-    def compactor=(compactor)
-      @compactor = compactor
+      @compactor[:klass]
     end
 
     ##
@@ -197,7 +193,7 @@ module LLM
     #   puts res.messages[0].content
     def talk(prompt, params = {})
       @owner = @llm.request_owner
-      compactor.compact!(prompt) if compactor.compact?(prompt)
+      @compactor[:klass].new(self).call(**@compactor[:options])
       repair!(@messages, prompt)
       prompt, params, res = mode == :responses ? respond(prompt, params) : complete(prompt, params)
       self.compacted = false
@@ -247,7 +243,7 @@ module LLM
     ##
     # Returns an array of functions that can be called
     # @return [Array<LLM::Function>]
-    def functions
+    def pending_functions
       return_ids = returns.map(&:id)
       @messages
         .select(&:assistant?)
@@ -259,16 +255,14 @@ module LLM
           end
         end.extend(LLM::Function::Array)
     end
-    alias_method :pending_functions, :functions
-
     ##
     # Returns whether there is pending tool work in this context.
     # This prefers queued streamed tool work when present, and otherwise
     # falls back to unresolved functions derived from the message history.
     # @return [Boolean]
-    def functions?
+    def pending_functions?
       pending = queue
-      (pending && !pending.empty?) || functions.any?
+      (pending && !pending.empty?) || pending_functions.any?
     end
 
     ##
@@ -283,7 +277,7 @@ module LLM
     def spawn(function, strategy)
       warning = guard&.call(self)
       return guarded_return_for(function, warning) if warning
-      function.spawn(strategy)
+      function.task(strategy)
     end
 
     ##
@@ -310,16 +304,16 @@ module LLM
     #  If the stream queue already has tool work, `wait` will drain it
     #  without using this argument.
     #  Otherwise, this controls how pending functions are resolved directly.
-    #  Use `:call` for sequential execution without spawning.
+    #  Use `:sequential` for sequential execution without spawning.
     # @param [Array<LLM::Function>] except
     #  A list of functions to exclude from the wait
     # @return [Array<LLM::Function::Return>]
     def wait(strategy, except: [])
       if stream.queue.empty?
-        tools  = except.empty? ? functions : functions - except
+        tools  = except.empty? ? pending_functions : pending_functions - except
         guards = guarded_returns(tools:)
         return guards if guards
-        @queue = tools.spawn(strategy)
+        @queue = tools.task(strategy)
         returns = @queue.wait
         emit_tool_returns(tools, returns)
         returns
@@ -339,7 +333,7 @@ module LLM
     def interrupt!
       llm.interrupt!(@owner)
       queue&.interrupt!
-      functions.each(&:interrupt!)
+      pending_functions.each(&:interrupt!)
       @queue = nil
       @owner = nil
       nil

data/lib/llm/contract.rb

@@ -2,32 +2,11 @@
 
 module LLM
   ##
-  # The `LLM::Contract` module provides the ability for modules
-  # who are extended by it to implement contracts which must be
-  # implemented by other modules who include a given contract.
+  # @api private
   #
-  # @example
-  #   module LLM::Contract
-  #      # ..
-  #   end
-  #
-  #   module LLM::Contract
-  #     module Completion
-  #       extend LLM::Contract
-  #       # inheriting modules must implement these methods
-  #       # otherwise an error is raised on include
-  #       def foo = nil
-  #       def bar = nil
-  #     end
-  #   end
-  #
-  #   module LLM::OpenAI::ResponseAdapter
-  #     module Completion
-  #       def foo = nil
-  #       def bar = nil
-  #       include LLM::Contract::Completion
-  #     end
-  #   end
+  # The `LLM::Contract` module enforces API contracts between
+  # provider response adapters and the runtime. Users never
+  # interact with this module directly.
   module Contract
     ContractError = Class.new(LLM::Error)
     require_relative "contract/completion"

data/lib/llm/function/array.rb

@@ -23,30 +23,31 @@ class LLM::Function
     #
     # @param [Symbol] strategy
     #   Controls concurrency strategy:
-    #   - `:call`: Call functions sequentially without spawning
+    #   - `:sequential`: Call functions sequentially without spawning
     #   - `:thread`: Use threads
-    #   - `:task`: Use async tasks (requires async gem)
+    #   - `:async`: Use async tasks (requires async gem)
     #   - `:fiber`: Use scheduler-backed fibers (requires Fiber.scheduler)
     #   - `:fork`: Use forked child processes
     #   - `:ractor`: Use Ruby ractors (class-based tools only; MCP tools are not supported)
     #
-    # @return [LLM::Function::CallGroup, LLM::Function::ThreadGroup, LLM::Function::TaskGroup, LLM::Function::FiberGroup, LLM::Function::Ractor::Group]
-    def spawn(strategy)
+    # @return [LLM::Function::Sequential::Group, LLM::Function::Thread::Group, LLM::Function::Async::Group, LLM::Function::Fiber::Group, LLM::Function::Fork::Group, LLM::Function::Ractor::Group]
+    def task(strategy)
       case strategy
-      when :call
-        CallGroup.new(self)
-      when :task
-        TaskGroup.new(map { |fn| fn.spawn(:task) })
+      when :sequential
+        Sequential::Group.new(self)
+      when :async
+        LLM.require "async" unless defined?(::Async)
+        Async::Group.new(map { |fn| fn.task(:async) })
       when :thread
-        ThreadGroup.new(map { |fn| fn.spawn(:thread) })
+        Thread::Group.new(map { |fn| fn.task(:thread) })
       when :fiber
-        FiberGroup.new(map { |fn| fn.spawn(:fiber) })
+        Fiber::Group.new(map { |fn| fn.task(:fiber) })
       when :fork
-        Fork::Group.new(map { |fn| fn.spawn(:fork) })
+        Fork::Group.new(map { |fn| fn.task(:fork) })
       when :ractor
-        Ractor::Group.new(map { |fn| fn.spawn(:ractor) })
+        Ractor::Group.new(map { |fn| fn.task(:ractor) })
       else
-        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :call, :thread, :task, :fiber, :fork, or :ractor"
+        raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :sequential, :thread, :async, :fiber, :fork, or :ractor"
       end
     end
 
@@ -66,7 +67,7 @@ class LLM::Function
     # @return [Array<LLM::Function::Return>]
     #  Returns values to be reported back to the LLM.
     def wait(strategy)
-      spawn(strategy).wait
+      task(strategy).wait
     end
 
     ##

data/lib/llm/function/async/group.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module LLM::Function::Async
+  ##
+  # Wraps an array of {Async::Task} objects running on a shared
+  # {LLM::Function::Async::Reactor}. The reactor is created on
+  # demand if not provided.
+  class Group < LLM::Function::Group
+    ##
+    # @param [Array<Async::Task>] tasks
+    # @param [Hash] options
+    # @option options [LLM::Function::Async::Reactor] :reactor
+    def initialize(tasks, options = {})
+      @tasks = tasks
+      @reactor = options[:reactor] || LLM::Function::Async::Reactor.new
+    end
+
+    ##
+    # @return [nil]
+    def spawn
+      @tasks.each do |task|
+        task.reactor = @reactor
+        task.spawn
+      end
+      nil
+    ensure
+      @spawned = true
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      @tasks.any?(&:alive?)
+    end
+
+    ##
+    # @return [nil]
+    def interrupt!
+      @tasks.each(&:interrupt!)
+      nil
+    end
+    alias_method :cancel!, :interrupt!
+
+    ##
+    # @return [Array<LLM::Function::Return>]
+    def wait
+      spawn unless @spawned
+      @tasks.map(&:wait)
+    ensure
+      @reactor.stop
+    end
+    alias_method :value, :wait
+  end
+end

data/lib/llm/function/async/reactor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module LLM::Function::Async
+  ##
+  # Manages an {::Async::Reactor} on a background thread. Work
+  # is submitted through a thread-safe queue and run inside the
+  # reactor. The reactor and its fibers stay on one thread.
+  class Reactor
+    ##
+    # @return [Thread]
+    attr_reader :thread
+
+    def initialize
+      @inbox = Queue.new
+      @thread = ::Thread.new { run }
+    end
+
+    ##
+    # Submit a block to run inside the reactor.
+    # @return [nil]
+    def submit(&block)
+      @inbox << block
+      nil
+    end
+
+    ##
+    # Stop the reactor and wait for the thread to finish.
+    def stop
+      @inbox << :stop
+      @thread.join(5)
+      @thread.kill if @thread.alive?
+    end
+
+    private
+
+    def run
+      reactor = ::Async::Reactor.new
+      reactor.async do
+        loop do
+          work = @inbox.pop
+          break if work == :stop
+          reactor.async { work.call }
+        end
+      end
+      reactor.run
+    end
+  end
+end