llm.rb 12.6.0 → 13.1.0

data/bin/llm.rb

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+
+require "llm"
+require "fileutils"
+
+##
+# utils
+
+def keys
+  @keys ||= Dir[File.join(__dir__, "..", "lib", "llm", "providers", "*")]
+    .select { File.file?(_1) }
+    .map { File.basename(_1, ".rb") }
+    .sort_by { _1 == "deepseek" ? 0 : 1 }
+    .map { "#{_1.upcase}_API_KEY" } - ["BEDROCK_API_KEY"]
+end
+
+def help
+  prog = File.basename($PROGRAM_NAME)
+  warn ""
+  warn "Usage: #{prog} [options]"
+  warn ""
+  warn "Options:"
+  warn "  -p PROVIDER    Choose a provider"
+  warn "  -t             Temporary session that doesn't persist to disk"
+  warn "  -h             Show this help"
+  warn ""
+  warn "Examples:"
+  warn "  #{prog}                     # auto-detect provider from $PROVIDER_API_KEY"
+  warn "  #{prog} -p openai           # use OpenAI"
+  warn "  #{prog} -h                  # this help"
+  warn ""
+end
+
+def loaderror(ex)
+  gem, = ex.message.split(" is an optional runtime dependency")
+  warn ""
+  warn "  ── llm.rb ──────────────────────────────────────────────"
+  warn ""
+  warn "  ✖  Missing dependency: #{gem}"
+  warn ""
+  warn "     The repl needs this gem, but it's not installed."
+  warn ""
+  warn "     Fix:  gem install #{gem}"
+  warn "     Or:   bundle add #{gem}"
+  warn ""
+  warn "     Tip:  If you don't need the repl, you can use the"
+  warn "           library directly with:  require \"llm\""
+  warn ""
+  warn "  ───────────────────────────────────────────────────────"
+  warn ""
+end
+
+##
+# main
+
+def main(argv)
+  ##
+  # Make sure the dependencies are satisified first
+  begin
+    require "llm/tools"
+    require "llm/repl"
+  rescue LLM::LoadError => ex
+    loaderror(ex)
+    exit 1
+  end
+
+  ##
+  # C-Style option parser
+  # No external dep
+  while option = argv.shift
+    case option
+    when '-h'
+      help
+      exit 0
+    when '-t'
+      temp = true
+    when '-p'
+      provider = argv.shift
+    else
+      warn "llm.rb: unknown option #{option}"
+    end
+  end
+
+  ##
+  # Setup the home directory
+  # But only if the `-t` switch has not been provided
+  if temp.nil?
+    home = File.join(Dir.home, ".llm.rb")
+    FileUtils.mkdir_p(File.join(home, Dir.getwd))
+    session = File.join(home, Dir.getwd, "session.json")
+  end
+
+  ##
+  # No provider has been given.
+  # Try to infer one.
+  if provider.nil?
+    key = keys.find { ENV[_1] }
+    if key.nil?
+      warn "llm.rb: provide a provider with the -p switch"
+      exit 1
+    else
+      provider, = key.split("_")
+    end
+  end
+
+  ##
+  # We're ready to start the REPL
+  # This should always succeed unless -p gave garbage
+  provider = provider.downcase
+  if LLM.respond_to?(provider)
+    key ||= "#{provider.upcase}_API_KEY"
+    if ENV[key].nil? || ENV[key].to_s.empty?
+      warn "llm.rb: set #{key} to use #{provider}"
+      exit 1
+    end
+    llm = LLM.method(provider).call(key: ENV[key])
+    agent = LLM::Agent.new(llm, path: temp ? nil : session, tools: LLM::Tool.subclasses)
+    agent.repl
+  else
+    warn "llm.rb: #{provider} was not recognized"
+    exit 1
+  end
+end
+main(ARGV)

data/data/deepinfra.json

@@ -503,6 +503,7 @@
         "context": 131072,
         "output": 131072
       },
+      "status": "deprecated",
       "cost": {
         "input": 0.4,
         "output": 0.4
@@ -537,6 +538,7 @@
         "context": 262144,
         "output": 65536
       },
+      "status": "deprecated",
       "cost": {
         "input": 0.2,
         "output": 0.8
@@ -832,6 +834,7 @@
         "context": 196608,
         "output": 131072
       },
+      "status": "deprecated",
       "cost": {
         "input": 0.15,
         "output": 1.15,

data/data/xai.json

@@ -312,7 +312,7 @@
       "cost": {
         "input": 2,
         "output": 6,
-        "cache_read": 0.5,
+        "cache_read": 0.3,
         "tiers": [
           {
             "input": 4,

data/lib/llm/a2a.rb

@@ -27,7 +27,7 @@
 #   a2a = LLM::A2A.rest(url: "https://agent.example.com")
 #   ctx = LLM::Context.new(llm, tools: a2a.skills)
 #   ctx.talk("Analyze this data using the remote agent.")
-#   ctx.talk(ctx.wait(:call)) while ctx.functions?
+#   ctx.talk(ctx.wait(:sequential)) while ctx.pending_functions?
 class LLM::A2A
   require_relative "a2a/card"
   require_relative "a2a/error"

data/lib/llm/active_record/acts_as_llm.rb

@@ -104,17 +104,17 @@ module LLM::ActiveRecord
       end
 
       ##
-      # @see LLM::Context#functions
+      # @see LLM::Context#pending_functions
       # @return [Array<LLM::Function>]
-      def functions
-        ctx.functions
+      def pending_functions
+        ctx.pending_functions
       end
 
       ##
-      # @see LLM::Context#functions?
+      # @see LLM::Context#pending_functions?
       # @return [Boolean]
-      def functions?
-        ctx.functions?
+      def pending_functions?
+        ctx.pending_functions?
       end
 
       ##

data/lib/llm/agent.rb

@@ -2,16 +2,17 @@
 
 module LLM
   ##
-  # {LLM::Agent LLM::Agent} provides a class-level DSL for defining
-  # reusable, preconfigured assistants with defaults for model,
-  # tools, schema, and instructions.
+  # {LLM::Agent LLM::Agent} is the recommended entry point for most
+  # use-cases. It provides a class-level DSL for defining reusable,
+  # preconfigured assistants with defaults for model, tools, schema,
+  # and instructions.
   #
   # It wraps the same stateful runtime surface as
   # {LLM::Context LLM::Context}: message history, usage, persistence,
   # streaming parameters, and provider-backed requests still flow through
   # an underlying context. The defining behavior of an agent is that it
-  # automatically resolves pending tool calls for you during `talk` and
-  # `respond`, instead of leaving tool loops to the caller.
+  # automatically resolves pending tool calls for you during `talk`,
+  # instead of leaving tool loops to the caller.
   #
   # **Notes:**
   # * Instructions are injected once unless a system message is already present.
@@ -22,10 +23,10 @@ module LLM
   # * The default tool attempt budget is `25`. After that, the agent sends
   #   advisory tool errors back through the model and keeps the loop in-band.
   #   Set `tool_attempts: nil` to disable that advisory behavior.
-  # * Tool loop execution can be configured with `concurrency :call`,
-  #   `:thread`, `:task`, `:fiber`, or `:ractor`.
+  # * Tool loop execution can be configured with `concurrency :sequential`,
+  #   `:thread`, `:async`, `:fiber`, `:fork`, or `:ractor`.
   #
-  # @example
+  # @example Subclass with defaults
   #   class SystemAdmin < LLM::Agent
   #     set model: "gpt-4.1-nano",
   #         instructions: "You are a Linux system admin",
@@ -36,7 +37,31 @@ module LLM
   #   llm = LLM.openai(key: ENV["KEY"])
   #   agent = SystemAdmin.new(llm)
   #   agent.talk("Run 'date'")
+  #
+  # @example Direct instance
+  #   llm = LLM.deepseek(key: ENV["KEY"])
+  #   agent = LLM::Agent.new(llm, stream: $stdout)
+  #   agent.talk "Hello world"
+  #
+  # @see LLM::Context The low-level runtime that Agent wraps
+  # @see LLM::Tool Tools that Agent can call on your behalf
+  # @see LLM::Stream Stream callbacks for model output
   class Agent
+    ##
+    # @api private
+    UNDEFINED = Object.new
+    private_constant :UNDEFINED
+
+    ##
+    # @api private
+    CASE_PATTERN = /(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/
+    private_constant :CASE_PATTERN
+
+    ##
+    # @api private
+    File = ::File
+    private_constant :File
+
     ##
     # Returns a provider
     # @return [LLM::Provider]
@@ -51,7 +76,8 @@ module LLM
     #
     # @example
     #   class AdminAgent < LLM::Agent
-    #     set instructions: "You are a system administrator",
+    #     set name: "admin",
+    #         instructions: "You are a system administrator",
     #         model: "gpt-4.1-nano",
     #         tools: [Shell, ReadFile]
     #   end
@@ -78,6 +104,46 @@ module LLM
       end
     end
 
+    ##
+    # Set or get an agent's name
+    # @note
+    #  This method serves as a self-documenting string
+    #  and it is used by {LLM::Repl LLM::Repl}. It is
+    #  optional but recommended.
+    # @param [String] name
+    #  The agent name
+    # @return [String]
+    #  Return's the agents name
+    def self.name(name = UNDEFINED, &block)
+      if name.equal?(UNDEFINED)
+        if @name.nil?
+          name  = to_s.split("::").last
+          @name = name.gsub(CASE_PATTERN, "-").downcase
+        else
+          @name
+        end
+      else
+        @name = block || name
+      end
+    end
+
+    ##
+    # Set or get an agent's description
+    # @note
+    #  This method serves as a self-documenting string.
+    #  It is optional but recommended.
+    # @param [String] desc
+    #  The agent's description
+    # @return [String, nil]
+    #  Returns the agent's description
+    def self.description(desc = UNDEFINED, &block)
+      if desc.equal?(UNDEFINED)
+        @desc
+      else
+        @desc = block || desc
+      end
+    end
+
     ##
     # Set or get the default model
     # @param [String, nil] model
@@ -146,9 +212,9 @@ module LLM
     #
     # @param [Symbol, Array<Symbol>, nil] concurrency
     #  Controls how pending tool loops are executed:
-    #  - `:call`: sequential calls
+    #  - `:sequential`: sequential calls
     #  - `:thread`: concurrent threads
-    #  - `:task`: concurrent async tasks
+    #  - `:async`: concurrent async tasks
     #  - `:fiber`: concurrent scheduler-backed fibers
     #  - `:fork`: forked child processes
     #  - `:ractor`: concurrent Ruby ractors for class-based tools; MCP tools are not supported,
@@ -231,6 +297,20 @@ module LLM
       end
     end
 
+    ##
+    # Set the file path where an agent's memory
+    # can be restored from, and written to.
+    # @param [String] path
+    #  The path to a file
+    # @return [String, nil]
+    def self.path(path = UNDEFINED, &block)
+      if path.equal?(UNDEFINED)
+        @path
+      else
+        @path = path || block
+      end
+    end
+
     ##
     # @param [LLM::Provider] llm
     #  A provider
@@ -247,8 +327,8 @@ module LLM
     # @option params [Symbol, Array<Symbol>, nil] :concurrency Defaults to the agent class concurrency
     def initialize(llm, params = {})
       @llm = llm
-      fields = %i[model skills schema tracer stream tools concurrency instructions confirm]
-      fields_ivar = %i[tracer concurrency instructions confirm]
+      fields = %i[name description path model skills schema tracer stream tools concurrency instructions confirm]
+      fields_ivar = %i[name description path tracer concurrency instructions confirm]
       fields.each do |field|
         resolvable = params.key?(field) ? params.delete(field) : self.class.public_send(field)
         resolve_symbol = !%i[concurrency].include?(field)
@@ -263,6 +343,29 @@ module LLM
         end
       end
       @ctx = LLM::Context.new(llm, {guard: true}.merge(params))
+      @path and File.readable?(@path) ? @ctx.restore(path:) : nil
+    end
+
+    ##
+    # Returns the agent's name
+    # @return [String]
+    def name
+      @name
+    end
+
+    ##
+    # Returns a file path where an agent's memory is
+    # restored from, and written to after each turn.
+    # @return [String, nil]
+    def path
+      @path
+    end
+
+    ##
+    # Returns the agent's description
+    # @return [String, nil]
+    def description
+      @description
     end
 
     ##
@@ -282,13 +385,17 @@ module LLM
     #   response = agent.talk("Hello, what is your name?")
     #   puts response.choices[0].content
     def talk(prompt, params = {})
-      run_loop(prompt, params, :talk)
+      res = run_loop(prompt, params, :talk)
+      path ? @ctx.save(path:) : nil
+      res
     end
 
     ##
     # @see LLM::Context#ask
     def ask(prompt, params = {})
-      run_loop(prompt, params, :ask)
+      res = run_loop(prompt, params, :ask)
+      path ? @ctx.save(path:) : nil
+      res
     end
 
     ##
@@ -299,10 +406,9 @@ module LLM
 
     ##
     # @return [Array<LLM::Function>]
-    def functions
-      @tracer ? @llm.with_tracer(@tracer) { @ctx.functions } : @ctx.functions
+    def pending_functions
+      @tracer ? @llm.with_tracer(@tracer) { @ctx.pending_functions } : @ctx.pending_functions
     end
-    alias_method :pending_functions, :functions
 
     ##
     # @see LLM::Context#returns
@@ -435,6 +541,9 @@ module LLM
     #  By default this method disables the tracer for
     #  the duration of the repl session, and restores
     #  it afterwards.
+    # @param [String] name
+    #  The agent's name.
+    #  Defaults to {LLM::Agent#name}.
     # @param [String] path
     #  The path to a file where runtime state is read
     #  from, and written to
@@ -446,7 +555,7 @@ module LLM
     #  When true, the tracer is kept alive during the
     #  repl session. Default is false.
     # @return [void]
-    def repl(path: nil, tools: [], skills: [], tracer: false, trace: nil)
+    def repl(name: self.name, path: nil, tools: [], skills: [], tracer: false, trace: nil)
       if trace != nil
         warn "llm.rb: trace option is deprecated, use tracer instead"
         tracer = trace
@@ -456,7 +565,7 @@ module LLM
         self.tracer = nil
       end
       require_relative "repl" unless defined?(::LLM::Repl)
-      LLM::Repl.new(agent: self, path:, tools:, skills:).start
+      LLM::Repl.new(agent: self, name:, path:, tools:, skills:).start
     ensure
       if !tracer
         self.tracer = previous
@@ -516,7 +625,7 @@ module LLM
     # @param [Symbol, Array<Symbol>] strategy
     #  The execution strategy that would be used for the tool call.
     # @return [LLM::Function::Return]
-    #  Return either `fn.spawn(strategy).wait` to approve execution or
+    #  Return either `fn.task(strategy).wait` to approve execution or
     #  `fn.cancel(...)` to cancel the call.
     def on_tool_confirmation(fn, strategy)
       fn.cancel
@@ -554,12 +663,12 @@ module LLM
     ##
     # @return [Array<LLM::Function::Return>]
     def call_functions
-      strategy = concurrency || :call
+      strategy = concurrency || :sequential
       return wait(strategy) unless @confirm&.any?
-      confirmables = @ctx.functions.select { @confirm.include?(_1.name.to_s) }
+      confirmables = @ctx.pending_functions.select { @confirm.include?(_1.name.to_s) }
       results = confirmables.map { method(:on_tool_confirmation).call(_1, strategy) }
       @ctx.method(:emit_tool_returns).call(confirmables, results)
-      if (@ctx.functions - confirmables).any?
+      if (@ctx.pending_functions - confirmables).any?
         [*results, *wait(strategy, except: confirmables)]
       else
         results
@@ -577,13 +686,13 @@ module LLM
         stream = params[:stream] || @ctx.params[:stream]
         params[:stream] = LLM::Stream.try(stream, extra: {concurrency:})
         res = talk.call(apply_instructions(prompt), params)
-        while @ctx.functions?
+        while @ctx.pending_functions?
           if max
             max.times do
-              break unless @ctx.functions?
+              break unless @ctx.pending_functions?
               res = talk.call(call_functions, params)
             end
-            res = talk.call(@ctx.functions.map(&:rate_limit), params) if @ctx.functions?
+            res = talk.call(@ctx.pending_functions.map(&:rate_limit), params) if @ctx.pending_functions?
           else
             res = talk.call(call_functions, params)
           end

data/lib/llm/buffer.rb

@@ -3,10 +3,24 @@
 module LLM
   ##
   # {LLM::Buffer LLM::Buffer} provides an Enumerable object that
-  # tracks messages in a conversation thread.
+  # tracks messages in a conversation thread. Access it through
+  # {LLM::Context#messages}.
+  #
+  # @example Working with message history
+  #   ctx.messages.last           # => most recent message
+  #   ctx.messages.first          # => oldest message
+  #   ctx.messages.select! { |m| m.assistant? }
+  #   ctx.messages.reverse        # => reversed copy
+  #   ctx.messages.reject! { |m| m.compaction? }
+  #
+  # @see LLM::Message Individual messages in the buffer
+  # @see LLM::Context Where the buffer lives (ctx.messages)
   class Buffer
     include Enumerable
 
+    UNDEFINED = Object.new
+    private_constant :UNDEFINED
+
     ##
     # @param [LLM::Provider] provider
     # @return [LLM::Buffer]
@@ -65,8 +79,69 @@ module LLM
     # @param [Integer, nil] n
     #  The number of messages to return
     # @return [LLM::Message, Array<LLM::Message>, nil]
-    def last(n = nil)
-      n.nil? ? @messages.last : @messages.last(n)
+    def last(n = UNDEFINED)
+      n.equal?(UNDEFINED) ? @messages.last : @messages.last(n)
+    end
+
+    ##
+    # Returns the first message(s) in the buffer
+    # @param [Integer, nil] n
+    #  The number of messages to return
+    # @return [LLM::Message, Array<LLM::Message>, nil]
+    def first(n = UNDEFINED)
+      n.equal?(UNDEFINED) ? @messages.first : @messages.first(n)
+    end
+
+    ##
+    # Removes messages matching the block in-place.
+    # @yield [LLM::Message]
+    # @return [LLM::Buffer]
+    def reject!(&)
+      @messages.reject!(&)
+      self
+    end
+    alias_method :delete_if, :reject!
+
+    ##
+    # Keeps messages matching the block in-place.
+    # @yield [LLM::Message]
+    # @return [LLM::Buffer]
+    def select!(&)
+      @messages.select!(&)
+      self
+    end
+
+    ##
+    # Removes and returns the first message.
+    # @return [LLM::Message, nil]
+    def shift
+      @messages.shift
+    end
+
+    ##
+    # Removes all messages.
+    # @return [LLM::Buffer]
+    def clear
+      @messages.clear
+      self
+    end
+
+    ##
+    # Returns all elements after the first n.
+    # @param [Integer] n
+    #  The number of messages to skip
+    # @return [Array<LLM::Message>]
+    def drop(n)
+      @messages.drop(n)
+    end
+
+    ##
+    # Returns the first n elements without removing them.
+    # @param [Integer] n
+    #  The number of messages to return
+    # @return [Array<LLM::Message>]
+    def take(n)
+      @messages.take(n)
     end
 
     ##
@@ -103,6 +178,13 @@ module LLM
       @messages[index]
     end
 
+    ##
+    # Returns a reversed copy of the internal array.
+    # @return [Array]
+    def reverse
+      @messages.reverse
+    end
+
     ##
     # @return [String]
     def to_json(...)

data/lib/llm/compactor/null.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class LLM::Compactor
+  ##
+  # An {LLM::Compactor::Null LLM::Compactor::Null} is a compactor that
+  # does nothing. It is used as the default when no compactor strategy
+  # is configured.
+  #
+  # All methods return nil and produce no side effects.
+  class Null < self
+    ##
+    # @param [Hash] opts
+    #  Ignored
+    # @return [nil]
+    def call(**opts)
+      nil
+    end
+  end
+end

data/lib/llm/compactor/truncate.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+class LLM::Compactor
+  ##
+  # An {LLM::Compactor::Truncate LLM::Compactor::Truncate}
+  # drops the oldest messages when the conversation grows
+  # beyond a configured size, keeping only the N most recent
+  # messages.
+  #
+  # No LLM call is made but this strategy is purely lossy. It
+  # also fast - no network required and operates purely on
+  # memory.
+  class Truncate < self
+    ##
+    # @param [String, Integer] keep
+    #  The last (approx) n number of messages to keep.
+    #  This parameter can also be a percentage: eg "80%"
+    #  to keep 80% of the most recent messages.
+    # @return [Array<LLM::Message>, nil]
+    def call(keep: 64)
+      keep = parse(keep)
+      if keep <= 0 || keep > messages.reject(&:system?).size
+        nil
+      else
+        stream.on_compaction(self)
+        kept = take(messages, keep)
+        messages.replace([messages.select(&:system?).first, *kept].compact)
+        ctx.compacted = true
+        stream.on_compaction_finish(self)
+        kept
+      end
+    end
+
+    private
+
+    ##
+    # @param [String, Integer] input
+    #  The given input
+    # @return [Integer]
+    #  Returns the number of messages to keep
+    def parse(input)
+      if String === input
+        if input.end_with?("%")
+          count = ctx.messages.reject(&:system?).size
+          (count * (Float(input[0..-2]) / 100)).round
+        else
+          Integer(input)
+        end
+      else
+        Integer(input)
+      end
+    end
+
+    def take(messages, limit)
+      subset, in_tool_call = [], false
+      messages.reverse_each.with_index(1) do |m, index|
+        # We travel backwards - so we see a
+        # tool return before we see a tool
+        # call.
+        #
+        # When we see a tool return, our next
+        # task is to find where it was called
+        # from, and we will even override the
+        # limit to do this.
+        #
+        # Otherwise, the conversation will become
+        # corrupted and any attempt to use it will
+        # be an API-level error.
+        in_tool_call = m.tool_return?
+        if index >= limit
+          subset.unshift(m)
+          in_tool_call ? next : break
+        else
+          subset.unshift(m)
+        end
+      end
+      subset
+    end
+  end
+end