llm.rb 5.4.0 → 6.1.0

data/lib/llm/context/deserializer.rb

@@ -26,6 +26,7 @@ class LLM::Context
         LLM.json.load(string)
       end
       @messages.concat [*ctx["messages"]].map { deserialize_message(_1) }
+      @compacted = !!ctx["compacted"]
       self
     end
     alias_method :restore, :deserialize

data/lib/llm/context.rb

@@ -40,6 +40,14 @@ module LLM
     include Serializer
     include Deserializer
 
+    ZERO_USAGE = LLM::Object.from(
+      input_tokens: 0,
+      output_tokens: 0,
+      reasoning_tokens: 0,
+      total_tokens: 0
+    )
+    private_constant :ZERO_USAGE
+
     ##
     # Returns the accumulated message history for this context
     # @return [LLM::Buffer<LLM::Message>]
@@ -104,6 +112,14 @@ module LLM
       @compactor = compactor
     end
 
+    ##
+    # Returns whether the context has been compacted and no later model
+    # response has cleared that state.
+    # @return [Boolean]
+    # @api private
+    attr_accessor :compacted
+    alias_method :compacted?, :compacted
+
     ##
     # Returns a guard, if configured.
     #
@@ -172,13 +188,14 @@ module LLM
     #   puts res.messages[0].content
     def talk(prompt, params = {})
       return respond(prompt, params) if mode == :responses
-      @owner = Fiber.current
+      @owner = @llm.request_owner
       compactor.compact!(prompt) if compactor.compact?(prompt)
       params = params.merge(messages: @messages.to_a)
       params = @params.merge(params)
       prompt, params = transform(prompt, params)
       bind!(params[:stream], params[:model], params[:tools])
       res = @llm.complete(prompt, params)
+      self.compacted = false
       role = params[:role] || @llm.user_role
       role = @llm.tool_role if params[:role].nil? && [*prompt].grep(LLM::Function::Return).any?
       @messages.concat LLM::Prompt === prompt ? prompt.to_a : [LLM::Message.new(role, prompt)]
@@ -201,7 +218,7 @@ module LLM
     #   res = ctx.respond("What is the capital of France?")
     #   puts res.output_text
     def respond(prompt, params = {})
-      @owner = Fiber.current
+      @owner = @llm.request_owner
       compactor.compact!(prompt) if compactor.compact?(prompt)
       params = @params.merge(params)
       prompt, params = transform(prompt, params)
@@ -209,6 +226,7 @@ module LLM
       res_id = params[:store] == false ? nil : @messages.find(&:assistant?)&.response&.response_id
       params = params.merge(previous_response_id: res_id, input: @messages.to_a).compact
       res = @llm.responses.create(prompt, params)
+      self.compacted = false
       role = params[:role] || @llm.user_role
       @messages.concat LLM::Prompt === prompt ? prompt.to_a : [LLM::Message.new(role, prompt)]
       @messages.concat [res.choices[-1]]
@@ -313,27 +331,31 @@ module LLM
     # This is inspired by Go's context cancellation model.
     # @return [nil]
     def interrupt!
+      pending = functions.to_a
       llm.interrupt!(@owner)
       queue&.interrupt!
+      return if pending.empty?
+      pending.each(&:interrupt!)
+      returns = pending.map { _1.cancel(reason: "function call cancelled") }
+      @messages << LLM::Message.new(@llm.tool_role, returns)
+      nil
     end
     alias_method :cancel!, :interrupt!
 
     ##
     # Returns token usage accumulated in this context
-    # @note
-    # This method returns token usage for the latest
-    # assistant message, and it returns nil for non-assistant
-    # messages.
-    # @return [LLM::Object, nil]
+    # @return [LLM::Object]
     def usage
-      usage = @messages.find(&:assistant?)&.usage
-      return unless usage
-      LLM::Object.from(
-        input_tokens: usage.input_tokens || 0,
-        output_tokens: usage.output_tokens || 0,
-        reasoning_tokens: usage.reasoning_tokens || 0,
-        total_tokens: usage.total_tokens || 0
-      )
+      if usage = @messages.find(&:assistant?)&.usage
+        LLM::Object.from(
+          input_tokens: usage.input_tokens || 0,
+          output_tokens: usage.output_tokens || 0,
+          reasoning_tokens: usage.reasoning_tokens || 0,
+          total_tokens: usage.total_tokens || 0
+        )
+      else
+        ZERO_USAGE
+      end
     end
 
     ##
@@ -403,7 +425,12 @@ module LLM
     ##
     # @return [Hash]
     def to_h
-      {schema_version: 1, model:, messages: @messages.map { serialize_message(_1) }}
+      {
+        schema_version: 1,
+        model:,
+        compacted:,
+        messages: @messages.map { serialize_message(_1) }
+      }
     end
 
     ##
@@ -432,12 +459,12 @@ module LLM
     #  Returns an _approximate_ cost for a given context
     #  based on both the provider, and model
     def cost
-      return LLM::Cost.new(0, 0) unless usage
       cost = LLM.registry_for(llm).cost(model:)
-      LLM::Cost.new(
-        (cost.input.to_f / 1_000_000.0)  * usage.input_tokens,
-        (cost.output.to_f / 1_000_000.0) * usage.output_tokens
-      )
+      input_cost = (cost.input.to_f / 1_000_000.0) * usage.input_tokens
+      output_cost = (cost.output.to_f / 1_000_000.0) * usage.output_tokens
+      LLM::Cost.new(input_cost, output_cost)
+    rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
+      LLM::Cost.new(0, 0)
     end
 
     ##

data/lib/llm/error.rb

@@ -63,6 +63,10 @@ module LLM
   # When a request is interrupted
   Interrupt = Class.new(Error)
 
+  ##
+  # When a concurrency strategy cannot execute a given tool
+  RactorError = Class.new(Error)
+
   ##
   # When a tool call cannot be mapped to a local tool
   NoSuchToolError = Class.new(Error)

data/lib/llm/function/ractor/task.rb

@@ -15,8 +15,12 @@ class LLM::Function
     # @param [String, nil] id
     # @param [String] name
     # @param [Hash, Array, nil] arguments
+    # @param [LLM::Tracer, nil] tracer
+    # @param [Object, nil] span
     # @return [LLM::Function::Ractor::Task]
-    def initialize(runner_class, id, name, arguments)
+    def initialize(runner_class, id, name, arguments, tracer: nil, span: nil)
+      @tracer = tracer
+      @span = span
       @mailbox = Ractor::Mailbox.new(build_task(runner_class, id, name, arguments))
     end
 
@@ -37,7 +41,9 @@ class LLM::Function
     # @return [LLM::Function::Return]
     def wait
       id, name, value = mailbox.wait
-      Return.new(id, name, value)
+      result = Return.new(id, name, value)
+      @tracer&.on_tool_finish(result:, span: @span)
+      result
     end
     alias_method :value, :wait
 

data/lib/llm/function.rb

@@ -228,8 +228,12 @@ class LLM::Function
         Fiber.yield
       end.tap(&:resume)
     when :ractor
-      raise ArgumentError, "Ractor concurrency only supports class-based tools" unless Class === @runner
-      Ractor::Task.new(@runner, id, name, arguments)
+      raise LLM::RactorError, "Ractor concurrency only supports class-based tools" unless Class === @runner
+      if @runner.respond_to?(:skill?) && @runner.skill?
+        raise LLM::RactorError, "Ractor concurrency does not support skill-backed tools"
+      end
+      span = @tracer&.on_tool_start(id:, name:, arguments:, model:)
+      Ractor::Task.new(@runner, id, name, arguments, tracer: @tracer, span:)
     else
       raise ArgumentError, "Unknown strategy: #{strategy.inspect}. Expected :thread, :task, :fiber, or :ractor"
     end

data/lib/llm/provider/transport/http/execution.rb

@@ -38,7 +38,7 @@ module LLM::Provider::Transport
           perform_request(http, request, stream, stream_parser, &b)
         end
         [handle_response(res, tracer, span), span, tracer]
-      rescue *LLM::Provider::Transport::HTTP::Interruptible::INTERRUPT_ERRORS
+      rescue *transport.interrupt_errors
         raise LLM::Interrupt, "request interrupted" if transport.interrupted?(owner)
         raise
       end

data/lib/llm/provider/transport/http/interruptible.rb

@@ -1,109 +1,114 @@
 # frozen_string_literal: true
 
 class LLM::Provider
-  module Transport
-    class HTTP
-      ##
-      # Internal request interruption methods for
-      # {LLM::Provider::Transport::HTTP}.
-      #
-      # This module tracks active requests by execution owner and provides
-      # the logic used to interrupt an in-flight request by closing the
-      # active HTTP connection.
-      #
-      # @api private
-      module Interruptible
-        INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
-        Request = Struct.new(:http, :connection, keyword_init: true)
+  ##
+  # Internal request interruption methods for
+  # {LLM::Provider::Transport::HTTP}.
+  #
+  # This module tracks active requests by execution owner and provides
+  # the logic used to interrupt an in-flight request by closing the
+  # active HTTP connection.
+  #
+  # @api private
+  module Transport::HTTP::Interruptible
+    INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+    Request = Struct.new(:http, :connection, keyword_init: true)
 
-        ##
-        # Interrupt an active request, if any.
-        # @param [Fiber] owner
-        #  The execution owner whose request should be interrupted
-        # @return [nil]
-        def interrupt!(owner)
-          req = request_for(owner) or return
-          lock { (@interrupts ||= {})[owner] = true }
-          if persistent_http?(req.http)
-            close_socket(req.connection&.http)
-            req.http.finish(req.connection)
-          elsif transient_http?(req.http)
-            close_socket(req.http)
-            req.http.finish if req.http.active?
-          end
-        rescue *INTERRUPT_ERRORS
-          nil
-        end
-
-        private
+    def interrupt_errors
+      [*INTERRUPT_ERRORS, *optional_interrupt_errors]
+    end
 
-        ##
-        # Closes the active socket for a request, if present.
-        # @param [Net::HTTP, nil] http
-        # @return [nil]
-        def close_socket(http)
-          socket = http&.instance_variable_get(:@socket) or return
-          socket = socket.io if socket.respond_to?(:io)
-          socket.close
-        rescue *INTERRUPT_ERRORS
-          nil
-        end
+    ##
+    # Interrupt an active request, if any.
+    # @param [Fiber] owner
+    #  The execution owner whose request should be interrupted
+    # @return [nil]
+    def interrupt!(owner)
+      req = request_for(owner) or return
+      lock { (@interrupts ||= {})[owner] = true }
+      if persistent_http?(req.http)
+        close_socket(req.connection&.http)
+        req.http.finish(req.connection)
+      elsif transient_http?(req.http)
+        close_socket(req.http)
+        req.http.finish if req.http.active?
+      end
+      owner.stop if owner.respond_to?(:stop)
+    rescue *interrupt_errors
+      nil
+    end
 
-        ##
-        # Returns whether the active request is using a transient HTTP client.
-        # @param [Object, nil] http
-        # @return [Boolean]
-        def transient_http?(http)
-          Net::HTTP === http
-        end
+    private
 
-        ##
-        # Returns whether the active request is using a persistent HTTP client.
-        # @param [Object, nil] http
-        # @return [Boolean]
-        def persistent_http?(http)
-          defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
-        end
+    ##
+    # Closes the active socket for a request, if present.
+    # @param [Net::HTTP, nil] http
+    # @return [nil]
+    def close_socket(http)
+      socket = http&.instance_variable_get(:@socket) or return
+      socket = socket.io if socket.respond_to?(:io)
+      socket.close
+    rescue *interrupt_errors
+      nil
+    end
 
-        ##
-        # Returns the active request for an execution owner.
-        # @param [Fiber] owner
-        # @return [Request, nil]
-        def request_for(owner)
-          lock do
-            @requests ||= {}
-            @requests[owner]
-          end
-        end
+    ##
+    # Returns whether the active request is using a transient HTTP client.
+    # @param [Object, nil] http
+    # @return [Boolean]
+    def transient_http?(http)
+      Net::HTTP === http
+    end
 
-        ##
-        # Records an active request for an execution owner.
-        # @param [Request] req
-        # @param [Fiber] owner
-        # @return [Request]
-        def set_request(req, owner)
-          lock do
-            @requests ||= {}
-            @requests[owner] = req
-          end
-        end
+    ##
+    # Returns whether the active request is using a persistent HTTP client.
+    # @param [Object, nil] http
+    # @return [Boolean]
+    def persistent_http?(http)
+      defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
+    end
 
-        ##
-        # Clears the active request for an execution owner.
-        # @param [Fiber] owner
-        # @return [Request, nil]
-        def clear_request(owner)
-          lock { @requests&.delete(owner) }
-        end
+    ##
+    # Returns the active request for an execution owner.
+    # @param [Fiber] owner
+    # @return [Request, nil]
+    def request_for(owner)
+      lock do
+        @requests ||= {}
+        @requests[owner]
+      end
+    end
 
-        ##
-        # Returns whether an execution owner was interrupted.
-        # @param [Fiber] owner
-        # @return [Boolean, nil]
-        def interrupted?(owner)
-          lock { @interrupts&.delete(owner) }
-        end
+    ##
+    # Records an active request for an execution owner.
+    # @param [Request] req
+    # @param [Fiber] owner
+    # @return [Request]
+    def set_request(req, owner)
+      lock do
+        @requests ||= {}
+        @requests[owner] = req
       end
     end
+
+    ##
+    # Clears the active request for an execution owner.
+    # @param [Fiber] owner
+    # @return [Request, nil]
+    def clear_request(owner)
+      lock { @requests&.delete(owner) }
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Fiber] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      lock { @interrupts&.delete(owner) }
+    end
+
+    def optional_interrupt_errors
+      defined?(::Async::Stop) ? [Async::Stop] : []
+    end
   end
 end

data/lib/llm/provider/transport/http.rb

@@ -50,9 +50,10 @@ class LLM::Provider
 
       ##
       # Returns the current request owner.
-      # @return [Fiber]
+      # @return [Object]
       def request_owner
-        Fiber.current
+        return Fiber.current unless defined?(::Async)
+        Async::Task.current || Fiber.current
       end
 
       ##

data/lib/llm/provider.rb

@@ -338,6 +338,14 @@ class LLM::Provider
   end
   alias_method :cancel!, :interrupt!
 
+  ##
+  # Returns the current request owner used by the transport.
+  # @return [Object]
+  # @api private
+  def request_owner
+    transport.request_owner
+  end
+
   ##
   # @param [Object] stream
   # @return [Boolean]

data/lib/llm/sequel/agent.rb

@@ -12,7 +12,6 @@ module LLM::Sequel
   module Agent
     require_relative "plugin"
     EMPTY_HASH = LLM::Sequel::Plugin::EMPTY_HASH
-    DEFAULT_USAGE_COLUMNS = LLM::Sequel::Plugin::DEFAULT_USAGE_COLUMNS
     DEFAULTS = LLM::Sequel::Plugin::DEFAULTS
     Utils = LLM::Sequel::Plugin::Utils
 
@@ -24,11 +23,8 @@ module LLM::Sequel
 
     def self.configure(model, options = EMPTY_HASH, &block)
       options = DEFAULTS.merge(options)
-      usage_columns = DEFAULT_USAGE_COLUMNS.merge(options[:usage_columns] || EMPTY_HASH)
-      model.instance_variable_set(
-        :@llm_agent_options,
-        options.merge(usage_columns: usage_columns.freeze).freeze
-      )
+      model.db.extension :pg_json if %i[json jsonb].include?(options[:format])
+      model.instance_variable_set(:@llm_agent_options, options.freeze)
       model.instance_exec(&block) if block
     end
 
@@ -80,7 +76,6 @@ module LLM::Sequel
           options = self.class.llm_plugin_options
           columns = Agent::Utils.columns(options)
           params = Agent::Utils.resolve_options(self, options[:context], Agent::EMPTY_HASH).dup
-          params[:model] ||= self[columns[:model_column]]
           ctx = self.class.agent.new(llm, params.compact)
           data = self[columns[:data_column]]
           if data.nil? || data == ""

data/lib/llm/sequel/plugin.rb

@@ -17,11 +17,6 @@ module LLM::Sequel
   # can also be configured as symbols that are called on the model.
   module Plugin
     EMPTY_HASH = {}.freeze
-    DEFAULT_USAGE_COLUMNS = {
-      input_tokens: :input_tokens,
-      output_tokens: :output_tokens,
-      total_tokens: :total_tokens
-    }.freeze
 
     ##
     # Shared helper methods for the ORM wrapper.
@@ -68,38 +63,46 @@ module LLM::Sequel
       # Maps wrapper options onto the record's storage columns.
       # @return [Hash]
       def self.columns(options)
-        usage_columns = options[:usage_columns]
         {
-          provider_column: options[:provider_column],
-          model_column: options[:model_column],
-          data_column: options[:data_column],
-          input_tokens: usage_columns[:input_tokens],
-          output_tokens: usage_columns[:output_tokens],
-          total_tokens: usage_columns[:total_tokens]
+          data_column: options[:data_column]
         }.freeze
       end
 
+      ##
+      # Resolves the provider runtime for a record.
+      # @return [LLM::Provider]
+      def self.resolve_provider(obj, options, empty_hash)
+        provider = resolve_option(obj, options[:provider])
+        return provider if LLM::Provider === provider
+        raise ArgumentError, "provider: must resolve to an LLM::Provider instance"
+      end
+
       ##
       # Persists the runtime state and usage columns back onto the record.
       # @return [void]
       def self.save(obj, ctx, options)
         columns = self.columns(options)
-        obj.update(
-          columns[:data_column] => serialize_context(ctx, options[:format]),
-          columns[:input_tokens] => ctx.usage.input_tokens,
-          columns[:output_tokens] => ctx.usage.output_tokens,
-          columns[:total_tokens] => ctx.usage.total_tokens
-        )
+        payload = serialize_context(ctx, options[:format])
+        payload = wrap_json_payload(payload, options[:format])
+        obj.update(columns[:data_column] => payload)
+      end
+
+      ##
+      # Wraps JSON payloads for Sequel PostgreSQL adapters when needed.
+      # @return [Object]
+      def self.wrap_json_payload(payload, format)
+        case format
+        when :json then Sequel.pg_json_wrap(payload)
+        when :jsonb then Sequel.pg_jsonb_wrap(payload)
+        else payload
+        end
       end
     end
     DEFAULTS = {
-      provider_column: :provider,
-      model_column: :model,
       data_column: :data,
       format: :string,
-      usage_columns: DEFAULT_USAGE_COLUMNS,
       tracer: nil,
-      provider: EMPTY_HASH,
+      provider: nil,
       context: EMPTY_HASH
     }.freeze
 
@@ -134,14 +137,13 @@ module LLM::Sequel
     # @option options [Proc, Symbol, LLM::Tracer, nil] :tracer
     #   Optional tracer, method name, or proc that resolves to one and is
     #   assigned through `llm.tracer = ...` on the resolved provider.
+    # @option options [Proc, Symbol, LLM::Provider] :provider
+    #   Must resolve to an `LLM::Provider` instance for the current record.
     # @return [void]
     def self.configure(model, options = EMPTY_HASH)
       options = DEFAULTS.merge(options)
-      usage_columns = DEFAULT_USAGE_COLUMNS.merge(options[:usage_columns] || EMPTY_HASH)
-      model.instance_variable_set(
-        :@llm_plugin_options,
-        options.merge(usage_columns: usage_columns.freeze).freeze
-      )
+      model.db.extension :pg_json if %i[json jsonb].include?(options[:format])
+      model.instance_variable_set(:@llm_plugin_options, options.freeze)
     end
   end
 
@@ -247,12 +249,7 @@ module LLM::Sequel
     # Returns usage from the mapped usage columns.
     # @return [LLM::Object]
     def usage
-      columns = Utils.columns(self.class.llm_plugin_options)
-      LLM::Object.from(
-        input_tokens: self[columns[:input_tokens]] || 0,
-        output_tokens: self[columns[:output_tokens]] || 0,
-        total_tokens: self[columns[:total_tokens]] || 0
-      )
+      ctx.usage || LLM::Object.from(input_tokens: 0, output_tokens: 0, total_tokens: 0)
     end
 
     ##
@@ -304,11 +301,8 @@ module LLM::Sequel
     # @return [LLM::Provider]
     def llm
       options = self.class.llm_plugin_options
-      columns = Utils.columns(options)
-      provider = self[columns[:provider_column]]
-      kwargs = Utils.resolve_options(self, options[:provider], Plugin::EMPTY_HASH)
       return @llm if @llm
-      @llm = LLM.method(provider).call(**kwargs)
+      @llm = Utils.resolve_provider(self, options, Plugin::EMPTY_HASH)
       @llm.tracer = Utils.resolve_option(self, options[:tracer]) if options[:tracer]
       @llm
     end
@@ -322,7 +316,6 @@ module LLM::Sequel
         options = self.class.llm_plugin_options
         columns = Utils.columns(options)
         params = Utils.resolve_options(self, options[:context], Plugin::EMPTY_HASH).dup
-        params[:model] ||= self[columns[:model_column]]
         ctx = LLM::Context.new(llm, params.compact)
         data = self[columns[:data_column]]
         if data.nil? || data == ""

data/lib/llm/skill.rb

@@ -76,6 +76,8 @@ module LLM
     def call(ctx)
       instructions, tools, tracer = self.instructions, self.tools, ctx.llm.tracer
       params = ctx.params.merge(mode: ctx.mode).reject { [:tools, :schema].include?(_1) }
+      concurrency = params[:stream].extra[:concurrency] if LLM::Stream === params[:stream]
+      params[:concurrency] = concurrency if concurrency
       agent = Class.new(LLM::Agent) do
         instructions(instructions)
         tools(*tools)
@@ -98,6 +100,10 @@ module LLM
         description skill.description
         attr_accessor :tracer
 
+        define_singleton_method(:skill?) do
+          true
+        end
+
         define_method(:call) do
           skill.call(ctx)
         end

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "5.4.0"
+  VERSION = "6.1.0"
 end

data/llm.gemspec

@@ -57,4 +57,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activerecord", "~> 8.0"
   spec.add_development_dependency "sequel", "~> 5.0"
   spec.add_development_dependency "sqlite3", "~> 2.0"
+  spec.add_development_dependency "pg", "~> 1.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm.rb
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 6.1.0
 platform: ruby
 authors:
 - Antar Azri
@@ -236,6 +236,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 description: |
   llm.rb is a lightweight runtime for building capable AI systems in Ruby.
   It is not just an API wrapper. llm.rb gives you one runtime for providers,