llm.rb 6.0.0 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8cc16b548be77e0a2bb78c0e130b14f0ac8fdabb2c2f082dd2824282e5a5733b
-  data.tar.gz: 668655ba6a7d65d44b53cc1b8b33afddf3563dc216df83181fb2b7394a2847ff
+  metadata.gz: 57b39b3b4b79d1d9f8cfd10426ad233d698dd6e3ed84bfef887c8c63f543f40f
+  data.tar.gz: 443ed7e2a04259c69d41b1da7a42e7637efaa4ab1075548706ce349bced7ed51
 SHA512:
-  metadata.gz: 2257aeec49a43c56bfc974e3fc190a850ea27c98c437c7c242efe6c645c000eebdbe2e731fcc0b287303690c1055768f2de32be6ac900391c5156260da9c2ce5
-  data.tar.gz: 8f4f8f3475ac1bd2d9ffbd8c0b413b46224936459eb0ed6bb395876b994c1ab67ec02cf670176f90b8d9b892b59ab4b67271f4b69c80fa7c094aa89310ab5c58
+  metadata.gz: f8e53dc41eacf16cea35f64a6048aa77852fcf7a135676b2b9c02e37beff174b5a500948477c4f931ff0a71d20c4503ba3e9eef19358d3aaa204040e77fe14c5
+  data.tar.gz: 358ce7f33d2dca51365f6581867006970fd66079dcaa189268e2deff2f297c89b8332fd11b714bedfd89124413b7a9e12fc09d928c2c28f2e9cb2368f2bc3e24

data/CHANGELOG.md

@@ -1,5 +1,56 @@
 # Changelog
 
+## v6.1.0
+
+Changes since `v6.0.0`.
+
+This release tightens interrupt and compaction behavior for long-running
+contexts. It adds `LLM::Buffer#rindex`, supports percentage-based token
+thresholds in `LLM::Compactor`, tracks persisted compaction state through
+context serialization, reliably interrupts Async-backed requests, preserves
+valid tool-call history on cancellation, keeps concurrent skill tool loops
+running on streamed agents, and returns zero-valued usage objects when no
+provider usage has been recorded yet.
+
+### Change
+
+* **Add `LLM::Buffer#rindex`** <br>
+  Add `LLM::Buffer#rindex` as a direct forward to the underlying message
+  array so callers can find the last matching message index through the
+  buffer API.
+
+* **Support percentage compaction token thresholds** <br>
+  Let `LLM::Compactor` accept `token_threshold:` values like `"90%"` so
+  compaction can trigger at a percentage of the active model context
+  window.
+
+### Fix
+
+* **Interrupt Async-backed requests reliably** <br>
+  Track request ownership through the provider transport so contexts use
+  the active Async task when available, letting `ctx.interrupt!`
+  reliably cancel streamed requests under Async runtimes and surface
+  them as `LLM::Interrupt`.
+
+* **Preserve valid tool-call history on cancellation** <br>
+  Append cancelled tool-return messages for unresolved tool calls during
+  `ctx.interrupt!` so follow-up provider requests do not fail with
+  invalid tool-call history after pending tool work is cancelled.
+
+* **Preserve concurrent skill tool loops on streamed agents** <br>
+  Propagate the active agent concurrency through the effective request
+  stream so nested skill agents keep using queued `wait(...)` tool
+  execution instead of falling back to direct `:call` execution.
+
+* **Track persisted compaction state on contexts** <br>
+  Mark contexts as compacted after `LLM::Compactor#compact!`, persist and
+  restore that state through context serialization, and clear it after the
+  next successful model response.
+
+* **Return zero-valued usage objects from contexts** <br>
+  Make `LLM::Context#usage` consistently return an `LLM::Object`, using a
+  zero-valued usage object when no provider usage has been recorded yet.
+
 ## v6.0.0
 
 Changes since `v5.4.0`.

data/README.md

@@ -4,7 +4,7 @@
 <p align="center">
   <a href="https://0x1eef.github.io/x/llm.rb?rebuild=1"><img src="https://img.shields.io/badge/docs-0x1eef.github.io-blue.svg" alt="RubyDoc"></a>
   <a href="https://opensource.org/license/0bsd"><img src="https://img.shields.io/badge/License-0BSD-orange.svg?" alt="License"></a>
-  <a href="https://github.com/llmrb/llm.rb/tags"><img src="https://img.shields.io/badge/version-6.0.0-green.svg?" alt="Version"></a>
+  <a href="https://github.com/llmrb/llm.rb/tags"><img src="https://img.shields.io/badge/version-6.1.0-green.svg?" alt="Version"></a>
 </p>
 
 ## About
@@ -163,12 +163,15 @@ and when a stream is present it emits `on_compaction` and
 `on_compaction_finish` through [`LLM::Stream`](https://0x1eef.github.io/x/llm.rb/LLM/Stream.html).
 The compactor can also use a different model from the main context, which is
 useful when you want summarization to run on a cheaper or faster model.
+`token_threshold:` accepts either a fixed token count or a percentage string
+like `"90%"`, which resolves against the active model context window and
+triggers compaction once total token usage goes over that percentage.
 
 ```ruby
 ctx = LLM::Context.new(
   llm,
   compactor: {
-    message_threshold: 200,
+    token_threshold: "90%",
     retention_window: 8,
     model: "gpt-5.4-mini"
   }
@@ -624,7 +627,10 @@ long-lived contexts can summarize older history and expose the lifecycle
 through stream hooks. This approach is inspired by General Intelligence
 Systems' [Brute](https://github.com/general-intelligence-systems/brute). The
 compactor can also use its own `model:` if you want summarization to run on a
-different model from the main context. <br> See the [deepdive (web)](https://0x1eef.github.io/x/llm.rb/file.deepdive.html) or [deepdive (markdown)](resources/deepdive.md) for more examples.
+different model from the main context. `token_threshold:` accepts either a
+fixed token count or a percentage string like `"90%"`, which resolves
+against the active model context window and triggers compaction once total
+token usage goes over that percentage. <br> See the [deepdive (web)](https://0x1eef.github.io/x/llm.rb/file.deepdive.html) or [deepdive (markdown)](resources/deepdive.md) for more examples.
 
 ```ruby
 require "llm"
@@ -644,7 +650,7 @@ ctx = LLM::Context.new(
   llm,
   stream: Stream.new,
   compactor: {
-    message_threshold: 200,
+    token_threshold: "90%",
     retention_window: 8,
     model: "gpt-5.4-mini"
   }

data/lib/llm/agent.rb

@@ -394,6 +394,8 @@ module LLM
     def run_loop(method, prompt, params)
       loop = proc do
         max = Integer(params.delete(:tool_attempts) || 25)
+        stream = params[:stream] || @ctx.params[:stream]
+        stream.extra[:concurrency] = concurrency if LLM::Stream === stream
         res = @ctx.public_send(method, apply_instructions(prompt), params)
         max.times do
           break if @ctx.functions.empty?

data/lib/llm/buffer.rb

@@ -52,6 +52,14 @@ module LLM
       reverse_each.find(...)
     end
 
+    ##
+    # Returns the index of the last message matching the given block.
+    # @yield [LLM::Message]
+    # @return [Integer, nil]
+    def rindex(...)
+      @messages.rindex(...)
+    end
+
     ##
     # Returns the last message(s) in the buffer
     # @param [Integer, nil] n

data/lib/llm/compactor.rb

@@ -11,7 +11,9 @@
 # The compactor can also use a different model from the main context by
 # setting `model:` in the compactor config. Compaction thresholds are opt-in:
 # provide `message_threshold:` and/or `token_threshold:` to enable policy-
-# driven compaction.
+# driven compaction. `token_threshold:` accepts either an integer token count
+# or a percentage string like `"90%"`, which resolves against the current
+# model context window.
 class LLM::Compactor
   DEFAULTS = {
     retention_window: 8,
@@ -25,8 +27,11 @@ class LLM::Compactor
   ##
   # @param [LLM::Context] ctx
   # @param [Hash] config
-  # @option config [Integer, nil] :token_threshold
-  #  Enables token-based compaction.
+  # @option config [Integer, String, nil] :token_threshold
+  #  Enables token-based compaction. Integer values are treated as a fixed
+  #  token count. Percentage strings like `"90%"` are resolved against
+  #  {LLM::Context#context_window}; if the context window is unknown, the
+  #  percentage threshold is treated as disabled.
   # @option config [Integer, nil] :message_threshold
   #  Enables message-count-based compaction.
   # @option config [Integer] :retention_window
@@ -39,18 +44,22 @@ class LLM::Compactor
   end
 
   ##
-  # Returns true when the context should be compacted
+  # Returns true when the context should be compacted.
+  #
+  # When `token_threshold:` is a percentage string such as `"90%"`, the
+  # threshold is resolved against the current context window and compared to
+  # the current total token usage.
   # @param [Object] prompt
   #  The next prompt or turn input
   # @return [Boolean]
-  def compact?(prompt = nil)
+  def compactable?(prompt = nil)
     return false if ctx.functions.any? || [*prompt].grep(LLM::Function::Return).any?
     messages = ctx.messages.reject(&:system?)
     return true if config[:message_threshold] && messages.size > config[:message_threshold]
-    usage = ctx.usage
-    return true if config[:token_threshold] && usage && usage.total_tokens > config[:token_threshold]
+    return true if token_threshold and ctx.usage.total_tokens > token_threshold
     false
   end
+  alias_method :compact?, :compactable?
 
   ##
   # Summarize older messages and replace them with a compact summary.
@@ -68,6 +77,7 @@ class LLM::Compactor
     older = messages[0...(messages.size - recent.size)]
     summary = LLM::Message.new(ctx.llm.user_role, "[Previous conversation summary]\n\n#{summarize(older)}", {compaction: true})
     ctx.messages.replace([*ctx.messages.take_while(&:system?), summary, *recent])
+    ctx.compacted = true
     stream.on_compaction_finish(ctx, self) if LLM::Stream === stream
     summary
   end
@@ -84,6 +94,15 @@ class LLM::Compactor
     messages[start..] || []
   end
 
+  def token_threshold
+    @token_threshold ||= begin
+      threshold = config[:token_threshold]
+      return threshold unless threshold.to_s.end_with?("%")
+      return if ctx.context_window <= 0
+      (ctx.context_window * threshold.delete_suffix("%").to_f / 100).floor
+    end
+  end
+
   def summarize(messages)
     model = config[:model] || ctx.params[:model] || ctx.llm.default_model
     ctx.llm.complete(summary_prompt(messages), model:).content

data/lib/llm/context/deserializer.rb

@@ -26,6 +26,7 @@ class LLM::Context
         LLM.json.load(string)
       end
       @messages.concat [*ctx["messages"]].map { deserialize_message(_1) }
+      @compacted = !!ctx["compacted"]
       self
     end
     alias_method :restore, :deserialize

data/lib/llm/context.rb

@@ -40,6 +40,14 @@ module LLM
     include Serializer
     include Deserializer
 
+    ZERO_USAGE = LLM::Object.from(
+      input_tokens: 0,
+      output_tokens: 0,
+      reasoning_tokens: 0,
+      total_tokens: 0
+    )
+    private_constant :ZERO_USAGE
+
     ##
     # Returns the accumulated message history for this context
     # @return [LLM::Buffer<LLM::Message>]
@@ -104,6 +112,14 @@ module LLM
       @compactor = compactor
     end
 
+    ##
+    # Returns whether the context has been compacted and no later model
+    # response has cleared that state.
+    # @return [Boolean]
+    # @api private
+    attr_accessor :compacted
+    alias_method :compacted?, :compacted
+
     ##
     # Returns a guard, if configured.
     #
@@ -172,13 +188,14 @@ module LLM
     #   puts res.messages[0].content
     def talk(prompt, params = {})
       return respond(prompt, params) if mode == :responses
-      @owner = Fiber.current
+      @owner = @llm.request_owner
       compactor.compact!(prompt) if compactor.compact?(prompt)
       params = params.merge(messages: @messages.to_a)
       params = @params.merge(params)
       prompt, params = transform(prompt, params)
       bind!(params[:stream], params[:model], params[:tools])
       res = @llm.complete(prompt, params)
+      self.compacted = false
       role = params[:role] || @llm.user_role
       role = @llm.tool_role if params[:role].nil? && [*prompt].grep(LLM::Function::Return).any?
       @messages.concat LLM::Prompt === prompt ? prompt.to_a : [LLM::Message.new(role, prompt)]
@@ -201,7 +218,7 @@ module LLM
     #   res = ctx.respond("What is the capital of France?")
     #   puts res.output_text
     def respond(prompt, params = {})
-      @owner = Fiber.current
+      @owner = @llm.request_owner
       compactor.compact!(prompt) if compactor.compact?(prompt)
       params = @params.merge(params)
       prompt, params = transform(prompt, params)
@@ -209,6 +226,7 @@ module LLM
       res_id = params[:store] == false ? nil : @messages.find(&:assistant?)&.response&.response_id
       params = params.merge(previous_response_id: res_id, input: @messages.to_a).compact
       res = @llm.responses.create(prompt, params)
+      self.compacted = false
       role = params[:role] || @llm.user_role
       @messages.concat LLM::Prompt === prompt ? prompt.to_a : [LLM::Message.new(role, prompt)]
       @messages.concat [res.choices[-1]]
@@ -313,27 +331,31 @@ module LLM
     # This is inspired by Go's context cancellation model.
     # @return [nil]
     def interrupt!
+      pending = functions.to_a
       llm.interrupt!(@owner)
       queue&.interrupt!
+      return if pending.empty?
+      pending.each(&:interrupt!)
+      returns = pending.map { _1.cancel(reason: "function call cancelled") }
+      @messages << LLM::Message.new(@llm.tool_role, returns)
+      nil
     end
     alias_method :cancel!, :interrupt!
 
     ##
     # Returns token usage accumulated in this context
-    # @note
-    # This method returns token usage for the latest
-    # assistant message, and it returns nil for non-assistant
-    # messages.
-    # @return [LLM::Object, nil]
+    # @return [LLM::Object]
     def usage
-      usage = @messages.find(&:assistant?)&.usage
-      return unless usage
-      LLM::Object.from(
-        input_tokens: usage.input_tokens || 0,
-        output_tokens: usage.output_tokens || 0,
-        reasoning_tokens: usage.reasoning_tokens || 0,
-        total_tokens: usage.total_tokens || 0
-      )
+      if usage = @messages.find(&:assistant?)&.usage
+        LLM::Object.from(
+          input_tokens: usage.input_tokens || 0,
+          output_tokens: usage.output_tokens || 0,
+          reasoning_tokens: usage.reasoning_tokens || 0,
+          total_tokens: usage.total_tokens || 0
+        )
+      else
+        ZERO_USAGE
+      end
     end
 
     ##
@@ -403,7 +425,12 @@ module LLM
     ##
     # @return [Hash]
     def to_h
-      {schema_version: 1, model:, messages: @messages.map { serialize_message(_1) }}
+      {
+        schema_version: 1,
+        model:,
+        compacted:,
+        messages: @messages.map { serialize_message(_1) }
+      }
     end
 
     ##
@@ -432,12 +459,12 @@ module LLM
     #  Returns an _approximate_ cost for a given context
     #  based on both the provider, and model
     def cost
-      return LLM::Cost.new(0, 0) unless usage
       cost = LLM.registry_for(llm).cost(model:)
-      LLM::Cost.new(
-        (cost.input.to_f / 1_000_000.0)  * usage.input_tokens,
-        (cost.output.to_f / 1_000_000.0) * usage.output_tokens
-      )
+      input_cost = (cost.input.to_f / 1_000_000.0) * usage.input_tokens
+      output_cost = (cost.output.to_f / 1_000_000.0) * usage.output_tokens
+      LLM::Cost.new(input_cost, output_cost)
+    rescue LLM::NoSuchModelError, LLM::NoSuchRegistryError
+      LLM::Cost.new(0, 0)
     end
 
     ##

data/lib/llm/provider/transport/http/execution.rb

@@ -38,7 +38,7 @@ module LLM::Provider::Transport
           perform_request(http, request, stream, stream_parser, &b)
         end
         [handle_response(res, tracer, span), span, tracer]
-      rescue *LLM::Provider::Transport::HTTP::Interruptible::INTERRUPT_ERRORS
+      rescue *transport.interrupt_errors
         raise LLM::Interrupt, "request interrupted" if transport.interrupted?(owner)
         raise
       end

data/lib/llm/provider/transport/http/interruptible.rb

@@ -1,109 +1,114 @@
 # frozen_string_literal: true
 
 class LLM::Provider
-  module Transport
-    class HTTP
-      ##
-      # Internal request interruption methods for
-      # {LLM::Provider::Transport::HTTP}.
-      #
-      # This module tracks active requests by execution owner and provides
-      # the logic used to interrupt an in-flight request by closing the
-      # active HTTP connection.
-      #
-      # @api private
-      module Interruptible
-        INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
-        Request = Struct.new(:http, :connection, keyword_init: true)
+  ##
+  # Internal request interruption methods for
+  # {LLM::Provider::Transport::HTTP}.
+  #
+  # This module tracks active requests by execution owner and provides
+  # the logic used to interrupt an in-flight request by closing the
+  # active HTTP connection.
+  #
+  # @api private
+  module Transport::HTTP::Interruptible
+    INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+    Request = Struct.new(:http, :connection, keyword_init: true)
 
-        ##
-        # Interrupt an active request, if any.
-        # @param [Fiber] owner
-        #  The execution owner whose request should be interrupted
-        # @return [nil]
-        def interrupt!(owner)
-          req = request_for(owner) or return
-          lock { (@interrupts ||= {})[owner] = true }
-          if persistent_http?(req.http)
-            close_socket(req.connection&.http)
-            req.http.finish(req.connection)
-          elsif transient_http?(req.http)
-            close_socket(req.http)
-            req.http.finish if req.http.active?
-          end
-        rescue *INTERRUPT_ERRORS
-          nil
-        end
-
-        private
+    def interrupt_errors
+      [*INTERRUPT_ERRORS, *optional_interrupt_errors]
+    end
 
-        ##
-        # Closes the active socket for a request, if present.
-        # @param [Net::HTTP, nil] http
-        # @return [nil]
-        def close_socket(http)
-          socket = http&.instance_variable_get(:@socket) or return
-          socket = socket.io if socket.respond_to?(:io)
-          socket.close
-        rescue *INTERRUPT_ERRORS
-          nil
-        end
+    ##
+    # Interrupt an active request, if any.
+    # @param [Fiber] owner
+    #  The execution owner whose request should be interrupted
+    # @return [nil]
+    def interrupt!(owner)
+      req = request_for(owner) or return
+      lock { (@interrupts ||= {})[owner] = true }
+      if persistent_http?(req.http)
+        close_socket(req.connection&.http)
+        req.http.finish(req.connection)
+      elsif transient_http?(req.http)
+        close_socket(req.http)
+        req.http.finish if req.http.active?
+      end
+      owner.stop if owner.respond_to?(:stop)
+    rescue *interrupt_errors
+      nil
+    end
 
-        ##
-        # Returns whether the active request is using a transient HTTP client.
-        # @param [Object, nil] http
-        # @return [Boolean]
-        def transient_http?(http)
-          Net::HTTP === http
-        end
+    private
 
-        ##
-        # Returns whether the active request is using a persistent HTTP client.
-        # @param [Object, nil] http
-        # @return [Boolean]
-        def persistent_http?(http)
-          defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
-        end
+    ##
+    # Closes the active socket for a request, if present.
+    # @param [Net::HTTP, nil] http
+    # @return [nil]
+    def close_socket(http)
+      socket = http&.instance_variable_get(:@socket) or return
+      socket = socket.io if socket.respond_to?(:io)
+      socket.close
+    rescue *interrupt_errors
+      nil
+    end
 
-        ##
-        # Returns the active request for an execution owner.
-        # @param [Fiber] owner
-        # @return [Request, nil]
-        def request_for(owner)
-          lock do
-            @requests ||= {}
-            @requests[owner]
-          end
-        end
+    ##
+    # Returns whether the active request is using a transient HTTP client.
+    # @param [Object, nil] http
+    # @return [Boolean]
+    def transient_http?(http)
+      Net::HTTP === http
+    end
 
-        ##
-        # Records an active request for an execution owner.
-        # @param [Request] req
-        # @param [Fiber] owner
-        # @return [Request]
-        def set_request(req, owner)
-          lock do
-            @requests ||= {}
-            @requests[owner] = req
-          end
-        end
+    ##
+    # Returns whether the active request is using a persistent HTTP client.
+    # @param [Object, nil] http
+    # @return [Boolean]
+    def persistent_http?(http)
+      defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
+    end
 
-        ##
-        # Clears the active request for an execution owner.
-        # @param [Fiber] owner
-        # @return [Request, nil]
-        def clear_request(owner)
-          lock { @requests&.delete(owner) }
-        end
+    ##
+    # Returns the active request for an execution owner.
+    # @param [Fiber] owner
+    # @return [Request, nil]
+    def request_for(owner)
+      lock do
+        @requests ||= {}
+        @requests[owner]
+      end
+    end
 
-        ##
-        # Returns whether an execution owner was interrupted.
-        # @param [Fiber] owner
-        # @return [Boolean, nil]
-        def interrupted?(owner)
-          lock { @interrupts&.delete(owner) }
-        end
+    ##
+    # Records an active request for an execution owner.
+    # @param [Request] req
+    # @param [Fiber] owner
+    # @return [Request]
+    def set_request(req, owner)
+      lock do
+        @requests ||= {}
+        @requests[owner] = req
       end
     end
+
+    ##
+    # Clears the active request for an execution owner.
+    # @param [Fiber] owner
+    # @return [Request, nil]
+    def clear_request(owner)
+      lock { @requests&.delete(owner) }
+    end
+
+    ##
+    # Returns whether an execution owner was interrupted.
+    # @param [Fiber] owner
+    # @return [Boolean, nil]
+    def interrupted?(owner)
+      lock { @interrupts&.delete(owner) }
+    end
+
+    def optional_interrupt_errors
+      defined?(::Async::Stop) ? [Async::Stop] : []
+    end
   end
 end

data/lib/llm/provider/transport/http.rb

@@ -50,9 +50,10 @@ class LLM::Provider
 
       ##
       # Returns the current request owner.
-      # @return [Fiber]
+      # @return [Object]
       def request_owner
-        Fiber.current
+        return Fiber.current unless defined?(::Async)
+        Async::Task.current || Fiber.current
       end
 
       ##

data/lib/llm/provider.rb

@@ -338,6 +338,14 @@ class LLM::Provider
   end
   alias_method :cancel!, :interrupt!
 
+  ##
+  # Returns the current request owner used by the transport.
+  # @return [Object]
+  # @api private
+  def request_owner
+    transport.request_owner
+  end
+
   ##
   # @param [Object] stream
   # @return [Boolean]

data/lib/llm/skill.rb

@@ -76,6 +76,8 @@ module LLM
     def call(ctx)
       instructions, tools, tracer = self.instructions, self.tools, ctx.llm.tracer
       params = ctx.params.merge(mode: ctx.mode).reject { [:tools, :schema].include?(_1) }
+      concurrency = params[:stream].extra[:concurrency] if LLM::Stream === params[:stream]
+      params[:concurrency] = concurrency if concurrency
       agent = Class.new(LLM::Agent) do
         instructions(instructions)
         tools(*tools)

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "6.0.0"
+  VERSION = "6.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm.rb
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.1.0
 platform: ruby
 authors:
 - Antar Azri