llm.rb 4.12.0 → 4.14.0

data/lib/llm/provider/transport/http/interruptible.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class LLM::Provider
+  module Transport
+    class HTTP
+      ##
+      # Internal request interruption methods for
+      # {LLM::Provider::Transport::HTTP}.
+      #
+      # This module tracks active requests by execution owner and provides
+      # the logic used to interrupt an in-flight request by closing the
+      # active HTTP connection.
+      #
+      # @api private
+      module Interruptible
+        INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+        Request = Struct.new(:http, :connection, keyword_init: true)
+
+        ##
+        # Interrupt an active request, if any.
+        # @param [Fiber] owner
+        #  The execution owner whose request should be interrupted
+        # @return [nil]
+        def interrupt!(owner)
+          req = request_for(owner) or return
+          lock { (@interrupts ||= {})[owner] = true }
+          if persistent_http?(req.http)
+            close_socket(req.connection&.http)
+            req.http.finish(req.connection)
+          elsif transient_http?(req.http)
+            close_socket(req.http)
+            req.http.finish if req.http.active?
+          end
+        rescue *INTERRUPT_ERRORS
+          nil
+        end
+
+        private
+
+        ##
+        # Closes the active socket for a request, if present.
+        # @param [Net::HTTP, nil] http
+        # @return [nil]
+        def close_socket(http)
+          socket = http&.instance_variable_get(:@socket) or return
+          socket = socket.io if socket.respond_to?(:io)
+          socket.close
+        rescue *INTERRUPT_ERRORS
+          nil
+        end
+
+        ##
+        # Returns whether the active request is using a transient HTTP client.
+        # @param [Object, nil] http
+        # @return [Boolean]
+        def transient_http?(http)
+          Net::HTTP === http
+        end
+
+        ##
+        # Returns whether the active request is using a persistent HTTP client.
+        # @param [Object, nil] http
+        # @return [Boolean]
+        def persistent_http?(http)
+          defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
+        end
+
+        ##
+        # Returns the active request for an execution owner.
+        # @param [Fiber] owner
+        # @return [Request, nil]
+        def request_for(owner)
+          lock do
+            @requests ||= {}
+            @requests[owner]
+          end
+        end
+
+        ##
+        # Records an active request for an execution owner.
+        # @param [Request] req
+        # @param [Fiber] owner
+        # @return [Request]
+        def set_request(req, owner)
+          lock do
+            @requests ||= {}
+            @requests[owner] = req
+          end
+        end
+
+        ##
+        # Clears the active request for an execution owner.
+        # @param [Fiber] owner
+        # @return [Request, nil]
+        def clear_request(owner)
+          lock { @requests&.delete(owner) }
+        end
+
+        ##
+        # Returns whether an execution owner was interrupted.
+        # @param [Fiber] owner
+        # @return [Boolean, nil]
+        def interrupted?(owner)
+          lock { @interrupts&.delete(owner) }
+        end
+      end
+    end
+  end
+end

data/lib/llm/provider/transport/http/stream_decoder.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module LLM::Provider::Transport
+  ##
+  # @private
+  class HTTP::StreamDecoder
+    ##
+    # @return [Object]
+    attr_reader :parser
+
+    ##
+    # @param [#parse!, #body] parser
+    # @return [LLM::Provider::Transport::HTTP::StreamDecoder]
+    def initialize(parser)
+      @buffer = +""
+      @cursor = 0
+      @data = []
+      @parser = parser
+    end
+
+    ##
+    # @param [String] chunk
+    # @return [void]
+    def <<(chunk)
+      @buffer << chunk
+      each_line { handle_line(_1) }
+    end
+
+    ##
+    # @return [Object]
+    def body
+      parser.body
+    end
+
+    ##
+    # @return [void]
+    def free
+      @buffer.clear
+      @cursor = 0
+      @data.clear
+      parser.free if parser.respond_to?(:free)
+    end
+
+    private
+
+    def handle_line(line)
+      if line == "\n" || line == "\r\n"
+        flush_sse_event
+      elsif line.start_with?("data:")
+        @data << field_value(line)
+      elsif line.start_with?("event:", "id:", "retry:", ":")
+      else
+        decode!(strip_newline(line))
+      end
+    end
+
+    def flush_sse_event
+      return if @data.empty?
+      decode!(@data.join("\n"))
+      @data.clear
+    end
+
+    def field_value(line)
+      value_start = line.getbyte(5) == 32 ? 6 : 5
+      strip_newline(line.byteslice(value_start..))
+    end
+
+    def strip_newline(line)
+      line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\n")
+      line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\r")
+      line
+    end
+
+    def decode!(payload)
+      return if payload.empty? || payload == "[DONE]"
+      chunk = LLM.json.load(payload)
+      parser.parse!(chunk) if chunk
+    rescue *LLM.json.parser_error
+    end
+
+    def each_line
+      while (newline = @buffer.index("\n", @cursor))
+        line = @buffer[@cursor..newline]
+        @cursor = newline + 1
+        yield(line)
+      end
+      return if @cursor.zero?
+      @buffer = @buffer[@cursor..] || +""
+      @cursor = 0
+    end
+  end
+end

data/lib/llm/provider/transport/http.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+class LLM::Provider
+  module Transport
+    ##
+    # The {LLM::Provider::Transport::HTTP LLM::Provider::Transport::HTTP}
+    # class manages HTTP connections for {LLM::Provider}. It handles
+    # transient and persistent clients, tracks active requests by owner,
+    # and interrupts in-flight requests when needed.
+    #
+    # @api private
+    class HTTP
+      require_relative "http/stream_decoder"
+      require_relative "http/interruptible"
+
+      include Interruptible
+
+      ##
+      # @param [String] host
+      # @param [Integer] port
+      # @param [Integer] timeout
+      # @param [Boolean] ssl
+      # @param [Boolean] persistent
+      # @return [LLM::Provider::Transport::HTTP]
+      def initialize(host:, port:, timeout:, ssl:, persistent: false)
+        @host = host
+        @port = port
+        @timeout = timeout
+        @ssl = ssl
+        @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
+        @persistent_client = persistent ? persistent_client : nil
+        @monitor = Monitor.new
+      end
+
+      ##
+      # Interrupt an active request, if any.
+      # @param [Fiber] owner
+      # @return [nil]
+      def interrupt!(owner)
+        super
+      end
+
+      ##
+      # Returns whether an execution owner was interrupted.
+      # @param [Fiber] owner
+      # @return [Boolean, nil]
+      def interrupted?(owner)
+        super
+      end
+
+      ##
+      # Returns the current request owner.
+      # @return [Fiber]
+      def request_owner
+        Fiber.current
+      end
+
+      ##
+      # Configures the transport to use a persistent HTTP connection pool.
+      # @return [LLM::Provider::Transport::HTTP]
+      def persist!
+        client = persistent_client
+        lock do
+          @persistent_client = client
+          self
+        end
+      end
+      alias_method :persistent, :persist!
+
+      ##
+      # @return [Boolean]
+      def persistent?
+        !persistent_client.nil?
+      end
+
+      ##
+      # Performs a request on the current HTTP transport.
+      # @param [Net::HTTPRequest] request
+      # @param [Fiber] owner
+      # @yieldparam [Net::HTTP] http
+      # @return [Object]
+      def request(request, owner:, &)
+        if persistent?
+          request_persistent(request, owner, &)
+        else
+          request_transient(request, owner, &)
+        end
+      ensure
+        clear_request(owner)
+      end
+
+      ##
+      # @return [String]
+      def inspect
+        "#<#{self.class.name}:0x#{object_id.to_s(16)} @persistent=#{persistent?}>"
+      end
+
+      private
+
+      attr_reader :host, :port, :timeout, :ssl, :base_uri
+
+      def request_transient(request, owner, &)
+        http = transient_client
+        set_request(Request.new(http:), owner)
+        yield http
+      end
+
+      def request_persistent(request, owner, &)
+        persistent_client.connection_for(URI.join(base_uri, request.path)) do |connection|
+          set_request(Request.new(http: persistent_client, connection:), owner)
+          yield connection.http
+        end
+      end
+
+      def persistent_client
+        LLM.lock(:clients) do
+          if LLM.clients[client_id]
+            LLM.clients[client_id]
+          else
+            require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
+            client = Net::HTTP::Persistent.new(name: self.class.name)
+            client.read_timeout = timeout
+            LLM.clients[client_id] = client
+          end
+        end
+      end
+
+      def transient_client
+        client = Net::HTTP.new(host, port)
+        client.read_timeout = timeout
+        client.use_ssl = ssl
+        client
+      end
+
+      def client_id
+        "#{host}:#{port}:#{timeout}:#{ssl}"
+      end
+
+      def lock(&)
+        @monitor.synchronize(&)
+      end
+    end
+  end
+end

data/lib/llm/provider.rb

@@ -7,14 +7,9 @@
 # @abstract
 class LLM::Provider
   require "net/http"
-  require_relative "client"
-  include LLM::Client
-
-  @@clients = {}
-
-  ##
-  # @api private
-  def self.clients = @@clients
+  require_relative "provider/transport/http"
+  require_relative "provider/transport/http/execution"
+  include Transport::HTTP::Execution
 
   ##
   # @param [String, nil] key
@@ -36,9 +31,9 @@ class LLM::Provider
     @port = port
     @timeout = timeout
     @ssl = ssl
-    @client = persistent ? persistent_client : nil
     @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
     @headers = {"User-Agent" => "llm.rb v#{LLM::VERSION}"}
+    @transport = Transport::HTTP.new(host:, port:, timeout:, ssl:, persistent:)
     @monitor = Monitor.new
   end
 
@@ -47,7 +42,7 @@ class LLM::Provider
   # @return [String]
   # @note The secret key is redacted in inspect for security reasons
   def inspect
-    "#<#{self.class.name}:0x#{object_id.to_s(16)} @key=[REDACTED] @client=#{@client.inspect} @tracer=#{tracer.inspect}>"
+    "#<#{self.class.name}:0x#{object_id.to_s(16)} @key=[REDACTED] @transport=#{transport.inspect} @tracer=#{tracer.inspect}>"
   end
 
   ##
@@ -312,13 +307,20 @@ class LLM::Provider
   #   # do something with 'llm'
   # @return [LLM::Provider]
   def persist!
-    client = persistent_client
-    lock do
-      tap { @client = client }
-    end
+    transport.persist!
+    self
   end
   alias_method :persistent, :persist!
 
+  ##
+  # Interrupt the active request, if any.
+  # @param [Fiber] owner
+  # @return [nil]
+  def interrupt!(owner)
+    transport.interrupt!(owner)
+  end
+  alias_method :cancel!, :interrupt!
+
   ##
   # @param [Object] stream
   # @return [Boolean]
@@ -328,7 +330,7 @@ class LLM::Provider
 
   private
 
-  attr_reader :client, :base_uri, :host, :port, :timeout, :ssl
+  attr_reader :base_uri, :host, :port, :timeout, :ssl, :transport
 
   ##
   # The headers to include with a request
@@ -360,94 +362,6 @@ class LLM::Provider
     raise NotImplementedError
   end
 
-  ##
-  # Executes a HTTP request
-  # @param [Net::HTTPRequest] request
-  #  The request to send
-  # @param [Proc] b
-  #  A block to yield the response to (optional)
-  # @return [Net::HTTPResponse]
-  #  The response from the server
-  # @raise [LLM::Error::Unauthorized]
-  #  When authentication fails
-  # @raise [LLM::Error::RateLimit]
-  #  When the rate limit is exceeded
-  # @raise [LLM::Error]
-  #  When any other unsuccessful status code is returned
-  # @raise [SystemCallError]
-  #  When there is a network error at the operating system level
-  # @return [Net::HTTPResponse]
-  def execute(request:, operation:, stream: nil, stream_parser: self.stream_parser, model: nil, inputs: nil, &b)
-    tracer = self.tracer
-    span = tracer.on_request_start(operation:, model:, inputs:)
-    http = client || transient_client
-    args = (Net::HTTP === http) ? [request] : [URI.join(base_uri, request.path), request]
-    res = if stream
-      http.request(*args) do |res|
-        if Net::HTTPSuccess === res
-          handler = event_handler.new stream_parser.new(stream)
-          parser = LLM::EventStream::Parser.new
-          parser.register(handler)
-          res.read_body(parser)
-          # If the handler body is empty, the response was
-          # most likely not streamed or parsing failed.
-          # Preserve the raw body in that case so standard
-          # JSON/error handling can parse it later.
-          body = handler.body.empty? ? parser.body : handler.body
-          res.body = Hash === body || Array === body ? LLM::Object.from(body) : body
-        else
-          body = +""
-          res.read_body { body << _1 }
-          res.body = body
-        end
-      ensure
-        handler&.free
-        parser&.free
-      end
-    else
-      b ? http.request(*args) { (Net::HTTPSuccess === _1) ? b.call(_1) : _1 } :
-          http.request(*args)
-    end
-    [handle_response(res, tracer, span), span, tracer]
-  end
-
-  ##
-  # Handles the response from a request
-  # @param [Net::HTTPResponse] res
-  #  The response to handle
-  # @param [Object, nil] span
-  #  The span
-  # @return [Net::HTTPResponse]
-  def handle_response(res, tracer, span)
-    case res
-    when Net::HTTPOK then res.body = parse_response(res)
-    else error_handler.new(tracer, span, res).raise_error!
-    end
-    res
-  end
-
-  ##
-  # Parse a HTTP response
-  # @param [Net::HTTPResponse] res
-  # @return [LLM::Object, String]
-  def parse_response(res)
-    case res["content-type"]
-    when %r|\Aapplication/json\s*| then LLM::Object.from(LLM.json.load(res.body))
-    else res.body
-    end
-  end
-
-  ##
-  # @param [Net::HTTPRequest] req
-  #  The request to set the body stream for
-  # @param [IO] io
-  #  The IO object to set as the body stream
-  # @return [void]
-  def set_body_stream(req, io)
-    req.body_stream = io
-    req["transfer-encoding"] = "chunked" unless req["content-length"]
-  end
-
   ##
   # Resolves tools to their function representations
   # @param [Array<LLM::Function, LLM::Tool>] tools

data/lib/llm/providers/openai/request_adapter/respond.rb

@@ -15,6 +15,8 @@ module LLM::OpenAI::RequestAdapter
       catch(:abort) do
         if Hash === message
           {role: message[:role], content: adapt_content(message[:content])}
+        elsif message.tool_call?
+          message.extra[:original_tool_calls]
         else
           adapt_message
         end
@@ -23,12 +25,12 @@ module LLM::OpenAI::RequestAdapter
 
     private
 
-    def adapt_content(content)
+    def adapt_content(content, role: message.role)
       case content
       when String
-        [{type: :input_text, text: content.to_s}]
+        [{type: text_content_type(role), text: content.to_s}]
       when LLM::Response then adapt_remote_file(content)
-      when LLM::Message then adapt_content(content.content)
+      when LLM::Message then adapt_content(content.content, role: content.role)
       when LLM::Object
         case content.kind
         when :image_url then [{type: :image_url, image_url: {url: content.value.to_s}}]
@@ -46,7 +48,7 @@ module LLM::OpenAI::RequestAdapter
       when Array
         adapt_array
       else
-        {role: message.role, content: adapt_content(content)}
+        {role: message.role, content: adapt_content(content, role: message.role)}
       end
     end
 
@@ -56,7 +58,7 @@ module LLM::OpenAI::RequestAdapter
       elsif returns.any?
         returns.map { {type: "function_call_output", call_id: _1.id, output: LLM.json.dump(_1.value)} }
       else
-        {role: message.role, content: content.flat_map { adapt_content(_1) }}
+        {role: message.role, content: content.flat_map { adapt_content(_1, role: message.role) }}
       end
     end
 
@@ -83,5 +85,9 @@ module LLM::OpenAI::RequestAdapter
     def message = @message
     def content = message.content
     def returns = content.grep(LLM::Function::Return)
+
+    def text_content_type(role)
+      role.to_s == "assistant" ? :output_text : :input_text
+    end
   end
 end

data/lib/llm/providers/openai/response_adapter/responds.rb

@@ -60,6 +60,13 @@ module LLM::OpenAI::ResponseAdapter
       body.model
     end
 
+    ##
+    # OpenAI's Responses API does not expose a system fingerprint.
+    # @return [nil]
+    def system_fingerprint
+      nil
+    end
+
     ##
     # Returns the aggregated text content from the response outputs.
     # @return [String]
@@ -88,10 +95,15 @@ module LLM::OpenAI::ResponseAdapter
     private
 
     def adapt_message
-      message = LLM::Message.new("assistant", +"", {response: self, tool_calls: [], reasoning_content: +""})
+      message = LLM::Message.new(
+        "assistant",
+        +"",
+        {response: self, tool_calls: [], original_tool_calls: [], reasoning_content: +""}
+      )
       output.each do |choice|
         if choice.type == "function_call"
           message.extra[:tool_calls] << adapt_tool(choice)
+          message.extra[:original_tool_calls] << choice
         elsif choice.type == "reasoning"
           (choice.summary || []).each do |summary|
             next unless summary["type"] == "summary_text"

data/lib/llm/providers/openai/responses/stream_parser.rb

@@ -43,11 +43,19 @@ class LLM::OpenAI
           @body[k] = v
         end
         @body["output"] ||= []
+      when "response.in_progress", "response.completed"
+        response = chunk["response"] || {}
+        response.each do |k, v|
+          next if k == "output" && @body["output"].is_a?(Array) && @body["output"].any?
+          @body[k] = v
+        end
+        @body["output"] ||= response["output"] || []
       when "response.output_item.added"
         output_index = chunk["output_index"]
         item = chunk["item"]
         @body["output"][output_index] = item
         @body["output"][output_index]["content"] ||= []
+        @body["output"][output_index]["summary"] ||= [] if item["type"] == "reasoning"
       when "response.content_part.added"
         output_index = chunk["output_index"]
         content_index = chunk["content_index"]
@@ -55,6 +63,25 @@ class LLM::OpenAI
         @body["output"][output_index] ||= {"content" => []}
         @body["output"][output_index]["content"] ||= []
         @body["output"][output_index]["content"][content_index] = part
+      when "response.reasoning_summary_text.delta"
+        output_item = @body["output"][chunk["output_index"]]
+        if output_item && output_item["type"] == "reasoning"
+          summary_index = chunk["summary_index"] || 0
+          output_item["summary"] ||= []
+          output_item["summary"][summary_index] ||= {"type" => "summary_text", "text" => +""}
+          output_item["summary"][summary_index]["text"] << chunk["delta"]
+          emit_reasoning_content(chunk["delta"])
+        end
+      when "response.reasoning_summary_text.done"
+        output_item = @body["output"][chunk["output_index"]]
+        if output_item && output_item["type"] == "reasoning"
+          summary_index = chunk["summary_index"] || 0
+          output_item["summary"] ||= []
+          output_item["summary"][summary_index] = {
+            "type" => "summary_text",
+            "text" => chunk["text"]
+          }
+        end
       when "response.output_text.delta"
         output_index = chunk["output_index"]
         content_index = chunk["content_index"]
@@ -102,6 +129,10 @@ class LLM::OpenAI
       end
     end
 
+    def emit_reasoning_content(value)
+      @stream.on_reasoning_content(value) if @stream.respond_to?(:on_reasoning_content)
+    end
+
     def emit_tool(index, tool)
       return unless @stream.respond_to?(:on_tool_call)
       return unless complete_tool?(tool)

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "4.12.0"
+  VERSION = "4.14.0"
 end

data/lib/llm.rb

@@ -40,6 +40,14 @@ module LLM
   # Model registry
   @registry = {}
 
+  ##
+  # Shared HTTP clients used by providers.
+  @clients = {}
+
+  ##
+  # @api private
+  def self.clients = @clients
+
   ##
   # @param [Symbol, LLM::Provider] llm
   #  The name of a provider, or an instance of LLM::Provider