llm.rb 4.13.0 → 4.15.0

data/lib/llm/mcp/router.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class LLM::MCP
+  ##
+  # Coordinates shared access to a transport by routing JSON-RPC
+  # responses to the mailbox waiting on the matching request id.
+  class Router
+    def initialize
+      @request_id = -1
+      @pending = {}
+      @lock = Monitor.new
+      @writer = Monitor.new
+      @reader = Monitor.new
+    end
+
+    def register
+      @lock.synchronize do
+        @request_id += 1
+        mailbox = LLM::MCP::Mailbox.new
+        @pending[@request_id] = mailbox
+        [@request_id, mailbox]
+      end
+    end
+
+    def clear(id)
+      @lock.synchronize { @pending.delete(id) }
+    end
+
+    def read(transport)
+      @reader.synchronize { transport.read_nonblock }
+    end
+
+    def write(transport, message)
+      @writer.synchronize { transport.write(message) }
+    end
+
+    def route(response)
+      mailbox = @lock.synchronize { @pending[response["id"]] }
+      raise LLM::MCP::MismatchError.new(expected_id: nil, actual_id: response["id"]) unless mailbox
+      mailbox << response
+      nil
+    end
+  end
+end

data/lib/llm/mcp/rpc.rb

@@ -27,13 +27,15 @@ class LLM::MCP
     def call(transport, method, params = {})
       message = {jsonrpc: "2.0", method:, params: default_params(method).merge(params)}
       if notification?(method)
-        transport.write(message)
-        nil
-      else
-        @request_id = (@request_id || -1) + 1
-        id = @request_id
-        transport.write(message.merge(id:))
-        recv(transport, id)
+        router.write(transport, message)
+        return nil
+      end
+      id, mailbox = router.register
+      begin
+        router.write(transport, message.merge(id:))
+        recv(transport, id, mailbox)
+      ensure
+        router.clear(id)
       end
     end
 
@@ -49,19 +51,12 @@ class LLM::MCP
     #  When the MCP process returns an error
     # @return [Object, nil]
     #  The result returned by the MCP process
-    def recv(transport, id)
+    def recv(transport, id, mailbox)
       poll(timeout:, ex: [IO::WaitReadable]) do
         loop do
-          res = transport.read_nonblock
-          if res["id"] == id && res["error"]
-            raise LLM::MCP::Error.from(response: res)
-          elsif res["id"] == id
-            break res["result"]
-          elsif res["method"]
-            next
-          elsif res.key?("id")
-            raise LLM::MCP::MismatchError.new(expected_id: id, actual_id: res["id"])
-          end
+          res = mailbox.pop
+          return handle_response(id, res) if res
+          route_response(router.read(transport), id)
         end
       end
     end
@@ -119,5 +114,21 @@ class LLM::MCP
         sleep 0.05
       end
     end
+
+    def handle_response(id, res)
+      raise LLM::MCP::Error.from(response: res) if res["error"]
+      return res["result"] if res["id"] == id
+      raise LLM::MCP::MismatchError.new(expected_id: id, actual_id: res["id"])
+    end
+
+    def route_response(res, id)
+      return nil if res["method"]
+      return router.route(res) if res.key?("id")
+      raise LLM::MCP::MismatchError.new(expected_id: id, actual_id: nil)
+    end
+
+    def router
+      @router ||= LLM::MCP::Router.new
+    end
   end
 end

data/lib/llm/mcp/transport/http/event_handler.rb

@@ -21,29 +21,31 @@ module LLM::MCP::Transport
 
     ##
     # Receives the SSE event name.
-    # @param [LLM::EventStream::Event] event
+    # @param [LLM::EventStream::Event, String, nil] event
+    # @param [String, nil] chunk
     #  The event stream event
     # @return [void]
-    def on_event(event)
-      @event = event.value
+    def on_event(event, chunk = nil)
+      @event = chunk ? event : event.value
     end
 
     ##
     # Receives one line of SSE data.
-    # @param [LLM::EventStream::Event] event
+    # @param [LLM::EventStream::Event, String, nil] event
+    # @param [String, nil] chunk
     #  The event stream event
     # @return [void]
-    def on_data(event)
-      @data << event.value.to_s
+    def on_data(event, chunk = nil)
+      @data << (chunk ? event : event.value).to_s
     end
 
     # The generic event stream parser dispatches one line at a time.
     # A blank line terminates the current SSE event.
-    # @param [LLM::EventStream::Event] event
+    # @param [LLM::EventStream::Event, String] event
     #  The event stream event
     # @return [void]
-    def on_chunk(event)
-      flush if event.chunk == "\n"
+    def on_chunk(event, chunk = nil)
+      flush if (chunk || event&.chunk || event) == "\n"
     end
 
     private

data/lib/llm/mcp/transport/http.rb

@@ -82,13 +82,13 @@ module LLM::MCP::Transport
     # Reads the next queued message without blocking.
     # @raise [LLM::MCP::Error]
     #  When the transport is not running
-    # @raise [IO::WaitReadable]
+    # @raise [IO::EAGAINWaitReadable]
     #  When no complete message is available to read
     # @return [Hash]
     def read_nonblock
       lock do
         raise LLM::MCP::Error, "MCP transport is not running" unless running?
-        raise IO::WaitReadable if @queue.empty?
+        raise IO::EAGAINWaitReadable, "no complete message available" if @queue.empty?
         @queue.shift
       end
     end

data/lib/llm/mcp/transport/stdio.rb

@@ -57,7 +57,7 @@ module LLM::MCP::Transport
     # Reads a message from the MCP process without blocking.
     # @raise [LLM::Error]
     #  When the transport is not running
-    # @raise [IO::WaitReadable]
+    # @raise [IO::EAGAINWaitReadable]
     #  When no complete message is available to read
     # @return [Hash]
     #  The next message from the MCP process

data/lib/llm/mcp.rb

@@ -10,11 +10,14 @@
 # transports and focuses on discovering tools that can be used through
 # {LLM::Context LLM::Context} and {LLM::Agent LLM::Agent}.
 #
-# Like {LLM::Context LLM::Context}, an MCP client is stateful and is
-# expected to remain isolated to a single thread.
+# An MCP client is stateful. Coordinate lifecycle operations such as
+# {#start} and {#stop}; request methods can be issued concurrently and
+# responses are matched by JSON-RPC id.
 class LLM::MCP
   require_relative "mcp/error"
   require_relative "mcp/command"
+  require_relative "mcp/mailbox"
+  require_relative "mcp/router"
   require_relative "mcp/rpc"
   require_relative "mcp/pipe"
   require_relative "mcp/transport/http"

data/lib/llm/provider/transport/http/execution.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module LLM::Provider::Transport
+  class HTTP
+    ##
+    # Internal HTTP request execution methods for {LLM::Provider}.
+    #
+    # This module handles provider-side HTTP execution, response parsing,
+    # streaming, and request body setup through
+    # {LLM::Provider::Transport::HTTP}.
+    #
+    # @api private
+    module HTTP::Execution
+      private
+
+      ##
+      # Executes a HTTP request
+      # @param [Net::HTTPRequest] request
+      #  The request to send
+      # @param [Proc] b
+      #  A block to yield the response to (optional)
+      # @return [Net::HTTPResponse]
+      #  The response from the server
+      # @raise [LLM::Error::Unauthorized]
+      #  When authentication fails
+      # @raise [LLM::Error::RateLimit]
+      #  When the rate limit is exceeded
+      # @raise [LLM::Error]
+      #  When any other unsuccessful status code is returned
+      # @raise [SystemCallError]
+      #  When there is a network error at the operating system level
+      # @return [Net::HTTPResponse]
+      def execute(request:, operation:, stream: nil, stream_parser: self.stream_parser, model: nil, inputs: nil, &b)
+        owner = transport.request_owner
+        tracer = self.tracer
+        span = tracer.on_request_start(operation:, model:, inputs:)
+        res = transport.request(request, owner:) do |http|
+          perform_request(http, request, stream, stream_parser, &b)
+        end
+        [handle_response(res, tracer, span), span, tracer]
+      rescue *LLM::Provider::Transport::HTTP::Interruptible::INTERRUPT_ERRORS
+        raise LLM::Interrupt, "request interrupted" if transport.interrupted?(owner)
+        raise
+      end
+
+      ##
+      # Handles the response from a request
+      # @param [Net::HTTPResponse] res
+      #  The response to handle
+      # @param [Object, nil] span
+      #  The span
+      # @return [Net::HTTPResponse]
+      def handle_response(res, tracer, span)
+        case res
+        when Net::HTTPOK then res.body = parse_response(res)
+        else error_handler.new(tracer, span, res).raise_error!
+        end
+        res
+      end
+
+      ##
+      # Parse a HTTP response
+      # @param [Net::HTTPResponse] res
+      # @return [LLM::Object, String]
+      def parse_response(res)
+        case res["content-type"]
+        when %r{\Aapplication/json\s*} then LLM::Object.from(LLM.json.load(res.body))
+        else res.body
+        end
+      end
+
+      ##
+      # @param [Net::HTTPRequest] req
+      #  The request to set the body stream for
+      # @param [IO] io
+      #  The IO object to set as the body stream
+      # @return [void]
+      def set_body_stream(req, io)
+        req.body_stream = io
+        req["transfer-encoding"] = "chunked" unless req["content-length"]
+      end
+
+      ##
+      # Performs the request on the given HTTP connection.
+      # @param [Net::HTTP] http
+      # @param [Net::HTTPRequest] request
+      # @param [Object, nil] stream
+      # @param [Class] stream_parser
+      # @param [Proc, nil] b
+      # @return [Net::HTTPResponse]
+      def perform_request(http, request, stream, stream_parser, &b)
+        if stream
+          http.request(request) do |res|
+            if Net::HTTPSuccess === res
+              parser = StreamDecoder.new(stream_parser.new(stream))
+              res.read_body(parser)
+              body = parser.body
+              res.body = (Hash === body || Array === body) ? LLM::Object.from(body) : body
+            else
+              body = +""
+              res.read_body { body << _1 }
+              res.body = body
+            end
+          ensure
+            parser&.free
+          end
+        elsif b
+          http.request(request) { (Net::HTTPSuccess === _1) ? b.call(_1) : _1 }
+        else
+          http.request(request)
+        end
+      end
+    end
+  end
+end

data/lib/llm/provider/transport/http/interruptible.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+class LLM::Provider
+  module Transport
+    class HTTP
+      ##
+      # Internal request interruption methods for
+      # {LLM::Provider::Transport::HTTP}.
+      #
+      # This module tracks active requests by execution owner and provides
+      # the logic used to interrupt an in-flight request by closing the
+      # active HTTP connection.
+      #
+      # @api private
+      module Interruptible
+        INTERRUPT_ERRORS = [::IOError, ::EOFError, Errno::EBADF].freeze
+        Request = Struct.new(:http, :connection, keyword_init: true)
+
+        ##
+        # Interrupt an active request, if any.
+        # @param [Fiber] owner
+        #  The execution owner whose request should be interrupted
+        # @return [nil]
+        def interrupt!(owner)
+          req = request_for(owner) or return
+          lock { (@interrupts ||= {})[owner] = true }
+          if persistent_http?(req.http)
+            close_socket(req.connection&.http)
+            req.http.finish(req.connection)
+          elsif transient_http?(req.http)
+            close_socket(req.http)
+            req.http.finish if req.http.active?
+          end
+        rescue *INTERRUPT_ERRORS
+          nil
+        end
+
+        private
+
+        ##
+        # Closes the active socket for a request, if present.
+        # @param [Net::HTTP, nil] http
+        # @return [nil]
+        def close_socket(http)
+          socket = http&.instance_variable_get(:@socket) or return
+          socket = socket.io if socket.respond_to?(:io)
+          socket.close
+        rescue *INTERRUPT_ERRORS
+          nil
+        end
+
+        ##
+        # Returns whether the active request is using a transient HTTP client.
+        # @param [Object, nil] http
+        # @return [Boolean]
+        def transient_http?(http)
+          Net::HTTP === http
+        end
+
+        ##
+        # Returns whether the active request is using a persistent HTTP client.
+        # @param [Object, nil] http
+        # @return [Boolean]
+        def persistent_http?(http)
+          defined?(Net::HTTP::Persistent) && Net::HTTP::Persistent === http
+        end
+
+        ##
+        # Returns the active request for an execution owner.
+        # @param [Fiber] owner
+        # @return [Request, nil]
+        def request_for(owner)
+          lock do
+            @requests ||= {}
+            @requests[owner]
+          end
+        end
+
+        ##
+        # Records an active request for an execution owner.
+        # @param [Request] req
+        # @param [Fiber] owner
+        # @return [Request]
+        def set_request(req, owner)
+          lock do
+            @requests ||= {}
+            @requests[owner] = req
+          end
+        end
+
+        ##
+        # Clears the active request for an execution owner.
+        # @param [Fiber] owner
+        # @return [Request, nil]
+        def clear_request(owner)
+          lock { @requests&.delete(owner) }
+        end
+
+        ##
+        # Returns whether an execution owner was interrupted.
+        # @param [Fiber] owner
+        # @return [Boolean, nil]
+        def interrupted?(owner)
+          lock { @interrupts&.delete(owner) }
+        end
+      end
+    end
+  end
+end

data/lib/llm/provider/transport/http/stream_decoder.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module LLM::Provider::Transport
+  ##
+  # @private
+  class HTTP::StreamDecoder
+    ##
+    # @return [Object]
+    attr_reader :parser
+
+    ##
+    # @param [#parse!, #body] parser
+    # @return [LLM::Provider::Transport::HTTP::StreamDecoder]
+    def initialize(parser)
+      @buffer = +""
+      @cursor = 0
+      @data = []
+      @parser = parser
+    end
+
+    ##
+    # @param [String] chunk
+    # @return [void]
+    def <<(chunk)
+      @buffer << chunk
+      each_line { handle_line(_1) }
+    end
+
+    ##
+    # @return [Object]
+    def body
+      parser.body
+    end
+
+    ##
+    # @return [void]
+    def free
+      @buffer.clear
+      @cursor = 0
+      @data.clear
+      parser.free if parser.respond_to?(:free)
+    end
+
+    private
+
+    def handle_line(line)
+      if line == "\n" || line == "\r\n"
+        flush_sse_event
+      elsif line.start_with?("data:")
+        @data << field_value(line)
+      elsif line.start_with?("event:", "id:", "retry:", ":")
+      else
+        decode!(strip_newline(line))
+      end
+    end
+
+    def flush_sse_event
+      return if @data.empty?
+      decode!(@data.join("\n"))
+      @data.clear
+    end
+
+    def field_value(line)
+      value_start = line.getbyte(5) == 32 ? 6 : 5
+      strip_newline(line.byteslice(value_start..))
+    end
+
+    def strip_newline(line)
+      line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\n")
+      line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\r")
+      line
+    end
+
+    def decode!(payload)
+      return if payload.empty? || payload == "[DONE]"
+      chunk = LLM.json.load(payload)
+      parser.parse!(chunk) if chunk
+    rescue *LLM.json.parser_error
+    end
+
+    def each_line
+      while (newline = @buffer.index("\n", @cursor))
+        line = @buffer[@cursor..newline]
+        @cursor = newline + 1
+        yield(line)
+      end
+      return if @cursor.zero?
+      @buffer = @buffer[@cursor..] || +""
+      @cursor = 0
+    end
+  end
+end

data/lib/llm/provider/transport/http.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+class LLM::Provider
+  module Transport
+    ##
+    # The {LLM::Provider::Transport::HTTP LLM::Provider::Transport::HTTP}
+    # class manages HTTP connections for {LLM::Provider}. It handles
+    # transient and persistent clients, tracks active requests by owner,
+    # and interrupts in-flight requests when needed.
+    #
+    # @api private
+    class HTTP
+      require_relative "http/stream_decoder"
+      require_relative "http/interruptible"
+
+      include Interruptible
+
+      ##
+      # @param [String] host
+      # @param [Integer] port
+      # @param [Integer] timeout
+      # @param [Boolean] ssl
+      # @param [Boolean] persistent
+      # @return [LLM::Provider::Transport::HTTP]
+      def initialize(host:, port:, timeout:, ssl:, persistent: false)
+        @host = host
+        @port = port
+        @timeout = timeout
+        @ssl = ssl
+        @base_uri = URI("#{ssl ? "https" : "http"}://#{host}:#{port}/")
+        @persistent_client = persistent ? persistent_client : nil
+        @monitor = Monitor.new
+      end
+
+      ##
+      # Interrupt an active request, if any.
+      # @param [Fiber] owner
+      # @return [nil]
+      def interrupt!(owner)
+        super
+      end
+
+      ##
+      # Returns whether an execution owner was interrupted.
+      # @param [Fiber] owner
+      # @return [Boolean, nil]
+      def interrupted?(owner)
+        super
+      end
+
+      ##
+      # Returns the current request owner.
+      # @return [Fiber]
+      def request_owner
+        Fiber.current
+      end
+
+      ##
+      # Configures the transport to use a persistent HTTP connection pool.
+      # @return [LLM::Provider::Transport::HTTP]
+      def persist!
+        client = persistent_client
+        lock do
+          @persistent_client = client
+          self
+        end
+      end
+      alias_method :persistent, :persist!
+
+      ##
+      # @return [Boolean]
+      def persistent?
+        !persistent_client.nil?
+      end
+
+      ##
+      # Performs a request on the current HTTP transport.
+      # @param [Net::HTTPRequest] request
+      # @param [Fiber] owner
+      # @yieldparam [Net::HTTP] http
+      # @return [Object]
+      def request(request, owner:, &)
+        if persistent?
+          request_persistent(request, owner, &)
+        else
+          request_transient(request, owner, &)
+        end
+      ensure
+        clear_request(owner)
+      end
+
+      ##
+      # @return [String]
+      def inspect
+        "#<#{self.class.name}:0x#{object_id.to_s(16)} @persistent=#{persistent?}>"
+      end
+
+      private
+
+      attr_reader :host, :port, :timeout, :ssl, :base_uri
+
+      def request_transient(request, owner, &)
+        http = transient_client
+        set_request(Request.new(http:), owner)
+        yield http
+      end
+
+      def request_persistent(request, owner, &)
+        persistent_client.connection_for(URI.join(base_uri, request.path)) do |connection|
+          set_request(Request.new(http: persistent_client, connection:), owner)
+          yield connection.http
+        end
+      end
+
+      def persistent_client
+        LLM.lock(:clients) do
+          if LLM.clients[client_id]
+            LLM.clients[client_id]
+          else
+            require "net/http/persistent" unless defined?(Net::HTTP::Persistent)
+            client = Net::HTTP::Persistent.new(name: self.class.name)
+            client.read_timeout = timeout
+            LLM.clients[client_id] = client
+          end
+        end
+      end
+
+      def transient_client
+        client = Net::HTTP.new(host, port)
+        client.read_timeout = timeout
+        client.use_ssl = ssl
+        client
+      end
+
+      def client_id
+        "#{host}:#{port}:#{timeout}:#{ssl}"
+      end
+
+      def lock(&)
+        @monitor.synchronize(&)
+      end
+    end
+  end
+end