restify 3.0.0.rc2 → 3.0.0.rc3

data/lib/restify/adapter/ethon.rb

@@ -3,6 +3,11 @@
 require 'ethon'
 require 'nio'
 
+require 'restify/adapter/ethon/options'
+require 'restify/adapter/ethon/easy'
+require 'restify/adapter/ethon/event_loop'
+require 'restify/adapter/ethon/pool'
+
 module Restify
   module Adapter
     #
@@ -27,6 +32,16 @@ module Restify
     class Ethon < Base
       include Logging
 
+      # Restify follows relations from URLs in server responses, and
+      # libcurl supports far more than HTTP, e.g. `file://` or `scp://`.
+      # Therefore only HTTP-like protocols must ever be used, both for
+      # the request itself and when following redirects.
+      #
+      # This is intentionally not configurable.
+      PROTOCOLS = %i[http https].freeze
+
+      # Set headers bypassing the wait for 100-continue responses many
+      # servers do not send correctly.
       DEFAULT_HEADERS = {
         'Expect' => '',
         'Transfer-Encoding' => '',
@@ -39,38 +54,31 @@ module Restify
         tcp_keepintvl: 5,
       }.freeze
 
-      # Restify follows relations from URLs in server responses, and
-      # libcurl supports far more than HTTP, e.g. `file://` or `scp://`.
-      # Therefore only HTTP-like protocols must ever be used, both for
-      # the request itself and when following redirects.
-      #
-      # This is intentionally not configurable.
-      PROTOCOLS = %i[http https].freeze
+      ENFORCED_OPTIONS = {
+        # Do not use signal handlers in libcurl, as it would interfere
+        # with the threads.
+        nosignal: true,
 
-      INTERESTS = {
-        none: nil,
-        in: :r,
-        out: :w,
-        inout: :rw,
+        # Do not allow using and redirecting to unexpected (e.g. local)
+        # protocols.
+        protocols: PROTOCOLS,
+        redir_protocols: PROTOCOLS,
       }.freeze
 
+      # Maximum number of idle easy handles kept for reuse.
+      POOL_SIZE = 64
+
       def initialize(options: {}, **multi)
-        @options = DEFAULT_OPTIONS.merge(options)
+        @options = Options.new({
+          **DEFAULT_OPTIONS,
+          **options,
+          **ENFORCED_OPTIONS,
+        })
         @multi_options = multi
 
-        # libcurl only stores the function pointers, therefore the procs
-        # must be referenced here too, or they would be garbage
-        # collected.
-        @socketfunction = method(:on_socket).to_proc
-        @timerfunction  = method(:on_timer).to_proc
-
         @mutex  = Mutex.new
         @thread = nil
 
-        # Wake up the background thread from `select` when a waiting
-        # thread wants to run the loop.
-        @loop = LoopLock.new { @selector.wakeup }
-
         setup
 
         super()
@@ -87,22 +95,16 @@ module Restify
           url: request.uri,
           timeout: request.timeout
 
-        @queue << easy
-
         # Ensure the event loop is running and let it pick up the new
         # request.
         thread
-        @selector.wakeup
+
+        @events.enqueue(easy)
       end
 
       # Run the event loop in the calling thread until the promise is
-      # complete or the timeout expires, see `Promise#wait`.
-      #
-      # Several threads can wait at the same time, but only one runs the
-      # loop. The others sleep until either their promise completes or
-      # the loop is released. The background thread keeps processing
-      # requests nobody waits on, but releases the loop as soon as a
-      # thread starts waiting.
+      # complete or the timeout expires, see `Promise#wait` and
+      # `EventLoop#drive`.
       #
       # This only processes transfers, i.e. fulfills or rejects the
       # adapter's promises. Callbacks chained with `Promise#then` still
@@ -111,22 +113,10 @@ module Restify
       # Returns false when the loop cannot be run in the calling thread,
       # i.e. when waiting from within the loop itself.
       #
-      def drive(promise, timeout) # rubocop:disable Naming/PredicateMethod
+      def drive(promise, timeout)
         check_fork!
 
-        return false if @loop.owned?
-
-        while @loop.acquire(timeout) { promise.complete? }
-          begin
-            step(timeout.remaining) until promise.complete? || !timeout.remaining.positive?
-          rescue StandardError => e
-            error(e)
-          ensure
-            @loop.release
-          end
-        end
-
-        true
+        @events.drive(promise, timeout)
       end
 
       private
@@ -136,15 +126,9 @@ module Restify
       end
 
       def setup
-        @pid      = Process.pid
-        @selector = NIO::Selector.new
-        @monitors = {}
-        @timer    = nil
-        @queue    = Queue.new
-
-        @multi = ::Ethon::Multi.new(execution_mode: :socket_action, **@multi_options)
-        @multi.socketfunction = @socketfunction
-        @multi.timerfunction  = @timerfunction
+        @pid    = Process.pid
+        @pool   = Pool.new(size: POOL_SIZE) { Easy.new }
+        @events = EventLoop.new(**@multi_options)
       end
 
       def check_fork!
@@ -160,91 +144,43 @@ module Restify
       # Therefore, abandon the state without releasing it, and set up
       # everything again.
       def forked!
-        abandoned = @multi.easy_handles.dup
-        abandoned << @queue.pop(true) until @queue.empty?
-
-        @multi.handle.autorelease = false
-        abandoned.each {|easy| easy.handle.autorelease = false }
+        @events.forked!
+        @pool.forked!
 
         @thread = nil
-        @loop.reset!
         setup
-
-        abandoned.each do |easy|
-          easy._restify_writer.reject(
-            Restify::NetworkError.new(easy._restify_request, 'Request started before fork'),
-          )
-        end
       end
 
       def convert(request, writer)
-        Easy.new.tap do |easy|
+        @pool.checkout.tap do |easy|
           easy._otel_span = OpenTelemetry::Trace.current_span
           easy._restify_request = request
           easy._restify_writer = writer
 
-          easy.http_request(
-            request.uri.to_s,
-            request.method,
-            request_options(request),
-          )
+          easy.prepare(request, @options)
 
           easy.on_complete do |completed|
             complete(completed, request, writer)
 
+            @pool.release(completed)
+
             # Wake up threads waiting on the loop to check if their
             # result is available now.
-            @loop.notify
+            @events.notify
           end
         end
       end
 
-      def request_options(request)
-        # libcurl requires millisecond-based timeouts so fractional
-        # seconds must be converted and rounded.
-        timeout = request.timeout && (request.timeout * 1000).ceil
-
-        @options.merge(
-          headers: DEFAULT_HEADERS.merge(request.headers),
-          body: request.body,
-          timeout_ms: timeout,
-          connecttimeout_ms: timeout,
-
-          # Merged last, so that neither can be overridden via `options`.
-          protocols: PROTOCOLS,
-          redir_protocols: PROTOCOLS,
-
-          # Do not use signal handlers in libcurl, as it would interfere
-          # with the threads.
-          nosignal: true,
-        )
-      end
-
       def complete(easy, request, writer)
         writer.set do
-          code   = easy.return_code
-          status = easy.response_code
-
-          debug 'request:complete',
-            tag: request.object_id,
-            status: status,
-            message: code
-
-          if code != :ok
-            raise Restify::NetworkError.new(
-              request,
-              ::Ethon::Curl.easy_strerror(code),
-            )
+          if logger&.debug?
+            debug 'request:complete',
+              tag: request.object_id,
+              status: easy.response_code,
+              message: easy.return_code
           end
 
-          if status.nil? || status.zero?
-            raise Restify::NetworkError.new(
-              request,
-              'Response without HTTP status',
-            )
-          end
-
-          convert_back(easy, request)
+          easy.response(request)
         end
       rescue StandardError => e
         # This runs inside a libcurl callback, therefore no exception
@@ -253,210 +189,23 @@ module Restify
         error(e)
       end
 
-      def convert_back(easy, request)
-        ::Restify::Response.new(
-          request,
-          effective_uri(easy, request),
-          easy.response_code,
-          convert_headers(easy.response_headers),
-          easy.response_body,
-        )
-      end
-
-      def effective_uri(easy, request)
-        url = easy.effective_url
-        url ? Addressable::URI.parse(url) : request.uri
-      end
-
-      def convert_headers(raw)
-        headers = {}
-        return headers if raw.nil?
-
-        # The raw headers can contain multiple blocks, e.g. from
-        # informational responses or when following redirects; only use
-        # the latest ones:
-        block = raw.split(/\r?\n\r?\n/).reject {|b| b.strip.empty? }.last
-
-        # Split on newlines that are not followed by whitespace to keep
-        # folded header values together.
-        block.to_s.split(/\r?\n(?!\s)/).each do |line|
-          line = line.strip
-          next if line.empty? || line.start_with?('HTTP/')
-
-          key, value = line.split(':', 2)
-          next if value.nil?
-
-          key = key.strip.upcase.tr('-', '_')
-          value = value.strip.gsub(/\r?\n\s*/, ' ')
-
-          case (current = headers[key])
-            when nil then headers[key] = value
-            when Array then current << value
-            else headers[key] = [current, value]
-          end
-        end
-
-        headers
-      end
-
       def thread
         @mutex.synchronize do
           # Spawn thread if not yet started, or recreate it if it died.
           if @thread.nil? || !@thread.status
             debug 'loop:spawn'
-            @thread = Thread.new { run }
-          end
-
-          @thread
-        end
-      end
-
-      def run
-        loop do
-          @loop.acquire_background
 
-          begin
-            step
-          ensure
-            @loop.release
+            events = @events
+            @thread = Thread.new { events.run }
           end
-        rescue StandardError => e
-          error(e)
-        end
-      ensure
-        debug 'loop:exit'
-      end
-
-      # Run one iteration of the event loop. Must only be called by the
-      # thread owning the loop.
-      def step(limit = nil)
-        dequeue_all
-
-        # libcurl needs to be notified about its own timeouts, e.g. to
-        # start newly added transfers or to time out stalled ones.
-        timeout!
-
-        timeout = select_timeout
-        timeout = limit if limit && (timeout.nil? || timeout > limit)
-        debug 'loop:select', timeout: timeout
-
-        # nil on timeout; empty array when woken up
-        @selector.select(timeout)&.each do |monitor|
-          # libcurl can remove sockets while earlier events of the same
-          # batch are processed, e.g. when a completed transfer tears
-          # down other connections. Skip monitors that are gone by now.
-          next unless @monitors[monitor.value].equal?(monitor)
-
-          socket_action(monitor.value, readiness(monitor))
-        end
-      end
-
-      def dequeue_all
-        loop do
-          easy = begin
-            @queue.pop(true)
-          rescue ThreadError
-            break
-          end
-
-          add(easy)
-        end
-      end
-
-      def add(easy)
-        # Restore the OpenTelemetry span the request originated from, so
-        # that the Ethon instrumentation picks up the correct parent
-        # when the easy handle is added to libcurl.
-        #
-        # Handle exceptions from Ethon or WebMock too, and reject the
-        # promise, so that the errors can be handled in user code.
-        OpenTelemetry::Trace.with_span(easy._otel_span) do
-          @multi.add(easy)
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          easy._restify_writer.reject(e)
-        end
-      end
-
-      # Seconds until libcurl wants to be called again, or nil to block
-      # until a socket becomes ready or the loop is woken up.
-      def select_timeout
-        return nil unless @timer
-
-        [@timer - now, 0].max
-      end
-
-      def timeout!
-        return unless @timer
-        return if now < @timer
 
-        # Reset before invoking libcurl, as it will set a new timeout
-        # from within the socket action.
-        @timer = nil
-
-        socket_action
-      end
-
-      def readiness(monitor)
-        readiness = []
-        readiness << :in if monitor.readable?
-        readiness << :out if monitor.writable?
-        readiness
-      end
-
-      def socket_action(socket = nil, readiness = 0)
-        @multi.socket_action(socket, readiness)
-      end
-
-      # libcurl: called when asking for readiness monitoring
-      def on_socket(_easy, socket, what, _userp, _socketp)
-        debug 'socket:action', tag: socket, what: what
-
-        if what == :remove
-          @monitors.delete(socket)&.close
-        else
-          monitor = (@monitors[socket] ||= register(socket))
-          monitor.interests = INTERESTS.fetch(what)
-        end
-
-        :ok
-      rescue StandardError => e
-        error(e)
-        :ok
-      end
-
-      # libcurl: called on timeout changes
-      def on_timer(_handle, timeout_ms, _userp)
-        debug 'timer:set', timeout: timeout_ms
-        @timer = timeout_ms.negative? ? nil : now + (timeout_ms / 1000.0)
-
-        :ok
-      rescue StandardError => e
-        error(e)
-        :ok
-      end
-
-      def register(socket)
-        # FD owned by libcurl: IO/ruby must never close it
-        io = IO.for_fd(socket, autoclose: false)
-
-        @selector.register(io, :r).tap do |monitor|
-          monitor.value = socket
+          @thread
         end
       end
 
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-
       def _log_prefix
         "[#{object_id}/#{Thread.current.object_id}]"
       end
-
-      # Keep track of the OTEL span, the request and the promise writer to
-      # reject on errors in the background thread or after fork
-      class Easy < ::Ethon::Easy
-        attr_accessor :_otel_span, :_restify_request, :_restify_writer
-      end
     end
   end
 end

data/lib/restify/adapter/telemetry.rb

@@ -26,7 +26,7 @@ module Restify
           OpenTelemetry.propagation.inject(request.headers)
 
           super.tap do |x|
-            x.add_observer do |_, response, err|
+            x.add_observer do |response, err|
               if response
                 span.set_attribute('http.response.status_code', response&.code)
                 span.status = OpenTelemetry::Trace::Status.error unless (100..399).cover?(response&.code)

data/lib/restify/adapter/typhoeus.rb

@@ -144,7 +144,11 @@ module Restify
 
       def effective_uri(response, request)
         url = response.effective_url
-        url ? Addressable::URI.parse(url) : request.uri
+
+        # Only parse the URL after redirects, as parsing is expensive.
+        return request.uri if url.nil? || url == request.uri.to_s
+
+        Addressable::URI.parse(url)
       end
 
       def convert_headers(headers)

data/lib/restify/context.rb

@@ -30,8 +30,18 @@ module Restify
       @options = kwargs
     end
 
+    # Resolve a URI against the context URI. Absolute URIs are returned
+    # as they are, e.g. when already joined by a relation.
+    #
+    # @param uri [Addressable::URI, String] URI to resolve.
+    # @return [Addressable::URI] Absolute URI.
+    #
     def join(uri)
-      self.uri.join uri
+      if uri.is_a?(Addressable::URI) && uri.absolute?
+        uri
+      else
+        self.uri.join(uri)
+      end
     end
 
     def inherit(uri, **)

data/lib/restify/processors/base/parsing.rb

@@ -9,6 +9,8 @@ module Restify
       # Parses generic data structures into resources
       #
       module Parsing
+        RELATION_NAME = /\A\w+_url\z/i
+
         def load
           # No data, e.g. when a server sets a content type but sends an
           # empty body.
@@ -27,41 +29,44 @@ module Restify
 
         def parse(object, root: false)
           case object
-            when Hash
-              data      = object.each_with_object({}) {|each, obj| parse_data(each, obj) }
-              relations = object.each_with_object({}) {|each, obj| parse_rels(each, obj) }
-
-              Resource.new context,
-                data:,
-                response: root ? response : nil,
-                relations:
-
-            when Array
-              object.map {|each| parse(each) }
-            else
-              object
+            when Hash then build_resource(object, root)
+            when Array then object.map {|each| parse(each) }
+            else object
           end
         end
 
         private
 
-        def parse_data(pair, data)
-          data[pair[0].to_s] = parse pair[1]
+        # Build data and relations of a resource in one pass.
+        def build_resource(object, root)
+          data = {}
+          relations = {}
+
+          object.each_pair do |key, value|
+            key = key.to_s
+            data[key] = parse(value)
+            parse_relation(relations, key, value) if value.is_a?(String)
+          end
+
+          Resource.new(
+            context,
+            data:,
+            response: root ? response : nil,
+            relations:,
+          )
         end
 
-        def parse_rels(pair, relations)
-          name = case pair[0].to_s.downcase
-                   when /\A(\w+)_url\z/
-                     Regexp.last_match[1]
-                   when 'url'
-                     'self'
-                   else
-                     return
+        def parse_relation(relations, key, value)
+          name = if key.match?(RELATION_NAME)
+                   key[0, key.length - 4].downcase
+                 elsif key.casecmp?('url')
+                   'self'
                  end
 
-          return if relations.key?(name) || pair[1].nil? || pair[1].to_s =~ /\A\w*\z/
+          return if name.nil? || relations.key?(name)
+          return if value.empty?
 
-          relations[name] = pair[1].to_s
+          relations[name] = value
         end
       end
     end