httpx 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52b8402031b6deeed1d1a5c92c56e1cc8d93cae1da650d3573ddca712a413948
-  data.tar.gz: b2c2528c1a187eb0cdfe72636eebf6a390a240e86b78e9ebed2b81f26f3f1c89
+  metadata.gz: 853641fa479bca4da2dca4d69bf8eddf6b323b12c40096cdb22faff1a8f80176
+  data.tar.gz: 8f74a2b53e2037863b9b66e4184e568f7de33bab9f6a65f27b9b9d8f6f3b145b
 SHA512:
-  metadata.gz: '0837f66f56ca695dbc89e9d1557703dba32d74c0748c246f3d94b5fa4baaf13b50a931c8dd018a8c6d4ec80324aac7c164c271be5d9a50afe573c7316e6bf751'
-  data.tar.gz: ee65a86527d020d422e68d1b5972f11d7ce710267ae4ba31d6bd7d42f6966fa606ddfdf9d4a788175516d2c7a5a857000ab0980d5762667236e8e9e2ef13ce12
+  metadata.gz: 736eb3d84089ac6d98576f6aeb1b20f3cb56cf32bc070c999fd41c0b1685612b89cf397884053e8d103c9464c1998f2651b9a219692b36dee9848a051f13d5d4
+  data.tar.gz: 6c17fbd06732769f09689ccc2f10e0f5fcb8e897ac6c32aa6e96ef707b2bcc26577b4dae8a8b3486e35495e002e833e06593421f99baa26f3c2586e651aaf393

data/lib/httpx.rb

@@ -12,12 +12,12 @@ require "httpx/registry"
 require "httpx/transcoder"
 require "httpx/options"
 require "httpx/timeout"
-require "httpx/connection"
+require "httpx/pool"
 require "httpx/headers"
 require "httpx/request"
 require "httpx/response"
 require "httpx/chainable"
-require "httpx/client"
+require "httpx/session"
 
 # Top-Level Namespace
 #
@@ -47,5 +47,11 @@ module HTTPX
     end
   end
 
+  def self.const_missing(const_name)
+    super unless const_name == :Client
+    warn "DEPRECATION WARNING: the class #{self}::Client is deprecated. Use #{self}::Session instead."
+    Session
+  end
+
   extend Chainable
 end

data/lib/httpx/adapters/faraday.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "httpx"
+require "faraday"
+
+module Faraday
+  class Adapter
+    class HTTPX < Faraday::Adapter
+      module RequestMixin
+        private
+
+        def build_request(env)
+          meth = env[:method]
+
+          request_options = {
+            headers: env.request_headers,
+            body: env.body,
+          }
+          [meth, env.url, request_options]
+        end
+      end
+
+      include RequestMixin
+
+      class Session < ::HTTPX::Session
+        plugin(:compression)
+        plugin(:persistent)
+
+        module ReasonPlugin
+          if RUBY_VERSION < "2.5"
+            def self.load_dependencies(*)
+              require "webrick"
+            end
+          else
+            def self.load_dependencies(*)
+              require "net/http/status"
+            end
+          end
+          module ResponseMethods
+            if RUBY_VERSION < "2.5"
+              def reason
+                WEBrick::HTTPStatus::StatusMessage.fetch(@status)
+              end
+            else
+              def reason
+                Net::HTTP::STATUS_CODES.fetch(@status)
+              end
+            end
+          end
+        end
+        plugin(ReasonPlugin)
+      end
+
+      class ParallelManager
+        class ResponseHandler
+          attr_reader :env
+
+          def initialize(env)
+            @env = env
+          end
+
+          def on_response(&blk)
+            if block_given?
+              @on_response = lambda do |response|
+                blk.call(response)
+              end
+              self
+            else
+              @on_response
+            end
+          end
+
+          def on_complete(&blk)
+            if block_given?
+              @on_complete = blk
+              self
+            else
+              @on_complete
+            end
+          end
+
+          def respond_to_missing?(meth)
+            @env.respond_to?(meth)
+          end
+
+          def method_missing(meth, *args, &blk)
+            if @env && @env.respond_to?(meth)
+              @env.__send__(meth, *args, &blk)
+            else
+              super
+            end
+          end
+        end
+
+        include RequestMixin
+
+        def initialize
+          @session = Session.new
+          @handlers = []
+        end
+
+        def enqueue(request)
+          handler = ResponseHandler.new(request)
+          @handlers << handler
+          handler
+        end
+
+        def run
+          requests = @handlers.map { |handler| build_request(handler.env) }
+          env = @handlers.last.env
+
+          timeout_options = {
+            connect_timeout: env.request.open_timeout,
+            operation_timeout: env.request.timeout,
+          }.reject { |_, v| v.nil? }
+
+          options = {
+            ssl: env.ssl,
+            timeout: timeout_options,
+          }
+
+          proxy_options = { uri: env.request.proxy }
+
+          session = @session.with(options)
+          session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
+
+          responses = session.request(requests)
+          responses.each_with_index do |response, index|
+            handler = @handlers[index]
+            handler.on_response.call(response)
+            handler.on_complete.call(handler.env)
+          end
+        end
+      end
+
+      self.supports_parallel = true
+
+      class << self
+        def setup_parallel_manager
+          ParallelManager.new
+        end
+      end
+
+      def initialize(app)
+        super(app)
+        @session = Session.new
+      end
+
+      def call(env)
+        if parallel?(env)
+          handler = env[:parallel_manager].enqueue(env)
+          handler.on_response do |response|
+            save_response(env, response.status, response.body, response.headers, response.reason) do |response_headers|
+              response_headers.merge!(response.headers)
+            end
+          end
+          return handler
+        end
+
+        request_options = build_request(env)
+
+        timeout_options = {
+          connect_timeout: env.request.open_timeout,
+          operation_timeout: env.request.timeout,
+        }.reject { |_, v| v.nil? }
+
+        options = {
+          ssl: env.ssl,
+          timeout: timeout_options,
+        }
+
+        proxy_options = { uri: env.request.proxy }
+
+        session = @session.with(options)
+        session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
+        response = session.__send__(*request_options)
+        response.raise_for_status unless response.is_a?(::HTTPX::Response)
+        save_response(env, response.status, response.body, response.headers, response.reason) do |response_headers|
+          response_headers.merge!(response.headers)
+        end
+        @app.call(env)
+      rescue OpenSSL::SSL::SSLError => err
+        raise Error::SSLError, err
+      rescue Errno::ECONNABORTED,
+             Errno::ECONNREFUSED,
+             Errno::ECONNRESET,
+             Errno::EHOSTUNREACH,
+             Errno::EINVAL,
+             Errno::ENETUNREACH,
+             Errno::EPIPE => err
+        raise Error::ConnectionFailed, err
+      end
+
+      private
+
+      def parallel?(env)
+        env[:parallel_manager]
+      end
+    end
+
+    register_middleware httpx: HTTPX
+  end
+end

data/lib/httpx/altsvc.rb

@@ -18,6 +18,7 @@ module HTTPX
       now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       @altsvc_mutex.synchronize do
         return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
+
         entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
         @altsvcs[origin] << entry
         entry
@@ -26,6 +27,7 @@ module HTTPX
 
     def lookup(origin, ttl)
       return [] unless @altsvcs.key?(origin)
+
       @altsvcs[origin] = @altsvcs[origin].select do |entry|
         !entry.key?("TTL") || entry["TTL"] > ttl
       end
@@ -35,6 +37,7 @@ module HTTPX
     def emit(request, response)
       # Alt-Svc
       return unless response.headers.key?("alt-svc")
+
       origin = request.origin
       host = request.uri.host
       parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
@@ -45,6 +48,7 @@ module HTTPX
 
     def parse(altsvc)
       return enum_for(__method__, altsvc) unless block_given?
+
       alt_origins, *alt_params = altsvc.split(/ *; */)
       alt_params = Hash[alt_params.map { |field| field.split("=") }]
       alt_origins.split(/ *, */).each do |alt_origin|

data/lib/httpx/callbacks.rb

@@ -19,12 +19,9 @@ module HTTPX
 
     protected
 
-    def inherit_callbacks(callbackable)
-      @callbacks = callbackable.callbacks
-    end
-
     def callbacks(type = nil)
       return @callbacks unless type
+
       @callbacks ||= Hash.new { |h, k| h[k] = [] }
       @callbacks[type]
     end

data/lib/httpx/chainable.rb

@@ -29,7 +29,7 @@ module HTTPX
     end
 
     def plugin(*plugins)
-      klass = is_a?(Client) ? self.class : Client
+      klass = is_a?(Session) ? self.class : Session
       klass = Class.new(klass)
       klass.instance_variable_set(:@default_options, klass.default_options.merge(default_options))
       klass.plugins(plugins).new
@@ -48,8 +48,9 @@ module HTTPX
 
     # :nodoc:
     def branch(options, &blk)
-      return self.class.new(options, &blk) if is_a?(Client)
-      Client.new(options, &blk)
+      return self.class.new(options, &blk) if is_a?(Session)
+
+      Session.new(options, &blk)
     end
   end
 end

data/lib/httpx/connection.rb

@@ -1,150 +1,372 @@
 # frozen_string_literal: true
 
-require "httpx/selector"
-require "httpx/channel"
-require "httpx/resolver"
+require "resolv"
+require "forwardable"
+require "httpx/io"
+require "httpx/buffer"
 
 module HTTPX
+  # The Connection can be watched for IO events.
+  #
+  # It contains the +io+ object to read/write from, and knows what to do when it can.
+  #
+  # It defers connecting until absolutely necessary. Connection should be triggered from
+  # the IO selector (until then, any request will be queued).
+  #
+  # A connection boots up its parser after connection is established. All pending requests
+  # will be redirected there after connection.
+  #
+  # A connection can be prevented from closing by the parser, that is, if there are pending
+  # requests. This will signal that the connection was prematurely closed, due to a possible
+  # number of conditions:
+  #
+  # * Remote peer closed the connection ("Connection: close");
+  # * Remote peer doesn't support pipelining;
+  #
+  # A connection may also route requests for a different host for which the +io+ was connected
+  # to, provided that the IP is the same and the port and scheme as well. This will allow to
+  # share the same socket to send HTTP/2 requests to different hosts.
+  #
   class Connection
-    def initialize(options)
+    extend Forwardable
+    include Registry
+    include Loggable
+    include Callbacks
+
+    using URIExtensions
+
+    require "httpx/connection/http2"
+    require "httpx/connection/http1"
+
+    BUFFER_SIZE = 1 << 14
+
+    def_delegator :@io, :closed?
+
+    def_delegator :@write_buffer, :empty?
+
+    attr_reader :origin, :state, :pending, :options
+
+    attr_reader :timeout
+
+    def initialize(type, uri, options)
+      @type = type
+      @origins = [uri.origin]
+      @origin = URI(uri.origin)
       @options = Options.new(options)
-      @timeout = options.timeout
-      resolver_type = @options.resolver_class
-      resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
-      @selector = Selector.new
-      @channels = []
-      @connected_channels = 0
-      @resolver = resolver_type.new(self, @options)
-      @resolver.on(:resolve, &method(:on_resolver_channel))
-      @resolver.on(:error, &method(:on_resolver_error))
-      @resolver.on(:close, &method(:on_resolver_close))
-    end
-
-    def running?
-      !@channels.empty?
-    end
-
-    def next_tick
-      catch(:jump_tick) do
-        @selector.select(next_timeout) do |monitor|
-          if (channel = monitor.value)
-            channel.call
-          end
-          monitor.interests = channel.interests
+      @window_size = @options.window_size
+      @read_buffer = Buffer.new(BUFFER_SIZE)
+      @write_buffer = Buffer.new(BUFFER_SIZE)
+      @pending = []
+      on(:error, &method(:on_error))
+      if @options.io
+        # if there's an already open IO, get its
+        # peer address, and force-initiate the parser
+        transition(:already_open)
+        @io = IO.registry(@type).new(@origin, nil, @options)
+        parser
+      else
+        transition(:idle)
+      end
+    end
+
+    # this is a semi-private method, to be used by the resolver
+    # to initiate the io object.
+    def addresses=(addrs)
+      @io ||= IO.registry(@type).new(@origin, addrs, @options) # rubocop:disable Naming/MemoizedInstanceVariableName
+    end
+
+    def addresses
+      @io && @io.addresses
+    end
+
+    def match?(uri, options)
+      return false if @state == :closing || @state == :closed
+
+      (@origins.include?(uri.origin) || match_altsvcs?(uri)) && @options == options
+    end
+
+    def mergeable?(connection)
+      return false if @state == :closing || @state == :closed || !@io
+
+      !(@io.addresses & connection.addresses).empty? && @options == connection.options
+    end
+
+    # coalescable connections need to be mergeable!
+    # but internally, #mergeable? is called before #coalescable?
+    def coalescable?(connection)
+      if @io.protocol == "h2" && @origin.scheme == "https"
+        @io.verify_hostname(connection.origin.host)
+      else
+        @origin == connection.origin
+      end
+    end
+
+    def merge(connection)
+      @origins += connection.instance_variable_get(:@origins)
+      pending = connection.instance_variable_get(:@pending)
+      pending.each do |req, args|
+        send(req, args)
+      end
+    end
+
+    def unmerge(connection)
+      @origins -= connection.instance_variable_get(:@origins)
+      purge_pending do |request|
+        request.uri.origin == connection.origin && begin
+          request.transition(:idle)
+          connection.send(request)
+          true
         end
       end
-    rescue TimeoutError => timeout_error
-      @channels.each do |ch|
-        ch.handle_timeout_error(timeout_error)
+    end
+
+    def purge_pending
+      [@parser.pending, @pending].each do |pending|
+        pending.reject! do |request, *args|
+          yield(request, args)
+        end
       end
-    rescue Errno::ECONNRESET,
-           Errno::ECONNABORTED,
-           Errno::EPIPE => ex
-      @channels.each do |ch|
-        ch.emit(:error, ex)
+    end
+
+    # checks if this is connection is an alternative service of
+    # +uri+
+    def match_altsvcs?(uri)
+      AltSvc.cached_altsvc(@origin).any? do |altsvc|
+        origin = altsvc["origin"]
+        origin.altsvc_match?(uri.origin)
       end
     end
 
-    def close
-      @resolver.close unless @resolver.closed?
-      @channels.each(&:close)
-      next_tick until @channels.empty?
+    def connecting?
+      @state == :idle
     end
 
-    def build_channel(uri, **options)
-      channel = Channel.by(uri, @options.merge(options))
-      resolve_channel(channel)
-      channel.on(:open) do
-        @connected_channels += 1
-        @timeout.transition(:open) if @channels.size == @connected_channels
+    def inflight?
+      @parser && !@parser.empty? && !@write_buffer.empty?
+    end
+
+    def interests
+      return :w if @state == :idle
+
+      readable = !@read_buffer.full?
+      writable = !@write_buffer.empty?
+      if readable
+        writable ? :rw : :r
+      else
+        writable ? :w : :r
       end
-      channel.on(:reset) do
-        @timeout.transition(:idle)
+    end
+
+    def to_io
+      case @state
+      when :idle
+        transition(:open)
       end
-      channel.on(:unreachable) do
-        @resolver.uncache(channel)
-        resolve_channel(channel)
+      @io.to_io
+    end
+
+    def close
+      @parser.close if @parser
+      transition(:closing)
+    end
+
+    def reset
+      transition(:closing)
+      transition(:closed)
+      emit(:close)
+    end
+
+    def send(request)
+      if @error_response
+        emit(:response, request, @error_response)
+      elsif @parser && !@write_buffer.full?
+        request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
+        parser.send(request)
+      else
+        @pending << request
       end
-      channel
     end
 
-    # opens a channel to the IP reachable through +uri+.
-    # Many hostnames are reachable through the same IP, so we try to
-    # maximize pipelining by opening as few channels as possible.
-    #
-    def find_channel(uri)
-      @channels.find do |channel|
-        channel.match?(uri)
+    def call
+      @timeout = @timeout_threshold
+      case @state
+      when :closed
+        return
+      when :closing
+        dwrite
+        transition(:closed)
+        emit(:close)
+      when :open
+        consume
       end
+      nil
     end
 
     private
 
-    def resolve_channel(channel)
-      @channels << channel unless @channels.include?(channel)
-      @resolver << channel
-      return if @resolver.empty?
-      @_resolver_monitor ||= begin # rubocop:disable Naming/MemoizedInstanceVariableName
-        monitor = @selector.register(@resolver, :w)
-        monitor.value = @resolver
-        monitor
+    def consume
+      catch(:called) do
+        dread
+        dwrite
+        parser.consume
       end
     end
 
-    def on_resolver_channel(channel, addresses)
-      found_channel = @channels.find do |ch|
-        ch != channel && ch.mergeable?(addresses)
+    def dread(wsize = @window_size)
+      loop do
+        siz = @io.read(wsize, @read_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          on_error(ex)
+          return
+        end
+        return if siz.zero?
+
+        log { "READ: #{siz} bytes..." }
+        parser << @read_buffer.to_s
+        return if @state == :closing || @state == :closed
       end
-      return register_channel(channel) unless found_channel
-      if found_channel.state == :open
-        coalesce_channels(found_channel, channel)
-      else
-        found_channel.once(:open) do
-          coalesce_channels(found_channel, channel)
+    end
+
+    def dwrite
+      loop do
+        return if @write_buffer.empty?
+
+        siz = @io.write(@write_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          on_error(ex)
+          return
         end
+        log { "WRITE: #{siz} bytes..." }
+        return if siz.zero?
+        return if @state == :closing || @state == :closed
       end
     end
 
-    def on_resolver_error(ch, error)
-      ch.emit(:error, error)
-      # must remove channel by hand, hasn't been started yet
-      unregister_channel(ch)
+    def send_pending
+      while !@write_buffer.full? && (req_args = @pending.shift)
+        request = req_args
+        parser.send(request)
+      end
     end
 
-    def on_resolver_close
-      @selector.deregister(@resolver)
-      @_resolver_monitor = nil
-      @resolver.close unless @resolver.closed?
+    def parser
+      @parser ||= build_parser
     end
 
-    def register_channel(channel)
-      @timeout.transition(:idle)
-      monitor = @selector.register(channel, :w)
-      monitor.value = channel
-      channel.on(:close) do
-        unregister_channel(channel)
-      end
+    def build_parser(protocol = @io.protocol)
+      parser = registry(protocol).new(@write_buffer, @options)
+      set_parser_callbacks(parser)
+      parser
     end
 
-    def unregister_channel(channel)
-      @channels.delete(channel)
-      @selector.deregister(channel)
-      @connected_channels -= 1
+    def set_parser_callbacks(parser)
+      parser.on(:response) do |request, response|
+        AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
+          emit(:altsvc, alt_origin, origin, alt_params)
+        end
+        request.emit(:response, response)
+      end
+      parser.on(:altsvc) do |alt_origin, origin, alt_params|
+        emit(:altsvc, alt_origin, origin, alt_params)
+      end
+
+      parser.on(:promise) do |request, stream|
+        request.emit(:promise, parser, stream)
+      end
+      parser.on(:close) do
+        transition(:closing)
+      end
+      parser.on(:reset) do
+        transition(:closing)
+        unless parser.empty?
+          transition(:closed)
+          emit(:reset)
+          transition(:idle)
+          transition(:open)
+        end
+      end
+      parser.on(:timeout) do |timeout|
+        @timeout = timeout
+      end
+      parser.on(:error) do |request, ex|
+        case ex
+        when MisdirectedRequestError
+          emit(:uncoalesce, request.uri)
+        else
+          response = ErrorResponse.new(ex, @options)
+          request.emit(:response, response)
+        end
+      end
     end
 
-    def coalesce_channels(ch1, ch2)
-      if ch1.coalescable?(ch2)
-        ch1.merge(ch2)
-        @channels.delete(ch2)
-      else
-        register_channel(ch2)
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        @error_response = nil
+        @timeout_threshold = @options.timeout.connect_timeout
+        @timeout = @timeout_threshold
+      when :open
+        return if @state == :closed
+
+        @io.connect
+        return unless @io.connected?
+
+        send_pending
+        @timeout_threshold = @options.timeout.operation_timeout
+        @timeout = @timeout_threshold
+        emit(:open)
+      when :closing
+        return unless @state == :open
+      when :closed
+        return unless @state == :closing
+        return unless @write_buffer.empty?
+
+        @io.close
+        @read_buffer.clear
+      when :already_open
+        nextstate = :open
+        send_pending
+        @timeout_threshold = @options.timeout.operation_timeout
+        @timeout = @timeout_threshold
       end
+      @state = nextstate
+    rescue Errno::EHOSTUNREACH
+      # at this point, all addresses from the IO object have failed
+      reset
+      emit(:unreachable)
+      throw(:jump_tick)
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           OpenSSL::SSL::SSLError => e
+      # connect errors, exit gracefully
+      handle_error(e)
+      @state = :closed
+      emit(:close)
+    end
+
+    def on_error(ex)
+      handle_error(ex)
+      reset
     end
 
-    def next_timeout
-      timeout = @timeout.timeout
-      return (@resolver.timeout || timeout) unless @resolver.closed?
-      timeout
+    def handle_error(e)
+      if e.instance_of?(TimeoutError) && @timeout
+        @timeout -= e.timeout
+        return unless @timeout <= 0
+
+        e = e.to_connection_error if connecting?
+      end
+
+      parser.handle_error(e) if @parser && parser.respond_to?(:handle_error)
+      @error_response = ErrorResponse.new(e, @options)
+      @pending.each do |request, _|
+        request.emit(:response, @error_response)
+      end
     end
   end
 end