httpx 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c5b48a5d65398810ef73e2361ca65c113091b91f1dcb88cf7602da6c680e1b1
-  data.tar.gz: 9995ed5b22888e9e0a737be6c197d1d30f5b21d06a7b304242cd3c86efe2f386
+  metadata.gz: bed8cbd465eb9d1b565024395767f619aaaff1bf2ee35d33dc15e9fe4eeec13e
+  data.tar.gz: 65ac8fee9f8c773d6b9a7894e68641547f75ee69159f01e60c8cdc1f30b648f0
 SHA512:
-  metadata.gz: bd3877426964516076b6516237169d06435ff9e39dc44ba977e993de3dfbdcbf90b5a7e486e5f76827f89032d0c9ac5e1d2bac144d025fbe888ff3e07384243d
-  data.tar.gz: 91fcd80f50d0ea4ca564291f4568e78a991b36e52eeb10033dd4e054ff27e7895961b2b69d6d1058eae1be646ee037707e8e9e056994305920cab5a6f3ed61a3
+  metadata.gz: 1e891362a86b73f6d2670eb14a1fd0815a0fab0173f6adf58bdb8e8225d83012560492c5563291d10d03e05ab9e2d9c786ba0cc238e886832276639bab06a8aa
+  data.tar.gz: 539853ff0c6ca209f32d56b8f5eb6209ef60a4df02f0073a588b915a5a583753b7bebe66cd91b76e599a55bb214c6d56a9d1a4d47d6d7a4d5ea2e9b18254af3c

data/lib/httpx.rb

@@ -5,6 +5,7 @@ require "httpx/version"
 require "httpx/extensions"
 
 require "httpx/errors"
+require "httpx/altsvc"
 require "httpx/callbacks"
 require "httpx/loggable"
 require "httpx/registry"

data/lib/httpx/altsvc.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module AltSvc
+    @altsvc_mutex = Mutex.new
+    @altsvcs = Hash.new { |h, k| h[k] = [] }
+
+    module_function
+
+    def cached_altsvc(origin)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @altsvc_mutex.synchronize do
+        lookup(origin, now)
+      end
+    end
+
+    def cached_altsvc_set(origin, entry)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @altsvc_mutex.synchronize do
+        return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
+        entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
+        @altsvcs[origin] << entry
+        entry
+      end
+    end
+
+    def lookup(origin, ttl)
+      return [] unless @altsvcs.key?(origin)
+      @altsvcs[origin] = @altsvcs[origin].select do |entry|
+        !entry.key?("TTL") || entry["TTL"] > ttl
+      end
+      @altsvcs[origin].reject { |entry| entry["noop"] }
+    end
+
+    def emit(request, response)
+      # Alt-Svc
+      return unless response.headers.key?("alt-svc")
+      origin = request.origin
+      host = request.uri.host
+      parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
+        alt_origin.host ||= host
+        yield(alt_origin, origin, alt_params)
+      end
+    end
+
+    def parse(altsvc)
+      return enum_for(__method__, altsvc) unless block_given?
+      alt_origins, *alt_params = altsvc.split(/ *; */)
+      alt_params = Hash[alt_params.map { |field| field.split("=") }]
+      alt_origins.split(/ *, */).each do |alt_origin|
+        yield(parse_altsvc_origin(alt_origin), alt_params)
+      end
+    end
+
+    if RUBY_VERSION < "2.2"
+      def parse_altsvc_origin(alt_origin)
+        alt_proto, alt_origin = alt_origin.split("=")
+        alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
+        if alt_origin.start_with?(":")
+          alt_origin = "dummy#{alt_origin}"
+          uri = URI.parse(alt_origin)
+          uri.host = nil
+          uri
+        else
+          URI.parse("#{alt_proto}://#{alt_origin}")
+        end
+      end
+    else
+      def parse_altsvc_origin(alt_origin)
+        alt_proto, alt_origin = alt_origin.split("=")
+        alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
+        URI.parse("#{alt_proto}://#{alt_origin}")
+      end
+    end
+  end
+end

data/lib/httpx/chainable.rb

@@ -24,6 +24,10 @@ module HTTPX
       headers("accept" => String(type))
     end
 
+    def wrap(&blk)
+      branch(default_options).wrap(&blk)
+    end
+
     def plugin(*plugins)
       klass = is_a?(Client) ? self.class : Client
       klass = Class.new(klass)

data/lib/httpx/channel.rb

@@ -35,6 +35,8 @@ module HTTPX
     include Loggable
     include Callbacks
 
+    using URIExtensions
+
     require "httpx/channel/http2"
     require "httpx/channel/http1"
 
@@ -44,10 +46,15 @@ module HTTPX
       def by(uri, options)
         type = options.transport || begin
           case uri.scheme
-          when "http" then "tcp"
-          when "https" then "ssl"
+          when "http"
+            "tcp"
+          when "https"
+            "ssl"
+          when "h2"
+            options = options.merge(ssl: { alpn_protocols: %(h2) })
+            "ssl"
           else
-            raise Error, "#{uri}: #{uri.scheme}: unsupported URI scheme"
+            raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
           end
         end
         new(type, uri, options)
@@ -58,19 +65,19 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :uri, :state
+    attr_reader :uri, :state, :pending
 
     def initialize(type, uri, options)
       @type = type
       @uri = uri
-      @hostnames = [@uri.host]
+      @origins = [@uri.origin]
       @options = Options.new(options)
       @window_size = @options.window_size
       @read_buffer = Buffer.new(BUFFER_SIZE)
       @write_buffer = Buffer.new(BUFFER_SIZE)
       @pending = []
-      @state = :idle
       on(:error) { |ex| on_error(ex) }
+      transition(:idle)
     end
 
     def addresses=(addrs)
@@ -88,14 +95,12 @@ module HTTPX
       if @io.protocol == "h2" && @uri.scheme == "https"
         @io.verify_hostname(channel.uri.host)
       else
-        @uri.host == channel.uri.host &&
-          @uri.port == channel.uri.port &&
-          @uri.scheme == channel.uri.scheme
+        @uri.origin == channel.uri.origin
       end
     end
 
     def merge(channel)
-      @hostnames += channel.instance_variable_get(:@hostnames)
+      @origins += channel.instance_variable_get(:@origins)
       pending = channel.instance_variable_get(:@pending)
       pending.each do |req, args|
         send(req, args)
@@ -103,14 +108,20 @@ module HTTPX
     end
 
     def unmerge(channel)
-      @hostnames -= channel.instance_variable_get(:@hostnames)
+      @origins -= channel.instance_variable_get(:@origins)
+      purge_pending do |request, args|
+        request.uri == channel.uri && begin
+          request.transition(:idle)
+          channel.send(request, *args)
+          true
+        end
+      end
+    end
+
+    def purge_pending
       [@parser.pending, @pending].each do |pending|
-        pending.reject! do |request|
-          request.uri == channel.uri && begin
-            request.transition(:idle)
-            channel.send(request)
-            true
-          end
+        pending.reject! do |request, *args|
+          yield(request, args)
         end
       end
     end
@@ -118,9 +129,20 @@ module HTTPX
     def match?(uri)
       return false if @state == :closing
 
-      @hostnames.include?(uri.host) &&
-        uri.port == @uri.port &&
-        uri.scheme == @uri.scheme
+      @origins.include?(uri.origin) || match_altsvcs?(uri)
+    end
+
+    # checks if this is channel is an alternative service of
+    # +uri+
+    def match_altsvcs?(uri)
+      AltSvc.cached_altsvc(@uri.origin).any? do |altsvc|
+        origin = altsvc["origin"]
+        origin.altsvc_match?(uri.origin)
+      end
+    end
+
+    def connecting?
+      @state == :idle
     end
 
     def interests
@@ -157,6 +179,7 @@ module HTTPX
       if @error_response
         emit(:response, request, @error_response)
       elsif @parser && !@write_buffer.full?
+        request.headers["alt-used"] = @uri.authority if match_altsvcs?(request.uri)
         parser.send(request, **args)
       else
         @pending << [request, args]
@@ -164,6 +187,7 @@ module HTTPX
     end
 
     def call
+      @timeout = @timeout_threshold
       case @state
       when :closed
         return
@@ -182,6 +206,17 @@ module HTTPX
       @parser = build_parser(protocol)
     end
 
+    def handle_timeout_error(e)
+      return emit(:error, e) unless @timeout
+      @timeout -= e.timeout
+      return unless @timeout <= 0
+      if connecting?
+        emit(:error, e.to_connection_error)
+      else
+        emit(:error, e)
+      end
+    end
+
     private
 
     def consume
@@ -204,6 +239,7 @@ module HTTPX
         return if siz.zero?
         log { "READ: #{siz} bytes..." }
         parser << @read_buffer.to_s
+        return if @state == :closing || @state == :closed
       end
     end
 
@@ -219,6 +255,7 @@ module HTTPX
         end
         log { "WRITE: #{siz} bytes..." }
         return if siz.zero?
+        return if @state == :closing || @state == :closed
       end
     end
 
@@ -236,8 +273,15 @@ module HTTPX
     def build_parser(protocol = @io.protocol)
       parser = registry(protocol).new(@write_buffer, @options)
       parser.on(:response) do |*args|
+        AltSvc.emit(*args) do |alt_origin, origin, alt_params|
+          emit(:altsvc, alt_origin, origin, alt_params)
+        end
         emit(:response, *args)
       end
+      parser.on(:altsvc) do |alt_origin, origin, alt_params|
+        emit(:altsvc, alt_origin, origin, alt_params)
+      end
+
       parser.on(:promise) do |*args|
         emit(:promise, *args)
       end
@@ -248,10 +292,14 @@ module HTTPX
         transition(:closing)
         unless parser.empty?
           transition(:closed)
+          emit(:reset)
           transition(:idle)
           transition(:open)
         end
       end
+      parser.on(:timeout) do |timeout|
+        @timeout = timeout
+      end
       parser.on(:error) do |request, ex|
         case ex
         when MisdirectedRequestError
@@ -269,11 +317,15 @@ module HTTPX
       # when :idle
       when :idle
         @error_response = nil
+        @timeout_threshold = @options.timeout.connect_timeout
+        @timeout = @timeout_threshold
       when :open
         return if @state == :closed
         @io.connect
         return unless @io.connected?
         send_pending
+        @timeout_threshold = @options.timeout.operation_timeout
+        @timeout = @timeout_threshold
         emit(:open)
       when :closing
         return unless @state == :open

data/lib/httpx/channel/http1.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "http_parser"
+require "httpx/parser/http1"
 
 module HTTPX
   class Channel::HTTP1
@@ -12,18 +12,16 @@ module HTTPX
     def initialize(buffer, options)
       @options = Options.new(options)
       @max_concurrent_requests = @options.max_concurrent_requests
-      @parser = HTTP::Parser.new(self)
-      @parser.header_value_type = :arrays
+      @max_requests = Float::INFINITY
+      @parser = Parser::HTTP1.new(self)
       @buffer = buffer
       @version = [1, 1]
       @pending = []
       @requests = []
-      @has_response = false
     end
 
     def reset
       @parser.reset!
-      @has_response = false
     end
 
     def close
@@ -39,15 +37,18 @@ module HTTPX
 
     def <<(data)
       @parser << data
-      dispatch if @has_response
     end
 
     def send(request, **)
-      if @requests.size >= @max_concurrent_requests
+      if @max_requests.positive? &&
+         @requests.size >= @max_concurrent_requests
         @pending << request
         return
       end
-      @requests << request unless @requests.include?(request)
+      unless @requests.include?(request)
+        @requests << request
+        @pipelining = true if @requests.size > 1
+      end
       handle(request)
     end
 
@@ -69,69 +70,59 @@ module HTTPX
     #
     # must be public methods, or else they won't be reachable
 
-    def on_message_begin
+    def on_start
       log(level: 2) { "parsing begins" }
     end
 
-    def on_headers_complete(h)
-      return on_trailer_headers_complete(h) if @parser_trailers
-      # Wait for fix: https://github.com/tmm1/http_parser.rb/issues/52
-      # callback is called 2 times when chunked
-      request = @requests.first
-      return if request.response
+    def on_headers(h)
+      @request = @requests.first
+      return if @request.response
 
       log(level: 2) { "headers received" }
       headers = @options.headers_class.new(h)
-      response = @options.response_class.new(@requests.last,
+      response = @options.response_class.new(@request,
                                              @parser.status_code,
                                              @parser.http_version.join("."),
                                              headers, @options)
       log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
       log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
 
-      request.response = response
+      @request.response = response
+      on_complete if response.complete?
+    end
+
+    def on_trailers(h)
+      return unless @request
+      response = @request.response
+      log(level: 2) { "trailer headers received" }
 
-      @has_response = true if response.complete?
+      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
+      response.merge_headers(h)
     end
 
-    def on_body(chunk)
+    def on_data(chunk)
+      return unless @request
       log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
       log(level: 2, color: :green) { "-> #{chunk.inspect}" }
-      response = @requests.first.response
+      response = @request.response
 
       response << chunk
-
-      @has_response = response.complete?
     end
 
-    def on_message_complete
+    def on_complete
+      return unless @request
       log(level: 2) { "parsing complete" }
-      request = @requests.first
-      response = request.response
-
-      if !@parser_trailers && response.headers.key?("trailer")
-        @parser_trailers = true
-        # this is needed, because the parser can't accept further headers.
-        # we need to reset it and artificially move it to receive headers state,
-        # hence the bogus headline
-        #
-        @parser.reset!
-        @parser << "#{request.verb.to_s.upcase} #{request.path} HTTP/#{response.version}#{CRLF}"
-      else
-        @has_response = true
-      end
-    end
-
-    def on_trailer_headers_complete(h)
-      response = @requests.first.response
-
-      response.merge_headers(h)
+      dispatch
     end
 
     def dispatch
-      request = @requests.first
-      return handle(request) if request.expects?
+      if @request.expects?
+        reset
+        return handle(@request)
+      end
 
+      request = @request
+      @request = nil
       @requests.shift
       response = request.response
       emit(:response, request, response)
@@ -140,28 +131,57 @@ module HTTPX
         response << @parser.upgrade_data
         throw(:called)
       end
+
       reset
+      @max_requests -= 1
       send(@pending.shift) unless @pending.empty?
-      return unless response.headers["connection"] == "close"
-      disable_concurrency
-      emit(:reset)
+      manage_connection(response)
     end
 
     def handle_error(ex)
-      @requests.each do |request|
-        emit(:error, request, ex)
+      if @pipelining
+        disable_pipelining
+        emit(:reset)
+        throw(:called)
+      else
+        @requests.each do |request|
+          emit(:error, request, ex)
+        end
       end
     end
 
     private
 
-    def disable_concurrency
+    def manage_connection(response)
+      connection = response.headers["connection"]
+      case connection
+      when /keep\-alive/i
+        keep_alive = response.headers["keep-alive"]
+        return unless keep_alive
+        parameters = Hash[keep_alive.split(/ *, */).map do |pair|
+          pair.split(/ *= */)
+        end]
+        @max_requests = parameters["max"].to_i if parameters.key?("max")
+        if parameters.key?("timeout")
+          keep_alive_timeout = parameters["timeout"].to_i
+          emit(:timeout, keep_alive_timeout)
+        end
+        # TODO: on keep alive timeout, reset
+      when /close/i, nil
+        disable_pipelining
+        @max_requests = Float::INFINITY
+        emit(:reset)
+      end
+    end
+
+    def disable_pipelining
       return if @requests.empty?
       @requests.each { |r| r.transition(:idle) }
       # server doesn't handle pipelining, and probably
       # doesn't support keep-alive. Fallback to send only
       # 1 keep alive request.
       @max_concurrent_requests = 1
+      @pipelining = false
     end
 
     def set_request_headers(request)
@@ -178,7 +198,6 @@ module HTTPX
     end
 
     def handle(request)
-      @has_response = false
       set_request_headers(request)
       catch(:buffer_full) do
         request.transition(:headers)