httpx 0.2.1 → 0.3.0

data/lib/httpx/channel/http2.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "io/wait"
 require "http/2"
 
 module HTTPX
@@ -7,6 +8,22 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    if HTTP2::VERSION < "0.10.1"
+      module HTTP2Extensions
+        refine ::HTTP2::Client do
+          def receive(*)
+            send_connection_preface
+            super
+          end
+
+          def <<(*args)
+            receive(*args)
+          end
+        end
+      end
+      using HTTP2Extensions
+    end
+
     Error = Class.new(Error) do
       def initialize(id, code)
         super("stream #{id} closed with error: #{code}")
@@ -18,11 +35,12 @@ module HTTPX
     def initialize(buffer, options)
       @options = Options.new(options)
       @max_concurrent_requests = @options.max_concurrent_requests
-      init_connection
       @pending = []
       @streams = {}
       @drains  = {}
       @buffer = buffer
+      @handshake_completed = false
+      init_connection
     end
 
     def close
@@ -38,7 +56,8 @@ module HTTPX
     end
 
     def send(request, **)
-      if @connection.active_stream_count >= @max_concurrent_requests
+      if !@handshake_completed ||
+         @connection.active_stream_count >= @max_concurrent_requests
         @pending << request
         return
       end
@@ -48,6 +67,7 @@ module HTTPX
         @streams[request] = stream
       end
       handle(request, stream)
+      true
     end
 
     def reenqueue!
@@ -73,6 +93,12 @@ module HTTPX
 
     private
 
+    def send_pending
+      while (request = @pending.shift)
+        break unless send(request)
+      end
+    end
+
     def headline_uri(request)
       request.path
     end
@@ -95,9 +121,16 @@ module HTTPX
       @connection.on(:frame_sent, &method(:on_frame_sent))
       @connection.on(:frame_received, &method(:on_frame_received))
       @connection.on(:promise, &method(:on_promise))
-      @connection.on(:altsvc, &method(:on_altsvc))
+      @connection.on(:altsvc) { |frame| on_altsvc(frame[:origin], frame) }
       @connection.on(:settings_ack, &method(:on_settings))
       @connection.on(:goaway, &method(:on_close))
+      #
+      # Some servers initiate HTTP/2 negotiation right away, some don't.
+      # As such, we have to check the socket buffer. If there is something
+      # to read, the server initiated the negotiation. If not, we have to
+      # initiate it.
+      #
+      @connection.send_connection_preface
     end
 
     def handle_stream(stream, request)
@@ -105,7 +138,7 @@ module HTTPX
       stream.on(:half_close) do
         log(level: 2, label: "#{stream.id}: ") { "waiting for response..." }
       end
-      # stream.on(:altsvc)
+      stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
       stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
       stream.on(:data, &method(:on_stream_data).curry[stream, request])
     end
@@ -187,8 +220,10 @@ module HTTPX
     end
 
     def on_settings(*)
+      @handshake_completed = true
       @max_concurrent_requests = [@max_concurrent_requests,
                                   @connection.remote_settings[:settings_max_concurrent_streams]].min
+      send_pending
     end
 
     def on_close(_last_frame, error, _payload)
@@ -225,9 +260,12 @@ module HTTPX
       end
     end
 
-    def on_altsvc(frame)
+    def on_altsvc(origin, frame)
       log(level: 2, label: "#{frame[:stream]}: ") { "altsvc frame was received" }
       log(level: 2, label: "#{frame[:stream]}: ") { frame.inspect }
+      alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
+      params = { "ma" => frame[:max_age] }
+      emit(:altsvc, origin, alt_origin, origin, params)
     end
 
     def on_promise(stream)

data/lib/httpx/client.rb

@@ -47,7 +47,6 @@ module HTTPX
     def on_promise(_, stream)
       log(level: 2, label: "#{stream.id}: ") { "refusing stream!" }
       stream.refuse
-      # TODO: policy for handling promises
     end
 
     def fetch_response(request)
@@ -66,6 +65,9 @@ module HTTPX
         other_channel = build_channel(uncoalesced_uri, options)
         channel.unmerge(other_channel)
       end
+      channel.on(:altsvc) do |alt_origin, origin, alt_params|
+        build_altsvc_channel(channel, alt_origin, origin, alt_params, options)
+      end
     end
 
     def build_channel(uri, options)
@@ -74,6 +76,35 @@ module HTTPX
       channel
     end
 
+    def build_altsvc_channel(existing_channel, alt_origin, origin, alt_params, options)
+      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+
+      # altsvc already exists, somehow it wasn't advertised, probably noop
+      return unless altsvc
+
+      channel = @connection.find_channel(alt_origin) || build_channel(alt_origin, options)
+      # advertised altsvc is the same origin being used, ignore
+      return if channel == existing_channel
+
+      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+
+      # get uninitialized requests
+      # incidentally, all requests will be re-routed to the first
+      # advertised alt-svc, which incidentally follows the spec.
+      existing_channel.purge_pending do |request, args|
+        is_idle = request.origin == origin &&
+                  request.state == :idle &&
+                  !request.headers.key?("alt-used")
+        if is_idle
+          log(level: 1) { "#{origin} alt-svc: sending #{request.uri} to #{alt_origin}" }
+          channel.send(request, args)
+        end
+        is_idle
+      end
+    rescue UnsupportedSchemeError
+      altsvc["noop"] = true
+    end
+
     def __build_reqs(*args, **options)
       requests = case args.size
                  when 1

data/lib/httpx/connection.rb

@@ -13,6 +13,7 @@ module HTTPX
       resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
       @selector = Selector.new
       @channels = []
+      @connected_channels = 0
       @resolver = resolver_type.new(self, @options)
       @resolver.on(:resolve, &method(:on_resolver_channel))
       @resolver.on(:error, &method(:on_resolver_error))
@@ -32,8 +33,11 @@ module HTTPX
           monitor.interests = channel.interests
         end
       end
-    rescue TimeoutError,
-           Errno::ECONNRESET,
+    rescue TimeoutError => timeout_error
+      @channels.each do |ch|
+        ch.handle_timeout_error(timeout_error)
+      end
+    rescue Errno::ECONNRESET,
            Errno::ECONNABORTED,
            Errno::EPIPE => ex
       @channels.each do |ch|
@@ -50,6 +54,13 @@ module HTTPX
     def build_channel(uri, **options)
       channel = Channel.by(uri, @options.merge(options))
       resolve_channel(channel)
+      channel.on(:open) do
+        @connected_channels += 1
+        @timeout.transition(:open) if @channels.size == @connected_channels
+      end
+      channel.on(:reset) do
+        @timeout.transition(:idle)
+      end
       channel.once(:unreachable) do
         @resolver.uncache(channel)
         resolve_channel(channel)
@@ -107,6 +118,7 @@ module HTTPX
     end
 
     def register_channel(channel)
+      @timeout.transition(:idle)
       monitor = @selector.register(channel, :w)
       monitor.value = channel
       channel.on(:close) do
@@ -117,12 +129,7 @@ module HTTPX
     def unregister_channel(channel)
       @channels.delete(channel)
       @selector.deregister(channel)
-    end
-
-    def next_timeout
-      timeout = @timeout.timeout # force log time
-      return (@resolver.timeout || timeout) unless @resolver.closed?
-      timeout
+      @connected_channels -= 1
     end
 
     def coalesce_channels(ch1, ch2)
@@ -133,5 +140,11 @@ module HTTPX
         register_channel(ch2)
       end
     end
+
+    def next_timeout
+      timeout = @timeout.timeout
+      return (@resolver.timeout || timeout) unless @resolver.closed?
+      timeout
+    end
   end
 end

data/lib/httpx/errors.rb

@@ -3,7 +3,24 @@
 module HTTPX
   Error = Class.new(StandardError)
 
-  TimeoutError = Class.new(Error)
+  UnsupportedSchemeError = Class.new(Error)
+
+  TimeoutError = Class.new(Error) do
+    attr_reader :timeout
+
+    def initialize(timeout, message)
+      @timeout = timeout
+      super(message)
+    end
+
+    def to_connection_error
+      ex = ConnectTimeoutError.new(@timeout, message)
+      ex.set_backtrace(backtrace)
+      ex
+    end
+  end
+
+  ConnectTimeoutError = Class.new(TimeoutError)
 
   ResolveError = Class.new(Error)
 

data/lib/httpx/extensions.rb

@@ -1,52 +1,83 @@
 # frozen_string_literal: true
 
-unless Method.method_defined?(:curry)
-
-  # Backport
-  #
-  # Ruby 2.1 and lower implement curry only for Procs.
-  #
-  # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
-  #
-  module CurryMethods # :nodoc:
-    # Backport for the Method#curry method, which is part of ruby core since 2.2 .
+require "uri"
+
+module HTTPX
+  unless Method.method_defined?(:curry)
+
+    # Backport
+    #
+    # Ruby 2.1 and lower implement curry only for Procs.
+    #
+    # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
     #
-    def curry(*args)
-      to_proc.curry(*args)
+    module CurryMethods # :nodoc:
+      # Backport for the Method#curry method, which is part of ruby core since 2.2 .
+      #
+      def curry(*args)
+        to_proc.curry(*args)
+      end
     end
+    Method.__send__(:include, CurryMethods)
   end
-  Method.__send__(:include, CurryMethods)
-end
-
-unless String.method_defined?(:+@)
-  # Backport for +"", to initialize unfrozen strings from the string literal.
-  #
-  module LiteralStringExtensions
-    def +@
-      frozen? ? dup : self
+
+  unless String.method_defined?(:+@)
+    # Backport for +"", to initialize unfrozen strings from the string literal.
+    #
+    module LiteralStringExtensions
+      def +@
+        frozen? ? dup : self
+      end
     end
+    String.__send__(:include, LiteralStringExtensions)
   end
-  String.__send__(:include, LiteralStringExtensions)
-end
-
-unless Numeric.method_defined?(:positive?)
-  # Ruby 2.3 Backport (Numeric#positive?)
-  #
-  module PosMethods
-    def positive?
-      self > 0
+
+  unless Numeric.method_defined?(:positive?)
+    # Ruby 2.3 Backport (Numeric#positive?)
+    #
+    module PosMethods
+      def positive?
+        self > 0
+      end
     end
+    Numeric.__send__(:include, PosMethods)
   end
-  Numeric.__send__(:include, PosMethods)
-end
-
-unless Numeric.method_defined?(:negative?)
-  # Ruby 2.3 Backport (Numeric#negative?)
-  #
-  module NegMethods
-    def negative?
-      self < 0
+
+  unless Numeric.method_defined?(:negative?)
+    # Ruby 2.3 Backport (Numeric#negative?)
+    #
+    module NegMethods
+      def negative?
+        self < 0
+      end
+    end
+    Numeric.__send__(:include, NegMethods)
+  end
+
+  module URIExtensions
+    refine URI::Generic do
+      def authority
+        port_string = port == default_port ? nil : ":#{port}"
+        "#{host}#{port_string}"
+      end
+
+      def origin
+        "#{scheme}://#{authority}"
+      end
+
+      def altsvc_match?(uri)
+        uri = URI.parse(uri)
+        self == uri || begin
+          case scheme
+          when 'h2'
+            uri.scheme == "https" &&
+            host == uri.host &&
+            (port || default_port) == (uri.port || uri.default_port)
+          else
+            false
+          end
+        end
+      end
     end
   end
-  Numeric.__send__(:include, NegMethods)
-end
+end

data/lib/httpx/parser/http1.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Parser
+    Error = Class.new(Error)
+
+    class HTTP1
+      VERSIONS = %w[1.0 1.1].freeze
+
+      attr_reader :status_code, :http_version, :headers
+
+      def initialize(observer, header_separator: ":")
+        @observer = observer
+        @state = :idle
+        @header_separator = header_separator
+        @buffer = "".b
+        @headers = {}
+      end
+
+      def <<(chunk)
+        @buffer << chunk
+        parse
+      end
+
+      def reset!
+        @state = :idle
+        @headers.clear
+        @content_length = nil
+        @_has_trailers = nil
+      end
+
+      def upgrade?
+        @upgrade
+      end
+
+      def upgrade_data
+        @buffer
+      end
+
+      private
+
+      def parse
+        state = @state
+        case @state
+        when :idle
+          parse_headline
+        when :headers
+          parse_headers
+        when :trailers
+          parse_headers
+        when :data
+          parse_data
+        end
+        parse if !@buffer.empty? && state != @state
+      end
+
+      def parse_headline
+        idx = @buffer.index("\n")
+        return unless idx
+        (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
+          raise(Error, "wrong head line format")
+        version, code, _ = m.captures
+        raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
+        @http_version = version.split(".").map(&:to_i)
+        @status_code = code.to_i
+        raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
+        @buffer.slice!(0, idx + 1)
+        nextstate(:headers)
+      end
+
+      def parse_headers
+        headers = @headers
+        while (idx = @buffer.index("\n"))
+          line = @buffer.slice!(0, idx + 1).sub(/\s+\z/, "")
+          if line.empty?
+            case @state
+            when :headers
+              prepare_data(headers)
+              @observer.on_headers(headers)
+              return unless @state == :headers
+              # state might have been reset
+              # in the :headers callback
+              nextstate(:data)
+              headers.clear
+            when :trailers
+              @observer.on_trailers(headers)
+              headers.clear
+              nextstate(:complete)
+            else
+              raise Error, "wrong header format"
+            end
+            return
+          end
+          separator_index = line.index(@header_separator)
+          raise Error, "wrong header format" unless separator_index
+          key = line[0..separator_index - 1]
+          raise Error, "wrong header format" if key.start_with?("\s", "\t")
+          key.strip!
+          value = line[separator_index + 1..-1]
+          value.strip!
+          raise Error, "wrong header format" if value.nil?
+          (headers[key.downcase] ||= []) << value
+        end
+      end
+
+      def parse_data
+        if @buffer.respond_to?(:each)
+          @buffer.each do |chunk|
+            @observer.on_data(chunk)
+          end
+        elsif @content_length
+          data = @buffer.slice!(0, @content_length)
+          @content_length -= data.bytesize
+          @observer.on_data(data)
+          data.clear
+        else
+          @observer.on_data(@buffer)
+          @buffer.clear
+        end
+        return unless no_more_data?
+        @buffer = @buffer.to_s
+        if @_has_trailers
+          nextstate(:trailers)
+        else
+          nextstate(:complete)
+        end
+      end
+
+      def prepare_data(headers)
+        @upgrade = headers.key?("upgrade")
+
+        @_has_trailers = headers.key?("trailer")
+
+        if (tr_encodings = headers["transfer-encoding"])
+          tr_encodings.reverse_each do |tr_encoding|
+            tr_encoding.split(/ *, */).each do |encoding|
+              case encoding
+              when "chunked"
+                @buffer = Transcoder::Chunker::Decoder.new(@buffer, @_has_trailers)
+              end
+            end
+          end
+        else
+          @content_length = headers["content-length"][0].to_i if headers.key?("content-length")
+        end
+      end
+
+      def no_more_data?
+        if @content_length
+          @content_length <= 0
+        elsif @buffer.respond_to?(:finished?)
+          @buffer.finished?
+        else
+          false
+        end
+      end
+
+      def nextstate(state)
+        @state = state
+        case state
+        when :headers
+          @observer.on_start
+        when :complete
+          @observer.on_complete
+          reset!
+          nextstate(:idle) unless @buffer.empty?
+        end
+      end
+    end
+  end
+end