httpx 0.7.0 → 0.8.0

data/lib/httpx/connection/http2.rb

@@ -30,8 +30,30 @@ module HTTPX
       init_connection
     end
 
+    def interests
+      # waiting for WINDOW_UPDATE frames
+      return :r if @buffer.full?
+
+      return :w if @connection.state == :closed
+
+      return :r unless (@connection.state == :connected && @handshake_completed)
+
+      return :w unless @pending.empty?
+
+      return :w if @streams.each_key.any? { |r| r.interests == :w }
+
+      return :r if @buffer.empty?
+
+      :rw
+    end
+
+    def reset
+      init_connection
+    end
+
     def close
       @connection.goaway unless @connection.state == :closed
+      emit(:close)
     end
 
     def empty?
@@ -39,6 +61,8 @@ module HTTPX
     end
 
     def exhausted?
+      return false if @max_requests.zero? && @connection.active_stream_count.zero?
+
       @connection.active_stream_count >= @max_requests
     end
 
@@ -68,6 +92,8 @@ module HTTPX
 
     def consume
       @streams.each do |request, stream|
+        next if request.state == :done
+
         handle(request, stream)
       end
     end
@@ -128,7 +154,7 @@ module HTTPX
     def handle_stream(stream, request)
       stream.on(:close, &method(:on_stream_close).curry[stream, request])
       stream.on(:half_close) do
-        log(level: 2, label: "#{stream.id}: ") { "waiting for response..." }
+        log(level: 2) { "#{stream.id}: waiting for response..." }
       end
       stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
       stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
@@ -143,8 +169,8 @@ module HTTPX
       headers[":path"]      = headline_uri(request)
       headers[":authority"] = request.authority
       headers = headers.merge(request.headers)
-      log(level: 1, label: "#{stream.id}: ", color: :yellow) do
-        headers.map { |k, v| "-> HEADER: #{k}: #{v}" }.join("\n")
+      log(level: 1, color: :yellow) do
+        headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
       stream.headers(headers, end_stream: request.empty?)
     end
@@ -155,8 +181,8 @@ module HTTPX
       chunk = @drains.delete(request) || request.drain_body
       while chunk
         next_chunk = request.drain_body
-        log(level: 1, label: "#{stream.id}: ", color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
-        log(level: 2, label: "#{stream.id}: ", color: :green) { "-> #{chunk.inspect}" }
+        log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
+        log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
         stream.data(chunk, end_stream: !next_chunk)
         if next_chunk && @buffer.full?
           @drains[request] = next_chunk
@@ -171,25 +197,25 @@ module HTTPX
     ######
 
     def on_stream_headers(stream, request, h)
-      log(label: "#{stream.id}:", color: :yellow) do
-        h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n")
+      log(color: :yellow) do
+        h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
       end
       _, status = h.shift
       headers = request.options.headers_class.new(h)
       response = request.options.response_class.new(request, status, "2.0", headers)
       request.response = response
       @streams[request] = stream
+
+      handle(request, stream) if request.expects?
     end
 
     def on_stream_data(stream, request, data)
-      log(level: 1, label: "#{stream.id}: ", color: :green) { "<- DATA: #{data.bytesize} bytes..." }
-      log(level: 2, label: "#{stream.id}: ", color: :green) { "<- #{data.inspect}" }
+      log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
+      log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
       request.response << data
     end
 
     def on_stream_close(stream, request, error)
-      return handle(request, stream) if request.expects?
-
       if error && error != :no_error
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
@@ -204,14 +230,13 @@ module HTTPX
           emit(:response, request, response)
         end
       end
-      log(level: 2, label: "#{stream.id}: ") { "closing stream" }
+      log(level: 2) { "#{stream.id}: closing stream" }
 
       @streams.delete(request)
       send(@pending.shift) unless @pending.empty?
       return unless @streams.empty? && exhausted?
 
       close
-      emit(:close)
       emit(:exhausted) unless @pending.empty?
     end
 
@@ -222,7 +247,11 @@ module HTTPX
     def on_settings(*)
       @handshake_completed = true
 
-      @max_requests = [@max_requests, @connection.remote_settings[:settings_max_concurrent_streams]].max
+      if @max_requests.zero?
+        @max_requests = @connection.remote_settings[:settings_max_concurrent_streams]
+
+        @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
+      end
 
       @max_concurrent_requests = [@max_concurrent_requests, @max_requests].min
       send_pending
@@ -242,32 +271,26 @@ module HTTPX
     end
 
     def on_frame_sent(frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "frame was sent!" }
-      log(level: 2, label: "#{frame[:stream]}: ", color: :blue) do
-        case frame[:type]
-        when :data
-          frame.merge(payload: frame[:payload].bytesize).inspect
-        else
-          frame.inspect
-        end
+      log(level: 2) { "#{frame[:stream]}: frame was sent!" }
+      log(level: 2, color: :blue) do
+        payload = frame
+        payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
+        "#{frame[:stream]}: #{payload}"
       end
     end
 
     def on_frame_received(frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "frame was received!" }
-      log(level: 2, label: "#{frame[:stream]}: ", color: :magenta) do
-        case frame[:type]
-        when :data
-          frame.merge(payload: frame[:payload].bytesize).inspect
-        else
-          frame.inspect
-        end
+      log(level: 2) { "#{frame[:stream]}: frame was received!" }
+      log(level: 2, color: :magenta) do
+        payload = frame
+        payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
+        "#{frame[:stream]}: #{payload}"
       end
     end
 
     def on_altsvc(origin, frame)
-      log(level: 2, label: "#{frame[:stream]}: ") { "altsvc frame was received" }
-      log(level: 2, label: "#{frame[:stream]}: ") { frame.inspect }
+      log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
+      log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
       alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
       params = { "ma" => frame[:max_age] }
       emit(:altsvc, origin, alt_origin, origin, params)

data/lib/httpx/extensions.rb

@@ -81,4 +81,4 @@ module HTTPX
       end
     end
   end
-end
+end

data/lib/httpx/io/ssl.rb

@@ -20,6 +20,10 @@ module HTTPX
       @state = :negotiated if @keep_open
     end
 
+    def interests
+      @interests || super
+    end
+
     def protocol
       @io.alpn_protocol || super
     rescue StandardError
@@ -62,15 +66,18 @@ module HTTPX
       @io.connect_nonblock
       @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
       transition(:negotiated)
-    rescue ::IO::WaitReadable,
-           ::IO::WaitWritable
+    rescue ::IO::WaitReadable
+      @interests = :r
+    rescue ::IO::WaitWritable
+      @interests = :w
     end
 
     # :nocov:
     if RUBY_VERSION < "2.3"
-      def read(*)
+      def read(_, buffer)
         super
       rescue ::IO::WaitWritable
+        buffer.clear
         0
       end
 
@@ -86,6 +93,7 @@ module HTTPX
           buffer.bytesize
         rescue ::IO::WaitReadable,
                ::IO::WaitWritable
+          buffer.clear
           0
         rescue EOFError
           nil

data/lib/httpx/io/tcp.rb

@@ -11,6 +11,8 @@ module HTTPX
 
     attr_reader :addresses
 
+    attr_reader :state
+
     alias_method :host, :ip
 
     def initialize(origin, addresses, options)
@@ -41,6 +43,10 @@ module HTTPX
       @io ||= build_socket
     end
 
+    def interests
+      :w
+    end
+
     def to_io
       @io.to_io
     end
@@ -67,7 +73,7 @@ module HTTPX
       @ip_index -= 1
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeout, e.message if @ip_index <= 0
+      raise ConnectTimeoutError, e.message if @ip_index <= 0
 
       @ip_index -= 1
       retry
@@ -82,6 +88,7 @@ module HTTPX
         @io.read_nonblock(size, buffer)
         buffer.bytesize
       rescue ::IO::WaitReadable
+        buffer.clear
         0
       rescue EOFError
         nil
@@ -100,7 +107,10 @@ module HTTPX
     else
       def read(size, buffer)
         ret = @io.read_nonblock(size, buffer, exception: false)
-        return 0 if ret == :wait_readable
+        if ret == :wait_readable
+          buffer.clear
+          return 0
+        end
         return if ret.nil?
 
         buffer.bytesize

data/lib/httpx/loggable.rb

@@ -13,35 +13,35 @@ module HTTPX
       white: 37,
     }.freeze
 
-    def log(level: @options.debug_level, label: "", color: nil, &msg)
+    def log(level: @options.debug_level, color: nil, &msg)
       return unless @options.debug
       return unless @options.debug_level >= level
 
       debug_stream = @options.debug
 
-      message = (+label << msg.call << "\n")
+      message = (+"" << msg.call << "\n")
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end
 
     if !Exception.instance_methods.include?(:full_message)
 
-      def log_exception(ex, level: @options.debug_level, label: "", color: nil)
+      def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
         message = +"#{ex.message} (#{ex.class})"
         message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
-        log(level: level, label: label, color: color) { message }
+        log(level: level, color: color) { message }
       end
 
     else
 
-      def log_exception(ex, level: @options.debug_level, label: "", color: nil)
+      def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        log(level: level, label: label, color: color) { ex.full_message }
+        log(level: level, color: color) { ex.full_message }
       end
 
     end

data/lib/httpx/options.rb

@@ -25,23 +25,28 @@ module HTTPX
 
       def def_option(name, &interpreter)
         defined_options << name.to_sym
-        interpreter ||= ->(v) { v }
 
         attr_reader name
 
-        define_method(:"#{name}=") do |value|
-          return if value.nil?
+        if interpreter
+          define_method(:"#{name}=") do |value|
+            return if value.nil?
 
-          instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
-        end
+            instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
+          end
 
-        protected :"#{name}="
+          define_method(:"with_#{name}") do |value|
+            merge(name => instance_exec(value, &interpreter))
+          end
+        else
+          attr_writer name
 
-        define_method(:"with_#{name}") do |value|
-          other = dup
-          other.send(:"#{name}=", other.instance_exec(value, &interpreter))
-          other
+          define_method(:"with_#{name}") do |value|
+            merge(name => value)
+          end
         end
+
+        protected :"#{name}="
       end
     end
 
@@ -70,7 +75,6 @@ module HTTPX
       }
 
       defaults.merge!(options)
-      defaults[:headers] = Headers.new(defaults[:headers])
       defaults.each do |(k, v)|
         __send__(:"#{k}=", v)
       end
@@ -80,7 +84,7 @@ module HTTPX
       if self.headers
         self.headers.merge(headers)
       else
-        headers
+        Headers.new(headers)
       end
     end
 
@@ -116,8 +120,7 @@ module HTTPX
     end
 
     %w[
-      params form json body
-      follow ssl http2_settings
+      params form json body ssl http2_settings
       request_class response_class headers_class request_body_class response_body_class connection_class
       io fallback_protocol debug debug_level transport_options resolver_class resolver_options
       persistent

data/lib/httpx/plugins/compression.rb

@@ -37,6 +37,8 @@ module HTTPX
           return if @body.nil?
 
           @headers.get("content-encoding").each do |encoding|
+            next if encoding == "identity"
+
             @body = Encoder.new(@body, Compression.registry(encoding).encoder)
           end
           @headers["content-length"] = @body.bytesize unless chunked?
@@ -54,6 +56,8 @@ module HTTPX
           return unless @headers.key?("content-encoding")
 
           @_decoders = @headers.get("content-encoding").map do |encoding|
+            next if encoding == "identity"
+
             decoder = Compression.registry(encoding).decoder
             # do not uncompress if there is no decoder available. In fact, we can't reliably
             # continue decompressing beyond that, so ignore.
@@ -61,7 +65,7 @@ module HTTPX
 
             @encodings << encoding
             decoder
-          end
+          end.compact
 
           # remove encodings that we are able to decode
           @headers["content-encoding"] = @headers.get("content-encoding") - @encodings

data/lib/httpx/plugins/compression/gzip.rb

@@ -15,31 +15,41 @@ module HTTPX
         end
 
         class Encoder
+          def initialize
+            @compressed_chunk = "".b
+          end
+
           def deflate(raw, buffer, chunk_size:)
             gzip = Zlib::GzipWriter.new(self)
 
-            while (chunk = raw.read(chunk_size))
-              gzip.write(chunk)
-              gzip.flush
-              compressed = compressed_chunk
-              buffer << compressed
-              yield compressed if block_given?
+            begin
+              while (chunk = raw.read(chunk_size))
+                gzip.write(chunk)
+                gzip.flush
+                compressed = compressed_chunk
+                buffer << compressed
+                yield compressed if block_given?
+              end
+            ensure
+              gzip.close
             end
-          ensure
-            gzip.close
+
+            return unless (compressed = compressed_chunk)
+
+            buffer << compressed
+            yield compressed if block_given?
           end
 
           private
 
           def write(chunk)
-            @compressed_chunk = chunk
+            @compressed_chunk << chunk
           end
 
           def compressed_chunk
-            compressed = @compressed_chunk
-            compressed
+            @compressed_chunk.dup
           ensure
-            @compressed_chunk = nil
+            @compressed_chunk.clear
           end
         end