httpx 0.7.0 → 0.10.0

data/lib/httpx/errors.rb

@@ -28,6 +28,7 @@ module HTTPX
 
   NativeResolveError = Class.new(ResolveError) do
     attr_reader :connection, :host
+
     def initialize(connection, host, message = "Can't resolve #{host}")
       @connection = connection
       @host = host

data/lib/httpx/extensions.rb

@@ -11,7 +11,7 @@ module HTTPX
     #
     # Why not using Refinements? Because they don't work for Method (tested with ruby 2.1.9).
     #
-    module CurryMethods # :nodoc:
+    module CurryMethods
       # Backport for the Method#curry method, which is part of ruby core since 2.2 .
       #
       def curry(*args)
@@ -54,11 +54,31 @@ module HTTPX
     Numeric.__send__(:include, NegMethods)
   end
 
+  module RegexpExtensions
+    # If you wonder why this is there: the oauth feature uses a refinement to enhance the
+    # Regexp class locally with #match? , but this is never tested, because ActiveSupport
+    # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
+    # :nocov:
+    refine(Regexp) do
+      def match?(*args)
+        !match(*args).nil?
+      end
+    end
+  end
+
   module URIExtensions
     refine URI::Generic do
+      def non_ascii_hostname
+        @non_ascii_hostname
+      end
+
+      def non_ascii_hostname=(hostname)
+        @non_ascii_hostname = hostname
+      end
+
       def authority
         port_string = port == default_port ? nil : ":#{port}"
-        "#{host}#{port_string}"
+        "#{@non_ascii_hostname || host}#{port_string}"
       end
 
       def origin
@@ -81,4 +101,4 @@ module HTTPX
       end
     end
   end
-end
+end

data/lib/httpx/headers.rb

@@ -2,7 +2,7 @@
 
 module HTTPX
   class Headers
-    EMPTY = [].freeze # :nodoc:
+    EMPTY = [].freeze
 
     class << self
       def new(headers = nil)
@@ -67,7 +67,7 @@ module HTTPX
     #
     def [](field)
       a = @headers[downcased(field)] || return
-      a.join(",")
+      a.join(", ")
     end
 
     # sets +value+ (if not nil) as single value for the +field+ header.

data/lib/httpx/io/ssl.rb

@@ -20,6 +20,10 @@ module HTTPX
       @state = :negotiated if @keep_open
     end
 
+    def interests
+      @interests || super
+    end
+
     def protocol
       @io.alpn_protocol || super
     rescue StandardError
@@ -38,7 +42,6 @@ module HTTPX
       # allow reconnections
       # connect only works if initial @io is a socket
       @io = @io.io if @io.respond_to?(:io)
-      @negotiated = false
     end
 
     def connected?
@@ -62,15 +65,18 @@ module HTTPX
       @io.connect_nonblock
       @io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
       transition(:negotiated)
-    rescue ::IO::WaitReadable,
-           ::IO::WaitWritable
+    rescue ::IO::WaitReadable
+      @interests = :r
+    rescue ::IO::WaitWritable
+      @interests = :w
     end
 
     # :nocov:
     if RUBY_VERSION < "2.3"
-      def read(*)
+      def read(_, buffer)
         super
       rescue ::IO::WaitWritable
+        buffer.clear
         0
       end
 
@@ -86,6 +92,7 @@ module HTTPX
           buffer.bytesize
         rescue ::IO::WaitReadable,
                ::IO::WaitWritable
+          buffer.clear
           0
         rescue EOFError
           nil

data/lib/httpx/io/tcp.rb

@@ -7,9 +7,7 @@ module HTTPX
   class TCP
     include Loggable
 
-    attr_reader :ip, :port
-
-    attr_reader :addresses
+    attr_reader :ip, :port, :addresses, :state
 
     alias_method :host, :ip
 
@@ -41,6 +39,10 @@ module HTTPX
       @io ||= build_socket
     end
 
+    def interests
+      :w
+    end
+
     def to_io
       @io.to_io
     end
@@ -67,7 +69,7 @@ module HTTPX
       @ip_index -= 1
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeout, e.message if @ip_index <= 0
+      raise ConnectTimeoutError, e.message if @ip_index <= 0
 
       @ip_index -= 1
       retry
@@ -80,8 +82,10 @@ module HTTPX
       # :nocov:
       def read(size, buffer)
         @io.read_nonblock(size, buffer)
+        log { "READ: #{buffer.bytesize} bytes..." }
         buffer.bytesize
       rescue ::IO::WaitReadable
+        buffer.clear
         0
       rescue EOFError
         nil
@@ -89,6 +93,7 @@ module HTTPX
 
       def write(buffer)
         siz = @io.write_nonblock(buffer)
+        log { "WRITE: #{siz} bytes..." }
         buffer.shift!(siz)
         siz
       rescue ::IO::WaitWritable
@@ -100,9 +105,13 @@ module HTTPX
     else
       def read(size, buffer)
         ret = @io.read_nonblock(size, buffer, exception: false)
-        return 0 if ret == :wait_readable
+        if ret == :wait_readable
+          buffer.clear
+          return 0
+        end
         return if ret.nil?
 
+        log { "READ: #{buffer.bytesize} bytes..." }
         buffer.bytesize
       end
 
@@ -111,6 +120,8 @@ module HTTPX
         return 0 if siz == :wait_writable
         return if siz.nil?
 
+        log { "WRITE: #{siz} bytes..." }
+
         buffer.shift!(siz)
         siz
       end

data/lib/httpx/io/udp.rb

@@ -7,11 +7,12 @@ module HTTPX
   class UDP
     include Loggable
 
-    def initialize(uri, _, _)
+    def initialize(uri, _, options)
       ip = IPAddr.new(uri.host)
       @host = ip.to_s
       @port = uri.port
       @io = UDPSocket.new(ip.family)
+      @options = options
     end
 
     def to_io
@@ -40,6 +41,7 @@ module HTTPX
 
     def write(buffer)
       siz = @io.send(buffer, 0, @host, @port)
+      log { "WRITE: #{siz} bytes..." }
       buffer.shift!(siz)
       siz
     end
@@ -49,6 +51,7 @@ module HTTPX
       def read(size, buffer)
         data, _ = @io.recvfrom_nonblock(size)
         buffer.replace(data)
+        log { "READ: #{buffer.bytesize} bytes..." }
         buffer.bytesize
       rescue ::IO::WaitReadable
         0

data/lib/httpx/loggable.rb

@@ -13,35 +13,35 @@ module HTTPX
       white: 37,
     }.freeze
 
-    def log(level: @options.debug_level, label: "", color: nil, &msg)
+    def log(level: @options.debug_level, color: nil, &msg)
       return unless @options.debug
       return unless @options.debug_level >= level
 
       debug_stream = @options.debug
 
-      message = (+label << msg.call << "\n")
+      message = (+"" << msg.call << "\n")
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end
 
     if !Exception.instance_methods.include?(:full_message)
 
-      def log_exception(ex, level: @options.debug_level, label: "", color: nil)
+      def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
         message = +"#{ex.message} (#{ex.class})"
         message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
-        log(level: level, label: label, color: color) { message }
+        log(level: level, color: color) { message }
       end
 
     else
 
-      def log_exception(ex, level: @options.debug_level, label: "", color: nil)
+      def log_exception(ex, level: @options.debug_level, color: nil)
         return unless @options.debug
         return unless @options.debug_level >= level
 
-        log(level: level, label: label, color: color) { ex.full_message }
+        log(level: level, color: color) { ex.full_message }
       end
 
     end

data/lib/httpx/options.rb

@@ -25,23 +25,28 @@ module HTTPX
 
       def def_option(name, &interpreter)
         defined_options << name.to_sym
-        interpreter ||= ->(v) { v }
 
         attr_reader name
 
-        define_method(:"#{name}=") do |value|
-          return if value.nil?
+        if interpreter
+          define_method(:"#{name}=") do |value|
+            return if value.nil?
 
-          instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
-        end
+            instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
+          end
 
-        protected :"#{name}="
+          define_method(:"with_#{name}") do |value|
+            merge(name => instance_exec(value, &interpreter))
+          end
+        else
+          attr_writer name
 
-        define_method(:"with_#{name}") do |value|
-          other = dup
-          other.send(:"#{name}=", other.instance_exec(value, &interpreter))
-          other
+          define_method(:"with_#{name}") do |value|
+            merge(name => value)
+          end
         end
+
+        protected :"#{name}="
       end
     end
 
@@ -70,8 +75,9 @@ module HTTPX
       }
 
       defaults.merge!(options)
-      defaults[:headers] = Headers.new(defaults[:headers])
       defaults.each do |(k, v)|
+        next if v.nil?
+
         __send__(:"#{k}=", v)
       end
     end
@@ -80,7 +86,7 @@ module HTTPX
       if self.headers
         self.headers.merge(headers)
       else
-        headers
+        Headers.new(headers)
       end
     end
 
@@ -116,8 +122,7 @@ module HTTPX
     end
 
     %w[
-      params form json body
-      follow ssl http2_settings
+      params form json body ssl http2_settings
       request_class response_class headers_class request_body_class response_body_class connection_class
       io fallback_protocol debug debug_level transport_options resolver_class resolver_options
       persistent
@@ -143,8 +148,10 @@ module HTTPX
     end
 
     def merge(other)
-      h1 = to_hash
       h2 = other.to_hash
+      return self if h2.empty?
+
+      h1 = to_hash
 
       merged = h1.merge(h2) do |k, v1, v2|
         case k

data/lib/httpx/parser/http1.rb

@@ -9,10 +9,9 @@ module HTTPX
 
       attr_reader :status_code, :http_version, :headers
 
-      def initialize(observer, header_separator: ":")
+      def initialize(observer)
         @observer = observer
         @state = :idle
-        @header_separator = header_separator
         @buffer = "".b
         @headers = {}
       end
@@ -40,25 +39,25 @@ module HTTPX
       private
 
       def parse
-        state = @state
-        case @state
-        when :idle
-          parse_headline
-        when :headers
-          parse_headers
-        when :trailers
-          parse_headers
-        when :data
-          parse_data
+        loop do
+          state = @state
+          case @state
+          when :idle
+            parse_headline
+          when :headers, :trailers
+            parse_headers
+          when :data
+            parse_data
+          end
+          return if @buffer.empty? || state == @state
         end
-        parse if !@buffer.empty? && state != @state
       end
 
       def parse_headline
         idx = @buffer.index("\n")
         return unless idx
 
-        (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
+        (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
           raise(Error, "wrong head line format")
         version, code, _ = m.captures
         raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
@@ -91,12 +90,10 @@ module HTTPX
               @observer.on_trailers(headers)
               headers.clear
               nextstate(:complete)
-            else
-              raise Error, "wrong header format"
             end
             return
           end
-          separator_index = line.index(@header_separator)
+          separator_index = line.index(":")
           raise Error, "wrong header format" unless separator_index
 
           key = line[0..separator_index - 1]

data/lib/httpx/plugins/compression.rb

@@ -14,13 +14,23 @@ module HTTPX
     #
     module Compression
       extend Registry
-      def self.load_dependencies(klass)
-        klass.plugin(:"compression/gzip")
-        klass.plugin(:"compression/deflate")
-      end
 
-      def self.extra_options(options)
-        options.merge(headers: { "accept-encoding" => Compression.registry.keys })
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:"compression/gzip")
+          klass.plugin(:"compression/deflate")
+        end
+
+        def extra_options(options)
+          Class.new(options.class) do
+            def_option(:compression_threshold_size) do |bytes|
+              bytes = Integer(bytes)
+              raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
+
+              bytes
+            end
+          end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
+        end
       end
 
       module RequestMethods
@@ -32,12 +42,17 @@ module HTTPX
       end
 
       module RequestBodyMethods
-        def initialize(*)
+        def initialize(*, options)
           super
           return if @body.nil?
 
+          threshold = options.compression_threshold_size
+          return if threshold && !unbounded_body? && @body.bytesize < threshold
+
           @headers.get("content-encoding").each do |encoding|
-            @body = Encoder.new(@body, Compression.registry(encoding).encoder)
+            next if encoding == "identity"
+
+            @body = Encoder.new(@body, Compression.registry(encoding).deflater)
           end
           @headers["content-length"] = @body.bytesize unless chunked?
         end
@@ -53,55 +68,53 @@ module HTTPX
 
           return unless @headers.key?("content-encoding")
 
-          @_decoders = @headers.get("content-encoding").map do |encoding|
-            decoder = Compression.registry(encoding).decoder
-            # do not uncompress if there is no decoder available. In fact, we can't reliably
-            # continue decompressing beyond that, so ignore.
-            break unless decoder
-
-            @encodings << encoding
-            decoder
-          end
-
           # remove encodings that we are able to decode
           @headers["content-encoding"] = @headers.get("content-encoding") - @encodings
 
-          @_compressed_length = if @headers.key?("content-length")
+          compressed_length = if @headers.key?("content-length")
             @headers["content-length"].to_i
           else
             Float::INFINITY
           end
+
+          @_inflaters = @headers.get("content-encoding").map do |encoding|
+            next if encoding == "identity"
+
+            inflater = Compression.registry(encoding).inflater(compressed_length)
+            # do not uncompress if there is no decoder available. In fact, we can't reliably
+            # continue decompressing beyond that, so ignore.
+            break unless inflater
+
+            @encodings << encoding
+            inflater
+          end.compact
+
+          # this can happen if the only declared encoding is "identity"
+          remove_instance_variable(:@_inflaters) if @_inflaters.empty?
         end
 
         def write(chunk)
-          return super unless defined?(@_compressed_length)
+          return super unless defined?(@_inflaters)
 
-          @_compressed_length -= chunk.bytesize
           chunk = decompress(chunk)
           super(chunk)
         end
 
-        def close
-          super
-
-          return unless defined?(@_decoders)
-
-          @_decoders.each(&:close)
-        end
-
         private
 
         def decompress(buffer)
-          @_decoders.reverse_each do |decoder|
-            buffer = decoder.decode(buffer)
-            buffer << decoder.finish if @_compressed_length <= 0
+          @_inflaters.reverse_each do |inflater|
+            buffer = inflater.inflate(buffer)
           end
           buffer
         end
       end
 
       class Encoder
+        attr_reader :content_type
+
         def initialize(body, deflater)
+          @content_type = body.content_type
           @body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
           @buffer = StringIO.new("".b, File::RDWR)
           @deflater = deflater
@@ -110,11 +123,10 @@ module HTTPX
         def each(&blk)
           return enum_for(__method__) unless block_given?
 
-          unless @buffer.size.zero?
-            @buffer.rewind
-            return @buffer.each(&blk)
-          end
-          deflate(&blk)
+          return deflate(&blk) if @buffer.size.zero?
+
+          @buffer.rewind
+          @buffer.each(&blk)
         end
 
         def bytesize
@@ -122,17 +134,6 @@ module HTTPX
           @buffer.size
         end
 
-        def to_s
-          deflate
-          @buffer.rewind
-          @buffer.read
-        end
-
-        def close
-          @buffer.close
-          @body.close
-        end
-
         private
 
         def deflate(&blk)
@@ -142,22 +143,6 @@ module HTTPX
           @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
         end
       end
-
-      class Decoder
-        extend Forwardable
-
-        def_delegator :@inflater, :finish
-
-        def_delegator :@inflater, :close
-
-        def initialize(inflater)
-          @inflater = inflater
-        end
-
-        def decode(chunk)
-          @inflater.inflate(chunk)
-        end
-      end
     end
     register_plugin :compression, Compression
   end