httpx 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdf61781fa437f8bf02932b38f029d56f25443690eee087fb09974c5d48595a3
-  data.tar.gz: 75ab23803f01f6c52bcc6f754a93c799efce99b5892c9a4f0815c6254a3e022c
+  metadata.gz: bbd100aaeeeb3425820db41745f3dbc9ff8f4932308a4396139025818054d8a9
+  data.tar.gz: a946a80e2e0060106799228bca3ff6b6c4a7317297758a90e65bd3599eb5cac0
 SHA512:
-  metadata.gz: 57ae0b1d04da8d0e19951a704a40d9c136785281fb5ec2677aaac6b54e98baab0542ad4c8d40553270af8b120d6cf856678bd62554fe1cab7192bfdc26ee08f2
-  data.tar.gz: 1e0f59ecd050838021578981d3ea1f3fa467671ef6c7efc740770cb33462ec81afa7e6a55bbfa2814cd19dacff0bf0e887c6b24ecea405c5440daf369074f766
+  metadata.gz: 95d038b06bad0457f34cb39a373691b64d4c8c2b323c84005166eb22e3240133c7e4f0975e5522c5846cf6dc1959bdec798534026e550e68bc930b0a6d68aad2
+  data.tar.gz: 5d1980dfab340b60851140f6d4976fd374ee74650f6b6e731ca4d6c16288dbcb2cf5acc956db09610e62eda33186afac0cc0b853218852074306644fd2acde7c

data/lib/httpx/adapters/faraday.rb

@@ -6,6 +6,18 @@ require "faraday"
 module Faraday
   class Adapter
     class HTTPX < Faraday::Adapter
+      SSL_ERROR = if defined?(Faraday::SSLError)
+        Faraday::SSLError
+      else
+        Faraday::Error::SSLError
+      end
+
+      CONNECTION_FAILED_ERROR = if defined?(Faraday::ConnectionFailed)
+        Faraday::ConnectionFailed
+      else
+        Faraday::Error::ConnectionFailed
+      end
+
       module RequestMixin
         private
 
@@ -140,7 +152,7 @@ module Faraday
           session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
 
           responses = session.request(requests)
-          responses.each_with_index do |response, index|
+          Array(responses).each_with_index do |response, index|
             handler = @handlers[index]
             handler.on_response.call(response)
             handler.on_complete.call(handler.env)
@@ -162,10 +174,11 @@ module Faraday
       end
 
       def call(env)
+        super
         if parallel?(env)
           handler = env[:parallel_manager].enqueue(env)
           handler.on_response do |response|
-            save_response(env, response.status, response.body, response.headers, response.reason) do |response_headers|
+            save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
               response_headers.merge!(response.headers)
             end
           end
@@ -178,12 +191,12 @@ module Faraday
         session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
         response = session.__send__(*request_options)
         response.raise_for_status unless response.is_a?(::HTTPX::Response)
-        save_response(env, response.status, response.body, response.headers, response.reason) do |response_headers|
+        save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
           response_headers.merge!(response.headers)
         end
         @app.call(env)
       rescue OpenSSL::SSL::SSLError => err
-        raise Error::SSLError, err
+        raise SSL_ERROR, err
       rescue Errno::ECONNABORTED,
              Errno::ECONNREFUSED,
              Errno::ECONNRESET,
@@ -191,7 +204,7 @@ module Faraday
              Errno::EINVAL,
              Errno::ENETUNREACH,
              Errno::EPIPE => err
-        raise Error::ConnectionFailed, err
+        raise CONNECTION_FAILED_ERROR, err
       end
 
       private

data/lib/httpx/altsvc.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "strscan"
+
 module HTTPX
   module AltSvc
     @altsvc_mutex = Mutex.new
@@ -49,9 +51,18 @@ module HTTPX
     def parse(altsvc)
       return enum_for(__method__, altsvc) unless block_given?
 
-      alt_origins, *alt_params = altsvc.split(/ *; */)
-      alt_params = Hash[alt_params.map { |field| field.split("=") }]
-      alt_origins.split(/ *, */).each do |alt_origin|
+      scanner = StringScanner.new(altsvc)
+      until scanner.eos?
+        alt_origin = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
+
+        alt_params = []
+        loop do
+          alt_param = scanner.scan(/[^=]+=("[^"]+"|[^;,]+)/)
+          alt_params << alt_param.strip if alt_param
+          scanner.skip(/;/)
+          break if scanner.eos? || scanner.scan(/ *, */)
+        end
+        alt_params = Hash[alt_params.map { |field| field.split("=") }]
         yield(parse_altsvc_origin(alt_origin), alt_params)
       end
     end

data/lib/httpx/connection.rb

@@ -46,8 +46,6 @@ module HTTPX
 
     attr_reader :origin, :state, :pending, :options
 
-    attr_reader :timeout
-
     def initialize(type, uri, options)
       @type = type
       @origins = [uri.origin]
@@ -148,13 +146,7 @@ module HTTPX
     def interests
       return :w if @state == :idle
 
-      readable = !@read_buffer.full?
-      writable = !@write_buffer.empty?
-      if readable
-        writable ? :rw : :r
-      else
-        writable ? :w : :r
-      end
+      :rw
     end
 
     def to_io
@@ -177,9 +169,7 @@ module HTTPX
     end
 
     def send(request)
-      if @error
-        emit(:response, request, ErrorResponse.new(request, @error, @options))
-      elsif @parser && !@write_buffer.full?
+      if @parser && !@write_buffer.full?
         request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
         parser.send(request)
       else
@@ -188,7 +178,6 @@ module HTTPX
     end
 
     def call
-      @timeout = @timeout_threshold
       case @state
       when :closed
         return
@@ -202,6 +191,14 @@ module HTTPX
       nil
     end
 
+    def timeout
+      return @timeout if defined?(@timeout)
+
+      return @options.timeout.connect_timeout if @state == :idle
+
+      @options.timeout.operation_timeout
+    end
+
     private
 
     def consume
@@ -289,8 +286,8 @@ module HTTPX
           transition(:open)
         end
       end
-      parser.on(:timeout) do |timeout|
-        @timeout = timeout
+      parser.on(:timeout) do |tout|
+        @timeout = tout
       end
       parser.on(:error) do |request, ex|
         case ex
@@ -305,10 +302,6 @@ module HTTPX
 
     def transition(nextstate)
       case nextstate
-      when :idle
-        @error = nil
-        @timeout_threshold = @options.timeout.connect_timeout
-        @timeout = @timeout_threshold
       when :open
         return if @state == :closed
 
@@ -316,8 +309,6 @@ module HTTPX
         return unless @io.connected?
 
         send_pending
-        @timeout_threshold = @options.timeout.operation_timeout
-        @timeout = @timeout_threshold
         emit(:open)
       when :closing
         return unless @state == :open
@@ -327,11 +318,10 @@ module HTTPX
 
         @io.close
         @read_buffer.clear
+        remove_instance_variable(:@timeout) if defined?(@timeout)
       when :already_open
         nextstate = :open
         send_pending
-        @timeout_threshold = @options.timeout.operation_timeout
-        @timeout = @timeout_threshold
       end
       @state = nextstate
     rescue Errno::EHOSTUNREACH
@@ -354,18 +344,17 @@ module HTTPX
       reset
     end
 
-    def handle_error(e)
-      if e.instance_of?(TimeoutError) && @timeout
-        @timeout -= e.timeout
+    def handle_error(error)
+      if error.instance_of?(TimeoutError) && @timeout
+        @timeout -= error.timeout
         return unless @timeout <= 0
 
-        e = e.to_connection_error if connecting?
+        error = error.to_connection_error if connecting?
       end
 
-      parser.handle_error(e) if @parser && parser.respond_to?(:handle_error)
-      @error = e
+      parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
       @pending.each do |request, _|
-        request.emit(:response, ErrorResponse.new(request, @error, @options))
+        request.emit(:response, ErrorResponse.new(request, error, @options))
       end
     end
   end

data/lib/httpx/plugins/follow_redirects.rb

@@ -61,6 +61,7 @@ module HTTPX
 
           connection = find_connection(retry_request, connections, options)
           connection.send(retry_request)
+          set_request_timeout(connection, retry_request, options)
           nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -44,6 +44,7 @@ module HTTPX
             connection = find_connection(request, connections, options)
             connections << connection unless connections.include?(connection)
             connection.upgrade(request, response)
+            set_request_timeout(connection, request, options)
           end
           response
         end

data/lib/httpx/plugins/proxy.rb

@@ -119,6 +119,7 @@ module HTTPX
             connection = find_connection(request, connections, options)
             connections << connection unless connections.include?(connection)
             connection.send(request)
+            set_request_timeout(connection, request, options)
             return
           end
           response

data/lib/httpx/plugins/retries.rb

@@ -54,6 +54,7 @@ module HTTPX
             request.transition(:idle)
             connection = find_connection(request, connections, options)
             connection.send(request)
+            set_request_timeout(connection, request, options)
             return
           end
           response

data/lib/httpx/pool.rb

@@ -1,14 +1,21 @@
 # frozen_string_literal: true
 
+require "forwardable"
+require "timers"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
 
 module HTTPX
   class Pool
+    extend Forwardable
+
+    def_delegator :@timers, :after
+
     def initialize
       @resolvers = {}
       @_resolver_monitors = {}
+      @timers = Timers::Group.new
       @selector = Selector.new
       @connections = []
       @connected_connections = 0
@@ -18,14 +25,13 @@ module HTTPX
       @connections.empty?
     end
 
-    def next_tick(timeout = nil)
+    def next_tick
       catch(:jump_tick) do
-        tout = timeout.total_timeout if timeout
-
-        @selector.select(next_timeout || tout) do |monitor|
+        @selector.select(next_timeout || @timers.wait_interval) do |monitor|
           monitor.io.call
           monitor.interests = monitor.io.interests
         end
+        @timers.fire
       end
     rescue StandardError => ex
       @connections.each do |connection|
@@ -34,6 +40,7 @@ module HTTPX
     end
 
     def close(connections = @connections)
+      @timers.cancel
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
       next_tick until connections.none? { |c| @connections.include?(c) }

data/lib/httpx/request.rb

@@ -35,9 +35,9 @@ module HTTPX
 
     attr_reader :verb, :uri, :headers, :body, :state
 
-    attr_reader :options
+    attr_reader :options, :response
 
-    attr_accessor :response
+    attr_accessor :timer
 
     def_delegator :@body, :<<
 
@@ -68,6 +68,13 @@ module HTTPX
       @uri.scheme
     end
 
+    def response=(response)
+      return unless response
+
+      @timer.cancel if @timer
+      @response = response
+    end
+
     def path
       path = uri.path.dup
       path << "/" if path.empty?

data/lib/httpx/resolver/native.rb

@@ -31,6 +31,18 @@ module HTTPX
       }.freeze
     end
 
+    # nameservers for ipv6 are misconfigured in certain systems;
+    # this can use an unexpected endless loop
+    # https://gitlab.com/honeyryderchuck/httpx/issues/56
+    DEFAULTS[:nameserver].select! do |nameserver|
+      begin
+        IPAddr.new(nameserver)
+        true
+      rescue IPAddr::InvalidAddressError
+        false
+      end
+    end if DEFAULTS[:nameserver]
+
     DNS_PORT = 53
 
     def_delegator :@connections, :empty?
@@ -87,7 +99,7 @@ module HTTPX
       else
         if e.respond_to?(:connection) &&
            e.respond_to?(:host)
-          emit_resolve_error(connection, host, e)
+          emit_resolve_error(e.connection, e.host, e)
         else
           @queries.each do |host, connection|
             emit_resolve_error(connection, host, e)

data/lib/httpx/response.rb

@@ -111,7 +111,7 @@ module HTTPX
           unless @state == :idle
             rewind
             while (chunk = @buffer.read(@window_size))
-              yield(chunk)
+              yield(chunk.force_encoding(@encoding))
             end
           end
         ensure
@@ -129,7 +129,7 @@ module HTTPX
             return content
           end
         end
-        ""
+        "".b
       ensure
         close
       end
@@ -178,7 +178,7 @@ module HTTPX
         when :idle
           if @length > @threshold_size
             @state = :buffer
-            @buffer = Tempfile.new("httpx", encoding: @encoding, mode: File::RDWR)
+            @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
           else
             @state = :memory
             @buffer = StringIO.new("".b, File::RDWR)
@@ -186,7 +186,7 @@ module HTTPX
         when :memory
           if @length > @threshold_size
             aux = @buffer
-            @buffer = Tempfile.new("httpx", encoding: @encoding, mode: File::RDWR)
+            @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
             aux.rewind
             ::IO.copy_stream(aux, @buffer)
             # (this looks like a bug from Ruby < 2.3

data/lib/httpx/selector.rb

@@ -48,44 +48,25 @@ class HTTPX::Selector
   end
 
   def initialize
-    @readers = {}
-    @writers = {}
+    @selectables = {}
     @__r__, @__w__ = IO.pipe
     @closed = false
   end
 
   # deregisters +io+ from selectables.
   def deregister(io)
-    rmonitor = @readers.delete(io)
-    wmonitor = @writers.delete(io)
-    monitor = rmonitor || wmonitor
+    monitor = @selectables.delete(io)
     monitor.close(false) if monitor
   end
 
   # register +io+ for +interests+ events.
   def register(io, interests)
-    readable = READABLE.include?(interests)
-    writable = WRITABLE.include?(interests)
-    if readable
-      monitor = @readers[io]
-      if monitor
-        monitor.interests = interests
-      else
-        monitor = Monitor.new(io, interests, self)
-      end
-      @readers[io] = monitor
-      @writers.delete(io) unless writable
-    end
-    if writable
-      monitor = @writers[io]
-      if monitor
-        monitor.interests = interests
-      else
-        # reuse object
-        monitor = readable ? @readers[io] : Monitor.new(io, interests, self)
-      end
-      @writers[io] = monitor
-      @readers.delete(io) unless readable
+    monitor = @selectables[io]
+    if monitor
+      monitor.interests = interests
+    else
+      monitor = Monitor.new(io, interests, self)
+      @selectables[io] = monitor
     end
     monitor
   end
@@ -95,16 +76,20 @@ class HTTPX::Selector
   #
   def select(interval)
     begin
-      r = @readers.keys
-      w = @writers.keys
-      r.unshift(@__r__)
+      r = [@__r__]
+      w = []
+
+      @selectables.each do |io, monitor|
+        r << io if monitor.interests == :r || monitor.interests == :rw
+        w << io if monitor.interests == :w || monitor.interests == :rw
+        monitor.readiness = nil
+      end
 
       readers, writers = IO.select(r, w, nil, interval)
 
       raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") if readers.nil? && writers.nil?
     rescue IOError, SystemCallError
-      @readers.reject! { |io, _| io.closed? }
-      @writers.reject! { |io, _| io.closed? }
+      @selectables.reject! { |io, _| io.closed? }
       retry
     end
 
@@ -113,7 +98,7 @@ class HTTPX::Selector
         # clean up wakeups
         @__r__.read(@__r__.stat.size)
       else
-        monitor = io.closed? ? @readers.delete(io) : @readers[io]
+        monitor = io.closed? ? @selectables.delete(io) : @selectables[io]
         next unless monitor
 
         monitor.readiness = writers.delete(io) ? :rw : :r
@@ -122,7 +107,7 @@ class HTTPX::Selector
     end if readers
 
     writers.each do |io|
-      monitor = io.closed? ? @writers.delete(io) : @writers[io]
+      monitor = io.closed? ? @selectables.delete(io) : @selectables[io]
       next unless monitor
 
       # don't double run this, the last iteration might have run this task already

data/lib/httpx/session.rb

@@ -154,11 +154,11 @@ module HTTPX
     def send_requests(*requests, options)
       connections = []
       request_options = @options.merge(options)
-      timeout = request_options.timeout
 
       requests.each do |request|
         connection = find_connection(request, connections, request_options)
         connection.send(request)
+        set_request_timeout(connection, request, request_options)
       end
 
       responses = []
@@ -168,7 +168,7 @@ module HTTPX
         loop do
           begin
             request = requests.first
-            pool.next_tick(timeout) until (response = fetch_response(request, connections, request_options))
+            pool.next_tick until (response = fetch_response(request, connections, request_options))
 
             responses << response
             requests.shift
@@ -190,6 +190,21 @@ module HTTPX
       request
     end
 
+    def set_request_timeout(connection, request, options)
+      total = options.timeout.total_timeout
+      return unless total
+
+      timer = pool.after(total) do
+        unless @responses[request]
+          error = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
+          response = ErrorResponse.new(request, error, options)
+          request.emit(:response, response)
+          connection.reset
+        end
+      end
+      request.timer = timer
+    end
+
     @default_options = Options.new
     @default_options.freeze
     @plugins = []

data/lib/httpx/timeout.rb

@@ -13,7 +13,7 @@ module HTTPX
       super
     end
 
-    attr_reader :connect_timeout, :operation_timeout
+    attr_reader :connect_timeout, :operation_timeout, :total_timeout
 
     def initialize(connect_timeout: CONNECT_TIMEOUT,
                    operation_timeout: OPERATION_TIMEOUT,
@@ -22,17 +22,11 @@ module HTTPX
       @connect_timeout = connect_timeout
       @operation_timeout = operation_timeout
       @total_timeout = total_timeout
-      if loop_timeout
-        warn ":loop_timeout is deprecated, use :operation_timeout instead"
-        @operation_timeout = loop_timeout
-      end
-      reset_counter
-    end
 
-    def total_timeout
-      @total_timeout
-    ensure
-      log_time
+      return unless loop_timeout
+
+      warn ":loop_timeout is deprecated, use :operation_timeout instead"
+      @operation_timeout = loop_timeout
     end
 
     def ==(other)
@@ -61,29 +55,5 @@ module HTTPX
         raise ArgumentError, "can't merge with #{other.class}"
       end
     end
-
-    def no_time_left?
-      @time_left <= 0
-    end
-
-    private
-
-    def reset_counter
-      @time_left = @total_timeout
-    end
-
-    def reset_timer
-      @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
-
-    def log_time
-      return unless @time_left
-      return reset_timer unless @started
-
-      @time_left -= (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started)
-      raise TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds") if no_time_left?
-
-      reset_timer
-    end
   end
 end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-27 00:00:00.000000000 Z
+date: 2019-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: timers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: http-form_data
   requirement: !ruby/object:Gem::Requirement