httpx 0.20.0 → 1.3.1

data/lib/httpx/pool/synch_pool.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "thread"
+
+module HTTPX
+  class SynchPool < Pool
+    def initialize(options)
+      super
+
+      @connections = ConnectionStore.new(options)
+    end
+
+    # TODO: #wrap
+    def find_or_new_connection(uri, options, &blk)
+      @connections.find_or_new(uri, options) do |new_conn|
+        catch(:coalesced) do
+          init_connection(new_conn, options)
+          blk.call(new_conn) if blk
+          new_conn
+        end
+      end
+      find_connection(uri, options) || new_connection(uri, options, &blk)
+    end
+
+    class ConnectionManager
+      include Enumerable
+
+      def initialize(limit = 3)
+        @connections = []
+        @used = 0
+        @limit = limit
+      end
+
+      def each(*args, &blk)
+        @connections.each(*args, &blk)
+      end
+
+      def find_or_new(uri, options, &blk)
+        raise "over limit" if @used >= @limit
+
+        @used += 1
+        conn = @connections.find do |connection|
+          connection.match?(uri, options)
+        end
+
+        if conn
+          @connections.delete(conn)
+        else
+          conn = options.connection_class.new(uri, options)
+          blk[conn]
+        end
+
+        conn
+      end
+    end
+
+    class ConnectionStore
+      include Enumerable
+
+      def initialize(options)
+        @connections = Hash.new { |hs, k| hs[k] ||= ConnectionManager.new }
+        @conn_mtx = Thread::Mutex.new
+        @conn_waiter = ConditionVariable.new
+        @timeout = Float(options.fetch(:pool_timeout, 5))
+      end
+
+      def each(&block)
+        return enum_for(__meth__) unless block
+
+        @conn_mtx.synchronize do
+          @connections.each_value do |conns|
+            conns.each(&block)
+          end
+        end
+      end
+
+      def find_or_new(uri, options, &blk)
+        @connections[uri.origin].find_or_new(uri, options, &blk)
+      end
+
+      # def <<(conn)
+      #   @conn_mtx.synchronize do
+      #     origin, conns = @connections.find { |_orig, _| conn.origins.include?(origin) }
+      #     (conns || @connections[conn.origin.to_s]) << conn
+      #   end
+      # end
+
+      def empty?
+        @conn_mtx.synchronize { super }
+      end
+    end
+  end
+end

data/lib/httpx/pool.rb

@@ -7,7 +7,7 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
     extend Forwardable
 
     def_delegator :@timers, :after
@@ -17,7 +17,17 @@ module HTTPX
       @timers = Timers.new
       @selector = Selector.new
       @connections = []
-      @connected_connections = 0
+    end
+
+    def wrap
+      connections = @connections
+      @connections = []
+
+      begin
+        yield self
+      ensure
+        @connections.unshift(*connections)
+      end
     end
 
     def empty?
@@ -44,16 +54,15 @@ module HTTPX
         connection.emit(:error, e)
       end
     rescue Exception # rubocop:disable Lint/RescueException
-      @connections.each(&:reset)
+      @connections.each(&:force_reset)
       raise
     end
 
     def close(connections = @connections)
       return if connections.empty?
 
-      @timers.cancel
       connections = connections.reject(&:inflight?)
-      connections.each(&:close)
+      connections.each(&:terminate)
       next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
 
       # close resolvers
@@ -67,22 +76,39 @@ module HTTPX
         resolver.close unless resolver.closed?
       end
       # for https resolver
-      resolver_connections.each(&:close)
+      resolver_connections.each(&:terminate)
       next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
     end
 
     def init_connection(connection, _options)
-      resolve_connection(connection)
       connection.timers = @timers
-      connection.on(:open) do
-        @connected_connections += 1
-      end
       connection.on(:activate) do
         select_connection(connection)
       end
+      connection.on(:exhausted) do
+        case connection.state
+        when :closed
+          connection.idling
+          @connections << connection
+          select_connection(connection)
+        when :closing
+          connection.once(:close) do
+            connection.idling
+            @connections << connection
+            select_connection(connection)
+          end
+        end
+      end
+      connection.on(:close) do
+        unregister_connection(connection)
+      end
+      connection.on(:terminate) do
+        unregister_connection(connection, true)
+      end
+      resolve_connection(connection) unless connection.family
     end
 
-    def deactivate(connections)
+    def deactivate(*connections)
       connections.each do |connection|
         connection.deactivate
         deselect_connection(connection) if connection.state == :inactive
@@ -94,9 +120,24 @@ module HTTPX
     # maximize pipelining by opening as few connections as possible.
     #
     def find_connection(uri, options)
-      @connections.find do |connection|
+      conn = @connections.find do |connection|
         connection.match?(uri, options)
       end
+
+      return unless conn
+
+      case conn.state
+      when :closed
+        conn.idling
+        select_connection(conn)
+      when :closing
+        conn.once(:close) do
+          conn.idling
+          select_connection(conn)
+        end
+      end
+
+      conn
     end
 
     private
@@ -113,19 +154,61 @@ module HTTPX
         #    resolve a name (not the same as name being an IP, yet)
         # 2. when the connection is initialized with an external already open IO.
         #
+        connection.once(:connect_error, &connection.method(:handle_error))
         on_resolver_connection(connection)
         return
       end
 
       find_resolver_for(connection) do |resolver|
-        resolver << connection
+        resolver << try_clone_connection(connection, resolver.family)
         next if resolver.empty?
 
         select_connection(resolver)
       end
     end
 
+    def try_clone_connection(connection, family)
+      connection.family ||= family
+
+      return connection if connection.family == family
+
+      new_connection = connection.class.new(connection.origin, connection.options)
+      new_connection.family = family
+
+      connection.once(:tcp_open) { new_connection.force_reset }
+      connection.once(:connect_error) do |err|
+        if new_connection.connecting?
+          new_connection.merge(connection)
+          connection.emit(:cloned, new_connection)
+          connection.force_reset
+        else
+          connection.__send__(:handle_error, err)
+        end
+      end
+
+      new_connection.once(:tcp_open) do |new_conn|
+        if new_conn != connection
+          new_conn.merge(connection)
+          connection.force_reset
+        end
+      end
+      new_connection.once(:connect_error) do |err|
+        if connection.connecting?
+          # main connection has the requests
+          connection.merge(new_connection)
+          new_connection.emit(:cloned, connection)
+          new_connection.force_reset
+        else
+          new_connection.__send__(:handle_error, err)
+        end
+      end
+
+      init_connection(new_connection, connection.options)
+      new_connection
+    end
+
     def on_resolver_connection(connection)
+      @connections << connection unless @connections.include?(connection)
       found_connection = @connections.find do |ch|
         ch != connection && ch.mergeable?(connection)
       end
@@ -142,9 +225,9 @@ module HTTPX
     end
 
     def on_resolver_error(connection, error)
+      return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
+
       connection.emit(:error, error)
-      # must remove connection by hand, hasn't been started yet
-      unregister_connection(connection)
     end
 
     def on_resolver_close(resolver)
@@ -158,21 +241,12 @@ module HTTPX
     end
 
     def register_connection(connection)
-      if connection.state == :open
-        # if open, an IO was passed upstream, therefore
-        # consider it connected already.
-        @connected_connections += 1
-      end
       select_connection(connection)
-      connection.on(:close) do
-        unregister_connection(connection)
-      end
     end
 
-    def unregister_connection(connection)
-      @connections.delete(connection)
+    def unregister_connection(connection, cleanup = !connection.used?)
+      @connections.delete(connection) if cleanup
       deselect_connection(connection)
-      @connected_connections -= 1
     end
 
     def select_connection(connection)
@@ -186,6 +260,7 @@ module HTTPX
     def coalesce_connections(conn1, conn2)
       return register_connection(conn2) unless conn1.coalescable?(conn2)
 
+      conn2.emit(:tcp_open, conn1)
       conn1.merge(conn2)
       @connections.delete(conn2)
     end
@@ -201,7 +276,7 @@ module HTTPX
     def find_resolver_for(connection)
       connection_options = connection.options
       resolver_type = connection_options.resolver_class
-      resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
+      resolver_type = Resolver.resolver_for(resolver_type)
 
       @resolvers[resolver_type] ||= begin
         resolver_manager = if resolver_type.multi?

data/lib/httpx/punycode.rb

@@ -1,304 +1,22 @@
 # frozen_string_literal: true
 
 module HTTPX
-  begin
-    require "idnx"
+  module Punycode
+    module_function
 
-    module Punycode
-      module_function
+    begin
+      require "idnx"
 
       def encode_hostname(hostname)
         Idnx.to_punycode(hostname)
       end
-    end
-
-  rescue LoadError
-    # :nocov:
-    # -*- coding: utf-8 -*-
-    #--
-    # punycode.rb - PunyCode encoder for the Domain Name library
-    #
-    # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
-    #
-    # Ported from puny.c, a part of VeriSign XCode (encode/decode) IDN
-    # Library.
-    #
-    # Copyright (C) 2000-2002 Verisign Inc., All rights reserved.
-    #
-    # Redistribution and use in source and binary forms, with or
-    # without modification, are permitted provided that the following
-    # conditions are met:
-    #
-    #  1) Redistributions of source code must retain the above copyright
-    #     notice, this list of conditions and the following disclaimer.
-    #
-    #  2) Redistributions in binary form must reproduce the above copyright
-    #     notice, this list of conditions and the following disclaimer in
-    #     the documentation and/or other materials provided with the
-    #     distribution.
-    #
-    #  3) Neither the name of the VeriSign Inc. nor the names of its
-    #     contributors may be used to endorse or promote products derived
-    #     from this software without specific prior written permission.
-    #
-    # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
-    # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-    # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    # POSSIBILITY OF SUCH DAMAGE.
-    #
-    # This software is licensed under the BSD open source license. For more
-    # information visit www.opensource.org.
-    #
-    # Authors:
-    #  John Colosi (VeriSign)
-    #  Srikanth Veeramachaneni (VeriSign)
-    #  Nagesh Chigurupati (Verisign)
-    #  Praveen Srinivasan(Verisign)
-    #++
-    module Punycode
-      BASE = 36
-      TMIN = 1
-      TMAX = 26
-      SKEW = 38
-      DAMP = 700
-      INITIAL_BIAS = 72
-      INITIAL_N = 0x80
-      DELIMITER = "-"
-
-      MAXINT = (1 << 32) - 1
-
-      LOBASE = BASE - TMIN
-      CUTOFF = LOBASE * TMAX / 2
-
-      RE_NONBASIC = /[^\x00-\x7f]/.freeze
-
-      # Returns the numeric value of a basic code point (for use in
-      # representing integers) in the range 0 to base-1, or nil if cp
-      # is does not represent a value.
-      DECODE_DIGIT = {}.tap do |map|
-        # ASCII A..Z map to 0..25
-        # ASCII a..z map to 0..25
-        (0..25).each { |i| map[65 + i] = map[97 + i] = i }
-        # ASCII 0..9 map to 26..35
-        (26..35).each { |i| map[22 + i] = i }
-      end
-
-      # Returns the basic code point whose value (when used for
-      # representing integers) is d, which must be in the range 0 to
-      # BASE-1.  The lowercase form is used unless flag is true, in
-      # which case the uppercase form is used.  The behavior is
-      # undefined if flag is nonzero and digit d has no uppercase
-      # form.
-      ENCODE_DIGIT = proc { |d, flag|
-        (d + 22 + (d < 26 ? 75 : 0) - (flag ? (1 << 5) : 0)).chr
-        #  0..25 map to ASCII a..z or A..Z
-        # 26..35 map to ASCII 0..9
-      }
-
-      DOT = "."
-      PREFIX = "xn--"
-
-      # Most errors we raise are basically kind of ArgumentError.
-      class ArgumentError < ::ArgumentError; end
-      class BufferOverflowError < ArgumentError; end
-
-      module_function
-
-      # Encode a +string+ in Punycode
-      def encode(string)
-        input = string.unpack("U*")
-        output = +""
-
-        # Initialize the state
-        n = INITIAL_N
-        delta = 0
-        bias = INITIAL_BIAS
-
-        # Handle the basic code points
-        input.each { |cp| output << cp.chr if cp < 0x80 }
-
-        h = b = output.length
-
-        # h is the number of code points that have been handled, b is the
-        # number of basic code points, and out is the number of characters
-        # that have been output.
-
-        output << DELIMITER if b > 0
-
-        # Main encoding loop
-
-        while h < input.length
-          # All non-basic code points < n have been handled already.  Find
-          # the next larger one
-
-          m = MAXINT
-          input.each do |cp|
-            m = cp if (n...m) === cp
-          end
-
-          # Increase delta enough to advance the decoder's <n,i> state to
-          # <m,0>, but guard against overflow
-
-          delta += (m - n) * (h + 1)
-          raise BufferOverflowError if delta > MAXINT
-
-          n = m
-
-          input.each do |cp|
-            # AMC-ACE-Z can use this simplified version instead
-            if cp < n
-              delta += 1
-              raise BufferOverflowError if delta > MAXINT
-            elsif cp == n
-              # Represent delta as a generalized variable-length integer
-              q = delta
-              k = BASE
-              loop do
-                t = k <= bias ? TMIN : k - bias >= TMAX ? TMAX : k - bias
-                break if q < t
-
-                q, r = (q - t).divmod(BASE - t)
-                output << ENCODE_DIGIT[t + r, false]
-                k += BASE
-              end
-
-              output << ENCODE_DIGIT[q, false]
-
-              # Adapt the bias
-              delta = h == b ? delta / DAMP : delta >> 1
-              delta += delta / (h + 1)
-              bias = 0
-              while delta > CUTOFF
-                delta /= LOBASE
-                bias += BASE
-              end
-              bias += (LOBASE + 1) * delta / (delta + SKEW)
-
-              delta = 0
-              h += 1
-            end
-          end
-
-          delta += 1
-          n += 1
-        end
-
-        output
-      end
-
-      # Encode a hostname using IDN/Punycode algorithms
+    rescue LoadError
       def encode_hostname(hostname)
-        hostname.match(RE_NONBASIC) || (return hostname)
-
-        hostname.split(DOT).map do |name|
-          if name.match(RE_NONBASIC)
-            PREFIX + encode(name)
-          else
-            name
-          end
-        end.join(DOT)
-      end
-
-      # Decode a +string+ encoded in Punycode
-      def decode(string)
-        # Initialize the state
-        n = INITIAL_N
-        i = 0
-        bias = INITIAL_BIAS
-
-        if j = string.rindex(DELIMITER)
-          b = string[0...j]
-
-          b.match(RE_NONBASIC) &&
-            raise(ArgumentError, "Illegal character is found in basic part: #{string.inspect}")
-
-          # Handle the basic code points
-
-          output = b.unpack("U*")
-          u = string[(j + 1)..-1]
-        else
-          output = []
-          u = string
-        end
-
-        # Main decoding loop: Start just after the last delimiter if any
-        # basic code points were copied; start at the beginning
-        # otherwise.
-
-        input = u.unpack("C*")
-        input_length = input.length
-        h = 0
-        out = output.length
-
-        while h < input_length
-          # Decode a generalized variable-length integer into delta,
-          # which gets added to i.  The overflow checking is easier
-          # if we increase i as we go, then subtract off its starting
-          # value at the end to obtain delta.
-
-          oldi = i
-          w = 1
-          k = BASE
-
-          loop do
-            (digit = DECODE_DIGIT[input[h]]) ||
-              raise(ArgumentError, "Illegal character is found in non-basic part: #{string.inspect}")
-            h += 1
-            i += digit * w
-            raise BufferOverflowError if i > MAXINT
-
-            t = k <= bias ? TMIN : k - bias >= TMAX ? TMAX : k - bias
-            break if digit < t
-
-            w *= BASE - t
-            raise BufferOverflowError if w > MAXINT
-
-            k += BASE
-            (h < input_length) || raise(ArgumentError, "Malformed input given: #{string.inspect}")
-          end
-
-          # Adapt the bias
-          delta = oldi == 0 ? i / DAMP : (i - oldi) >> 1
-          delta += delta / (out + 1)
-          bias = 0
-          while delta > CUTOFF
-            delta /= LOBASE
-            bias += BASE
-          end
-          bias += (LOBASE + 1) * delta / (delta + SKEW)
-
-          # i was supposed to wrap around from out+1 to 0, incrementing
-          # n each time, so we'll fix that now:
-
-          q, i = i.divmod(out + 1)
-          n += q
-          raise BufferOverflowError if n > MAXINT
-
-          # Insert n at position i of the output:
-
-          output[i, 0] = n
-
-          out += 1
-          i += 1
-        end
-        output.pack("U*")
-      end
+        warn "#{hostname} cannot be converted to punycode. Install the " \
+             "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
 
-      # Decode a hostname using IDN/Punycode algorithms
-      def decode_hostname(hostname)
-        hostname.gsub(/(\A|#{Regexp.quote(DOT)})#{Regexp.quote(PREFIX)}([^#{Regexp.quote(DOT)}]*)/o) do
-          Regexp.last_match(1) << decode(Regexp.last_match(2))
-        end
+        hostname
       end
     end
-    # :nocov:
   end
-end
+end