httpx 0.14.2 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5049a0cfb698c15800ea618a2775831183765f24ccb946c2adfb6b067c976c4d
-  data.tar.gz: 498a626981251e0ffbff3dde9d63fcc57e0b3183d29fafbc1ecd921d3bfe8172
+  metadata.gz: 71d64789176a94e104957666f409c327a17827e4384e1387c14c2471e1b98994
+  data.tar.gz: 0a516128ac747a041ae8e5d86da78c7687d124f830850484ed372c83d67e4745
 SHA512:
-  metadata.gz: 6fb23f5a45bd521bf30c25b8e30af3b35e4359093a470fb29f9ab6c8f5d0f09674a08b28ba2c20f92c51fe46702f4f3751b855b6b31f202be01728f56273d6a2
-  data.tar.gz: f64d1057e2483502dc3751a74489cfec0e7672d906f5901a81f98227c98ee907dc6be0b8d0ad7f4c4688546a2a7c22886f437807e28ba5b2a5e09c6b588de023
+  metadata.gz: 83f6d5041140e1670e7175db3d9a3269a3766a5b8b29076ba7ee4025ef035411a4458c5df25cce4d70a10fd34fdc1cbda30fa315b3a77c585d58672ec35f737d
+  data.tar.gz: '05298bf9e3fc41c8fe7315558a6842af2dbe915c8f5cfb119ca019cc8e83dbde93a9d57222bbf0098603b3fde00a5e8f1de05f4d62cf5fe81a6f7213a8c8107d'

data/doc/release_notes/0_13_2.md

@@ -1,4 +1,4 @@
-# 0.13.1
+# 0.13.2
 
 ## Improvements
 

data/doc/release_notes/0_14_3.md

@@ -0,0 +1,5 @@
+# 0.14.3
+
+## Bugfixes
+
+* fixed: HTTP/1 "connection: close" header was "leaking" into subsequent redirect follow, including HTTP/2 requests which would fail due to the invalid header.

data/doc/release_notes/0_14_4.md

@@ -0,0 +1,5 @@
+# 0.14.4
+
+## Bugfixes
+
+* The HTTP/1 handler was miscalculating the last request for a given connection, and potentially freezing it.

data/doc/release_notes/0_14_5.md

@@ -0,0 +1,11 @@
+# 0.14.5
+
+## Bugfixes
+
+* After a connection had been initiated, sending multiple concurrent requests (ex: `open_httpx.request(req1, req2, req3)`) could freeze; this happened when the first request would fill the write buffer (like a file upload request), and the subsequent requests would never be buffered afterwards; this was fixed by making pending requests flushing a part of a connection's consumption loop.
+* Fixing v0.14.1's fixed bug again; The HTTP/1 "Connection: close" header was not being set in the last possible request on a connection, due to ann off-by-one error on connection bookkeeping;
+* HTTP/1 connections didn't respect a server-set max nunmber of requests after a reconnect; Fixed by making this accounting part of the reset process;
+
+## Chore
+
+* Added regression test suite, which reproduce reported bugs before the fix (backported all 0.14.x releases here)

data/doc/release_notes/0_15_0.md

@@ -0,0 +1,53 @@
+# 0.15.0
+
+## Features
+
+### HTTP response pattern-matching (ruby 3 only)
+
+You can now apply pattern matching in responses:
+
+```ruby
+case response = HTTPX.get("https://google.com")
+in { status: 200..399, headers: [*, ["x-special-token", token], *], body: }
+  puts "success: #{body.to_s}"
+in { status: 400..499, body: }
+  puts "client error: #{body.to_s}"
+in { status: 500.., body: }
+  puts "server error: #{body.to_s}"
+in { error: error }
+  puts "error: #{error.message}"
+else
+  raise "unexpected: #{response}"
+end
+```
+
+### NTLM Authentication
+
+A new plugin, `:ntml_authentication`, is now available. Like the name suggests, it allows authenticating requests via [NTLM](https://docs.microsoft.com/en-us/windows-server/security/kerberos/ntlm-overview).
+
+```ruby
+ntlm_http = HTTPX.plugin(:ntlm_authentication)
+
+ntlm.ntlm_authentication("user", "password").get("http://protected-area-requiring-ntlm.net")
+# or for a specific domain
+ntlm.ntlm_authentication("user", "password", "Domain\\User").get("http://protected-area-requiring-ntlm.net")
+```
+
+## Improvemennts
+
+A new timeout option, `settings_timeout`, is supported for the HTTP/2 handshake; after the TCP and TLS handshakes are complete, and initiating the HTTP/2 handshake, the client terminates the connection with SETTINGS_TIMEOUT error code, if it doesn't receive the server settings for the amount of seconds set in `settings_timeout` (by default, 10 seconds).
+
+```ruby
+# if you want to change
+HTTPX.with(timeout: {settings_timeout: 5})....
+
+```
+
+IDNA 2008 support is now possibly, by integrating [idnx](https://github.com/HoneyryderChuck/idnx) into your dependencies:
+
+
+```ruby
+# in Gemfile
+gem "httpx"
+gem "idnx"
+```

data/doc/release_notes/0_15_1.md

@@ -0,0 +1,8 @@
+# 0.15.1
+
+## Bugfixes
+
+Fixed HTTP/1 connection accounting on requests:
+
+* when persistent, Connection: close will be set based on the request position on the batch against the allowed requests on the open connection.
+* when not persistent, Connnection: close will be set on the last request of the batch, being the batch a subset based on allowed requests, or the whole of it.

data/lib/httpx.rb

@@ -6,6 +6,7 @@ require "httpx/extensions"
 
 require "httpx/errors"
 require "httpx/utils"
+require "httpx/punycode"
 require "httpx/domain_name"
 require "httpx/altsvc"
 require "httpx/callbacks"

data/lib/httpx/connection.rb

@@ -51,7 +51,7 @@ module HTTPX
     def initialize(type, uri, options)
       @type = type
       @origins = [uri.origin]
-      @origin = Utils.uri(uri.origin)
+      @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
       @window_size = @options.window_size
       @read_buffer = Buffer.new(BUFFER_SIZE)
@@ -254,6 +254,8 @@ module HTTPX
     end
 
     def consume
+      return unless @io
+
       catch(:called) do
         epiped = false
         loop do
@@ -311,7 +313,7 @@ module HTTPX
 
             # exit #consume altogether if all outstanding requests have been dealt with
             return if @pending.size.zero? && @inflight.zero?
-          end unless (interests == :w || @state == :closing) && !epiped
+          end unless (interests.nil? || interests == :w || @state == :closing) && !epiped
 
           #
           # tight write loop.
@@ -360,6 +362,8 @@ module HTTPX
             write_drained = false
           end unless interests == :r
 
+          send_pending if @state == :open
+
           # return if socket is drained
           next unless (interests != :r || read_drained) &&
                       (interests != :w || write_drained)
@@ -422,6 +426,9 @@ module HTTPX
           emit(:close)
         end
       end
+      parser.on(:close_handshake) do
+        consume
+      end
       parser.on(:reset) do
         if parser.empty?
           reset
@@ -434,6 +441,9 @@ module HTTPX
           transition(:open)
         end
       end
+      parser.on(:current_timeout) do
+        @current_timeout = @timeout = parser.timeout
+      end
       parser.on(:timeout) do |tout|
         @timeout = tout
       end
@@ -464,10 +474,11 @@ module HTTPX
 
         send_pending
 
-        @timeout = @current_timeout = @options.timeout[:operation_timeout]
+        @timeout = @current_timeout = parser.timeout
         emit(:open)
       when :closing
         return unless @state == :open
+
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?

data/lib/httpx/connection/http1.rb

@@ -24,6 +24,10 @@ module HTTPX
       @handshake_completed = false
     end
 
+    def timeout
+      @options.timeout[:operation_timeout]
+    end
+
     def interests
       # this means we're processing incoming response already
       return :r if @request
@@ -40,6 +44,7 @@ module HTTPX
     def reset
       @max_requests = @options.max_requests || MAX_REQUESTS
       @parser.reset!
+      @handshake_completed = false
     end
 
     def close
@@ -80,7 +85,6 @@ module HTTPX
         break if idx >= concurrent_requests_limit
         next if request.state == :done
 
-        request.headers["connection"] ||= request.options.persistent || idx < requests_limit - 1 ? "keep-alive" : "close"
         handle(request)
       end
     end
@@ -259,9 +263,29 @@ module HTTPX
         request.chunk!
       end
 
+      connection = if request.options.persistent
+        # when in a persistent connection, the request can't be at
+        # the edge of a renegotiation
+        if @requests.index(request) + 1 < @max_requests
+          "keep-alive"
+        else
+          "close"
+        end
+      else
+        # when it's not a persistent connection, it sets "Connection: close" always
+        # on the last request of the possible batch (either allowed max requests,
+        # or if smaller, the size of the batch itself)
+        requests_limit = [@max_requests, @requests.size].min
+        if request != @requests[requests_limit - 1]
+          "keep-alive"
+        else
+          "close"
+        end
+      end
+
       {
         "host" => (request.headers["host"] || request.authority),
-        "connection" => (request.headers["connection"] || (request.options.persistent ? "keep-alive" : "close")),
+        "connection" => connection,
       }
     end
 

data/lib/httpx/connection/http2.rb

@@ -34,6 +34,12 @@ module HTTPX
       init_connection
     end
 
+    def timeout
+      return @options.timeout[:operation_timeout] if @handshake_completed
+
+      @options.timeout[:settings_timeout]
+    end
+
     def interests
       # waiting for WINDOW_UPDATE frames
       return :r if @buffer.full?
@@ -117,6 +123,13 @@ module HTTPX
     end
 
     def handle_error(ex)
+      if ex.instance_of?(TimeoutError) && !@handshake_completed && @connection.state != :closed
+        @connection.goaway(:settings_timeout, "closing due to settings timeout")
+        emit(:close_handshake)
+        settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
+        settings_ex.set_backtrace(ex.backtrace)
+        ex = settings_ex
+      end
       @streams.each_key do |request|
         emit(:error, request, ex)
       end
@@ -312,6 +325,7 @@ module HTTPX
 
     def on_settings(*)
       @handshake_completed = true
+      emit(:current_timeout)
 
       if @max_requests.zero?
         @max_requests = @connection.remote_settings[:settings_max_concurrent_streams]

data/lib/httpx/domain_name.rb

@@ -144,295 +144,5 @@ module HTTPX
         1
       end
     end
-
-    # :nocov:
-    # rubocop:disable all
-    # -*- coding: utf-8 -*-
-    #--
-    # punycode.rb - PunyCode encoder for the Domain Name library
-    #
-    # Copyright (C) 2011-2017 Akinori MUSHA, All rights reserved.
-    #
-    # Ported from puny.c, a part of VeriSign XCode (encode/decode) IDN
-    # Library.
-    #
-    # Copyright (C) 2000-2002 Verisign Inc., All rights reserved.
-    #
-    # Redistribution and use in source and binary forms, with or
-    # without modification, are permitted provided that the following
-    # conditions are met:
-    #
-    #  1) Redistributions of source code must retain the above copyright
-    #     notice, this list of conditions and the following disclaimer.
-    #
-    #  2) Redistributions in binary form must reproduce the above copyright
-    #     notice, this list of conditions and the following disclaimer in
-    #     the documentation and/or other materials provided with the
-    #     distribution.
-    #
-    #  3) Neither the name of the VeriSign Inc. nor the names of its
-    #     contributors may be used to endorse or promote products derived
-    #     from this software without specific prior written permission.
-    #
-    # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-    # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-    # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-    # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-    # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
-    # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-    # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-    # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-    # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    # POSSIBILITY OF SUCH DAMAGE.
-    #
-    # This software is licensed under the BSD open source license. For more
-    # information visit www.opensource.org.
-    #
-    # Authors:
-    #  John Colosi (VeriSign)
-    #  Srikanth Veeramachaneni (VeriSign)
-    #  Nagesh Chigurupati (Verisign)
-    #  Praveen Srinivasan(Verisign)
-    #++
-    module Punycode
-      BASE = 36
-      TMIN = 1
-      TMAX = 26
-      SKEW = 38
-      DAMP = 700
-      INITIAL_BIAS = 72
-      INITIAL_N = 0x80
-      DELIMITER = "-"
-
-      MAXINT = (1 << 32) - 1
-
-      LOBASE = BASE - TMIN
-      CUTOFF = LOBASE * TMAX / 2
-
-      RE_NONBASIC = /[^\x00-\x7f]/.freeze
-
-      # Returns the numeric value of a basic code point (for use in
-      # representing integers) in the range 0 to base-1, or nil if cp
-      # is does not represent a value.
-      DECODE_DIGIT = {}.tap do |map|
-        # ASCII A..Z map to 0..25
-        # ASCII a..z map to 0..25
-        (0..25).each { |i| map[65 + i] = map[97 + i] = i }
-        # ASCII 0..9 map to 26..35
-        (26..35).each { |i| map[22 + i] = i }
-      end
-
-      # Returns the basic code point whose value (when used for
-      # representing integers) is d, which must be in the range 0 to
-      # BASE-1.  The lowercase form is used unless flag is true, in
-      # which case the uppercase form is used.  The behavior is
-      # undefined if flag is nonzero and digit d has no uppercase
-      # form.
-      ENCODE_DIGIT = proc { |d, flag|
-        (d + 22 + (d < 26 ? 75 : 0) - (flag ? (1 << 5) : 0)).chr
-        #  0..25 map to ASCII a..z or A..Z
-        # 26..35 map to ASCII 0..9
-      }
-
-      DOT = "."
-      PREFIX = "xn--"
-
-      # Most errors we raise are basically kind of ArgumentError.
-      class ArgumentError < ::ArgumentError; end
-      class BufferOverflowError < ArgumentError; end
-
-      class << self
-        # Encode a +string+ in Punycode
-        def encode(string)
-          input = string.unpack("U*")
-          output = +""
-
-          # Initialize the state
-          n = INITIAL_N
-          delta = 0
-          bias = INITIAL_BIAS
-
-          # Handle the basic code points
-          input.each { |cp| output << cp.chr if cp < 0x80 }
-
-          h = b = output.length
-
-          # h is the number of code points that have been handled, b is the
-          # number of basic code points, and out is the number of characters
-          # that have been output.
-
-          output << DELIMITER if b > 0
-
-          # Main encoding loop
-
-          while h < input.length
-            # All non-basic code points < n have been handled already.  Find
-            # the next larger one
-
-            m = MAXINT
-            input.each do |cp|
-              m = cp if (n...m) === cp
-            end
-
-            # Increase delta enough to advance the decoder's <n,i> state to
-            # <m,0>, but guard against overflow
-
-            delta += (m - n) * (h + 1)
-            raise BufferOverflowError if delta > MAXINT
-
-            n = m
-
-            input.each do |cp|
-              # AMC-ACE-Z can use this simplified version instead
-              if cp < n
-                delta += 1
-                raise BufferOverflowError if delta > MAXINT
-              elsif cp == n
-                # Represent delta as a generalized variable-length integer
-                q = delta
-                k = BASE
-                loop do
-                  t = k <= bias ? TMIN : k - bias >= TMAX ? TMAX : k - bias
-                  break if q < t
-
-                  q, r = (q - t).divmod(BASE - t)
-                  output << ENCODE_DIGIT[t + r, false]
-                  k += BASE
-                end
-
-                output << ENCODE_DIGIT[q, false]
-
-                # Adapt the bias
-                delta = h == b ? delta / DAMP : delta >> 1
-                delta += delta / (h + 1)
-                bias = 0
-                while delta > CUTOFF
-                  delta /= LOBASE
-                  bias += BASE
-                end
-                bias += (LOBASE + 1) * delta / (delta + SKEW)
-
-                delta = 0
-                h += 1
-              end
-            end
-
-            delta += 1
-            n += 1
-          end
-
-          output
-        end
-
-        # Encode a hostname using IDN/Punycode algorithms
-        def encode_hostname(hostname)
-          hostname.match(RE_NONBASIC) || (return hostname)
-
-          hostname.split(DOT).map do |name|
-            if name.match(RE_NONBASIC)
-              PREFIX + encode(name)
-            else
-              name
-            end
-          end.join(DOT)
-        end
-
-        # Decode a +string+ encoded in Punycode
-        def decode(string)
-          # Initialize the state
-          n = INITIAL_N
-          i = 0
-          bias = INITIAL_BIAS
-
-          if j = string.rindex(DELIMITER)
-            b = string[0...j]
-
-            b.match(RE_NONBASIC) &&
-              raise(ArgumentError, "Illegal character is found in basic part: #{string.inspect}")
-
-            # Handle the basic code points
-
-            output = b.unpack("U*")
-            u = string[(j + 1)..-1]
-          else
-            output = []
-            u = string
-          end
-
-          # Main decoding loop: Start just after the last delimiter if any
-          # basic code points were copied; start at the beginning
-          # otherwise.
-
-          input = u.unpack("C*")
-          input_length = input.length
-          h = 0
-          out = output.length
-
-          while h < input_length
-            # Decode a generalized variable-length integer into delta,
-            # which gets added to i.  The overflow checking is easier
-            # if we increase i as we go, then subtract off its starting
-            # value at the end to obtain delta.
-
-            oldi = i
-            w = 1
-            k = BASE
-
-            loop do
-              (digit = DECODE_DIGIT[input[h]]) ||
-                raise(ArgumentError, "Illegal character is found in non-basic part: #{string.inspect}")
-              h += 1
-              i += digit * w
-              raise BufferOverflowError if i > MAXINT
-
-              t = k <= bias ? TMIN : k - bias >= TMAX ? TMAX : k - bias
-              break if digit < t
-
-              w *= BASE - t
-              raise BufferOverflowError if w > MAXINT
-
-              k += BASE
-              (h < input_length) || raise(ArgumentError, "Malformed input given: #{string.inspect}")
-            end
-
-            # Adapt the bias
-            delta = oldi == 0 ? i / DAMP : (i - oldi) >> 1
-            delta += delta / (out + 1)
-            bias = 0
-            while delta > CUTOFF
-              delta /= LOBASE
-              bias += BASE
-            end
-            bias += (LOBASE + 1) * delta / (delta + SKEW)
-
-            # i was supposed to wrap around from out+1 to 0, incrementing
-            # n each time, so we'll fix that now:
-
-            q, i = i.divmod(out + 1)
-            n += q
-            raise BufferOverflowError if n > MAXINT
-
-            # Insert n at position i of the output:
-
-            output[i, 0] = n
-
-            out += 1
-            i += 1
-          end
-          output.pack("U*")
-        end
-
-        # Decode a hostname using IDN/Punycode algorithms
-        def decode_hostname(hostname)
-          hostname.gsub(/(\A|#{Regexp.quote(DOT)})#{Regexp.quote(PREFIX)}([^#{Regexp.quote(DOT)}]*)/o) do
-            Regexp.last_match(1) << decode(Regexp.last_match(2))
-          end
-        end
-      end
-      # rubocop:enable all
-      # :nocov:
-    end
   end
 end