httpx 0.11.3 → 0.14.0

data/lib/httpx/plugins/aws_sigv4.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require "set"
+require "aws-sdk-s3"
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds AWS Sigv4 authentication.
+    #
+    # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
+    #
+    module AWSSigV4
+      Credentials = Struct.new(:username, :password, :security_token)
+
+      class Signer
+        def initialize(
+          service:,
+          region:,
+          credentials: nil,
+          username: nil,
+          password: nil,
+          security_token: nil,
+          provider_prefix: "aws",
+          header_provider_field: "amz",
+          unsigned_headers: [],
+          apply_checksum_header: true,
+          algorithm: "SHA256"
+        )
+          @credentials = credentials || Credentials.new(username, password, security_token)
+          @service = service
+          @region = region
+
+          @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
+          @unsigned_headers << "authorization"
+          @unsigned_headers << "x-amzn-trace-id"
+          @unsigned_headers << "expect"
+
+          @apply_checksum_header = apply_checksum_header
+          @provider_prefix = provider_prefix
+          @header_provider_field = header_provider_field
+
+          @algorithm = algorithm
+        end
+
+        def sign!(request)
+          lower_provider_prefix = "#{@provider_prefix}4"
+          upper_provider_prefix = lower_provider_prefix.upcase
+
+          downcased_algorithm = @algorithm.downcase
+
+          datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
+          date = datetime[0, 8]
+
+          content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
+
+          request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
+          request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
+
+          signature_headers = request.headers.each.reject do |k, _|
+            @unsigned_headers.include?(k)
+          end
+          # aws sigv4 needs to declare the host, regardless of protocol version
+          signature_headers << ["host", request.authority] unless request.headers.key?("host")
+          signature_headers.sort_by!(&:first)
+
+          signed_headers = signature_headers.map(&:first).join(";")
+
+          canonical_headers = signature_headers.map do |k, v|
+            # eliminate whitespace between value fields, unless it's a quoted value
+            "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
+          end.join
+
+          # canonical request
+          creq = "#{request.verb.to_s.upcase}" \
+            "\n#{request.canonical_path}" \
+            "\n#{request.canonical_query}" \
+            "\n#{canonical_headers}" \
+            "\n#{signed_headers}" \
+            "\n#{content_hashed}"
+
+          credential_scope = "#{date}" \
+            "/#{@region}" \
+            "/#{@service}" \
+            "/#{lower_provider_prefix}_request"
+
+          algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
+          # string to sign
+          sts = "#{algo_line}" \
+                "\n#{datetime}" \
+                "\n#{credential_scope}" \
+                "\n#{hexdigest(creq)}"
+
+          # signature
+          k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
+          k_region = hmac(k_date, @region)
+          k_service = hmac(k_region, @service)
+          k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
+          sig = hexhmac(k_credentials, sts)
+
+          credential = "#{@credentials.username}/#{credential_scope}"
+          # apply signature
+          request.headers["authorization"] =
+            "#{algo_line} " \
+            "Credential=#{credential}, " \
+            "SignedHeaders=#{signed_headers}, " \
+            "Signature=#{sig}"
+        end
+
+        private
+
+        def hexdigest(value)
+          if value.respond_to?(:to_path)
+            # files, pathnames
+            OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
+          elsif value.respond_to?(:each)
+            digest = OpenSSL::Digest.new(@algorithm)
+
+            mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
+              buffer << chunk
+              break if buffer.bytesize >= 1024 * 1024
+            end
+
+            digest.update(mb_buffer)
+            value.rewind
+            digest.hexdigest
+          else
+            OpenSSL::Digest.new(@algorithm).hexdigest(value)
+          end
+        end
+
+        def hmac(key, value)
+          OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
+        end
+
+        def hexhmac(key, value)
+          OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
+        end
+      end
+
+      class << self
+        def extra_options(options)
+          Class.new(options.class) do
+            def_option(:sigv4_signer, <<-OUT)
+              value.is_a?(#{Signer}) ? value : #{Signer}.new(value)
+            OUT
+          end.new(options)
+        end
+
+        def load_dependencies(klass)
+          require "digest/sha2"
+          require "openssl"
+          klass.plugin(:expect)
+          klass.plugin(:compression)
+        end
+      end
+
+      module InstanceMethods
+        def aws_sigv4_authentication(**options)
+          with(sigv4_signer: Signer.new(**options))
+        end
+
+        def build_request(*, _)
+          request = super
+
+          return request if request.headers.key?("authorization")
+
+          signer = request.options.sigv4_signer
+
+          return request unless signer
+
+          signer.sign!(request)
+
+          request
+        end
+      end
+
+      module RequestMethods
+        def canonical_path
+          path = uri.path.dup
+          path << "/" if path.empty?
+          path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
+        end
+
+        def canonical_query
+          params = query.split("&")
+          # params = params.map { |p| p.match(/=/) ? p : p + '=' }
+          # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+          # Sort the parameter names by character code point in ascending order.
+          # Parameters with duplicate names should be sorted by value.
+          #
+          # Default sort <=> in JRuby will swap members
+          # occasionally when <=> is 0 (considered still sorted), but this
+          # causes our normalized query string to not match the sent querystring.
+          # When names match, we then sort by their values.  When values also
+          # match then we sort by their original order
+          params.each.with_index.sort do |a, b|
+            a, a_offset = a
+            b, b_offset = b
+            a_name, a_value = a.split("=")
+            b_name, b_value = b.split("=")
+            if a_name == b_name
+              if a_value == b_value
+                a_offset <=> b_offset
+              else
+                a_value <=> b_value
+              end
+            else
+              a_name <=> b_name
+            end
+          end.map(&:first).join("&")
+        end
+      end
+    end
+    register_plugin :aws_sigv4, AWSSigV4
+  end
+end

data/lib/httpx/plugins/basic_authentication.rb

@@ -8,9 +8,14 @@ module HTTPX
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Authentication#basic-authentication
     #
     module BasicAuthentication
-      def self.load_dependencies(klass)
-        require "base64"
-        klass.plugin(:authentication)
+      class << self
+        def load_dependencies(_klass)
+          require "base64"
+        end
+
+        def configure(klass)
+          klass.plugin(:authentication)
+        end
       end
 
       module InstanceMethods

data/lib/httpx/plugins/compression.rb

@@ -13,23 +13,31 @@ module HTTPX
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Compression
     #
     module Compression
-      extend Registry
-
       class << self
-        def load_dependencies(klass)
+        def configure(klass)
           klass.plugin(:"compression/gzip")
           klass.plugin(:"compression/deflate")
         end
 
         def extra_options(options)
+          encodings = Module.new do
+            extend Registry
+          end
+
           Class.new(options.class) do
-            def_option(:compression_threshold_size) do |bytes|
-              bytes = Integer(bytes)
+            def_option(:compression_threshold_size, <<-OUT)
+              bytes = Integer(value)
               raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
 
               bytes
-            end
-          end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
+            OUT
+
+            def_option(:encodings, <<-OUT)
+              raise Error, ":encodings must be a registry" unless value.respond_to?(:registry)
+
+              value
+            OUT
+          end.new(options).merge(encodings: encodings)
         end
       end
 
@@ -37,7 +45,11 @@ module HTTPX
         def initialize(*)
           super
           # forego compression in the Range cases
-          @headers.delete("accept-encoding") if @headers.key?("range")
+          if @headers.key?("range")
+            @headers.delete("accept-encoding")
+          else
+            @headers["accept-encoding"] ||= @options.encodings.registry.keys
+          end
         end
       end
 
@@ -52,16 +64,16 @@ module HTTPX
           @headers.get("content-encoding").each do |encoding|
             next if encoding == "identity"
 
-            @body = Encoder.new(@body, Compression.registry(encoding).deflater)
+            @body = Encoder.new(@body, options.encodings.registry(encoding).deflater)
           end
-          @headers["content-length"] = @body.bytesize unless chunked?
+          @headers["content-length"] = @body.bytesize unless unbounded_body?
         end
       end
 
       module ResponseBodyMethods
         attr_reader :encodings
 
-        def initialize(*, **)
+        def initialize(*)
           @encodings = []
 
           super
@@ -80,7 +92,7 @@ module HTTPX
           @_inflaters = @headers.get("content-encoding").map do |encoding|
             next if encoding == "identity"
 
-            inflater = Compression.registry(encoding).inflater(compressed_length)
+            inflater = @options.encodings.registry(encoding).inflater(compressed_length)
             # do not uncompress if there is no decoder available. In fact, we can't reliably
             # continue decompressing beyond that, so ignore.
             break unless inflater

data/lib/httpx/plugins/compression/brotli.rb

@@ -4,24 +4,27 @@ module HTTPX
   module Plugins
     module Compression
       module Brotli
-        def self.load_dependencies(klass)
-          klass.plugin(:compression)
-          require "brotli"
-        end
+        class << self
+          def load_dependencies(_klass)
+            require "brotli"
+          end
 
-        def self.configure(*)
-          Compression.register "br", self
+          def configure(klass)
+            klass.plugin(:compression)
+            klass.default_options.encodings.register "br", self
+          end
         end
 
         module Deflater
           module_function
 
-          def deflate(raw, buffer, chunk_size:)
+          def deflate(raw, buffer = "".b, chunk_size: 16_384)
             while (chunk = raw.read(chunk_size))
               compressed = ::Brotli.deflate(chunk)
               buffer << compressed
               yield compressed if block_given?
             end
+            buffer
           end
         end
 

data/lib/httpx/plugins/compression/deflate.rb

@@ -4,24 +4,21 @@ module HTTPX
   module Plugins
     module Compression
       module Deflate
-        def self.load_dependencies(klass)
+        def self.load_dependencies(_klass)
           require "stringio"
           require "zlib"
-          klass.plugin(:"compression/gzip")
         end
 
-        def self.configure(*)
-          Compression.register "deflate", self
+        def self.configure(klass)
+          klass.plugin(:"compression/gzip")
+          klass.default_options.encodings.register "deflate", self
         end
 
         module Deflater
           module_function
 
-          def deflate(raw, buffer, chunk_size:)
-            deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,
-                                         Zlib::MAX_WBITS,
-                                         Zlib::MAX_MEM_LEVEL,
-                                         Zlib::HUFFMAN_ONLY)
+          def deflate(raw, buffer = "".b, chunk_size: 16_384)
+            deflater = Zlib::Deflate.new
             while (chunk = raw.read(chunk_size))
               compressed = deflater.deflate(chunk)
               buffer << compressed
@@ -30,8 +27,9 @@ module HTTPX
             last = deflater.finish
             buffer << last
             yield last if block_given?
+            buffer
           ensure
-            deflater.close
+            deflater.close if deflater
           end
         end
 

data/lib/httpx/plugins/compression/gzip.rb

@@ -10,8 +10,8 @@ module HTTPX
           require "zlib"
         end
 
-        def self.configure(*)
-          Compression.register "gzip", self
+        def self.configure(klass)
+          klass.default_options.encodings.register "gzip", self
         end
 
         class Deflater
@@ -19,7 +19,7 @@ module HTTPX
             @compressed_chunk = "".b
           end
 
-          def deflate(raw, buffer, chunk_size:)
+          def deflate(raw, buffer = "".b, chunk_size: 16_384)
             gzip = Zlib::GzipWriter.new(self)
 
             begin
@@ -38,6 +38,7 @@ module HTTPX
 
             buffer << compressed
             yield compressed if block_given?
+            buffer
           end
 
           private

data/lib/httpx/plugins/cookies.rb

@@ -20,13 +20,9 @@ module HTTPX
 
       def self.extra_options(options)
         Class.new(options.class) do
-          def_option(:cookies) do |cookies|
-            if cookies.is_a?(Jar)
-              cookies
-            else
-              Jar.new(cookies)
-            end
-          end
+          def_option(:cookies, <<-OUT)
+            value.is_a?(#{Jar}) ? value : #{Jar}.new(value)
+          OUT
         end.new(options)
       end
 

data/lib/httpx/plugins/digest_authentication.rb

@@ -14,11 +14,11 @@ module HTTPX
 
       def self.extra_options(options)
         Class.new(options.class) do
-          def_option(:digest) do |digest|
-            raise Error, ":digest must be a Digest" unless digest.is_a?(Digest)
+          def_option(:digest, <<-OUT)
+            raise Error, ":digest must be a Digest" unless value.is_a?(#{Digest})
 
-            digest
-          end
+            value
+          OUT
         end.new(options)
       end
 
@@ -29,7 +29,7 @@ module HTTPX
 
       module InstanceMethods
         def digest_authentication(user, password)
-          branch(default_options.with_digest(Digest.new(user, password)))
+          with(digest: Digest.new(user, password))
         end
 
         alias_method :digest_auth, :digest_authentication