httpx 0.3.1 → 0.4.0

data/lib/httpx/parser/http1.rb

@@ -57,13 +57,16 @@ module HTTPX
       def parse_headline
         idx = @buffer.index("\n")
         return unless idx
+
         (m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
           raise(Error, "wrong head line format")
         version, code, _ = m.captures
         raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
+
         @http_version = version.split(".").map(&:to_i)
         @status_code = code.to_i
         raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
+
         @buffer.slice!(0, idx + 1)
         nextstate(:headers)
       end
@@ -78,6 +81,7 @@ module HTTPX
               prepare_data(headers)
               @observer.on_headers(headers)
               return unless @state == :headers
+
               # state might have been reset
               # in the :headers callback
               nextstate(:data)
@@ -93,12 +97,15 @@ module HTTPX
           end
           separator_index = line.index(@header_separator)
           raise Error, "wrong header format" unless separator_index
+
           key = line[0..separator_index - 1]
           raise Error, "wrong header format" if key.start_with?("\s", "\t")
+
           key.strip!
           value = line[separator_index + 1..-1]
           value.strip!
           raise Error, "wrong header format" if value.nil?
+
           (headers[key.downcase] ||= []) << value
         end
       end
@@ -118,6 +125,7 @@ module HTTPX
           @buffer.clear
         end
         return unless no_more_data?
+
         @buffer = @buffer.to_s
         if @_has_trailers
           nextstate(:trailers)

data/lib/httpx/plugins/authentication.rb

@@ -2,6 +2,10 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds a shim #authentication method to the session, which will fill
+    # the HTTP Authorization header.
+    #
     module Authentication
       module InstanceMethods
         def authentication(token)

data/lib/httpx/plugins/basic_authentication.rb

@@ -2,6 +2,10 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds helper methods to implement HTTP Basic Auth
+    # https://tools.ietf.org/html/rfc7617
+    #
     module BasicAuthentication
       def self.load_dependencies(klass, *)
         require "base64"

data/lib/httpx/plugins/compression.rb

@@ -2,23 +2,30 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds compression support. Namely it:
+    #
+    # * Compresses the request body when passed a supported "Content-Encoding" mime-type;
+    # * Decompresses the response body from a supported "Content-Encoding" mime-type;
+    #
+    # It supports both *gzip* and *deflate*.
+    #
     module Compression
       extend Registry
-      def self.configure(klass, *)
+      def self.load_dependencies(klass, *)
         klass.plugin(:"compression/gzip")
         klass.plugin(:"compression/deflate")
       end
 
-      module InstanceMethods
-        def initialize(opts = {})
-          super(opts.merge(headers: { "accept-encoding" => Compression.registry.keys }))
-        end
+      def self.extra_options(options)
+        options.merge(headers: { "accept-encoding" => Compression.registry.keys })
       end
 
       module RequestBodyMethods
         def initialize(*)
           super
           return if @body.nil?
+
           @headers.get("content-encoding").each do |encoding|
             @body = Encoder.new(@body, Compression.registry(encoding).encoder)
           end
@@ -29,6 +36,9 @@ module HTTPX
       module ResponseBodyMethods
         def initialize(*)
           super
+
+          return unless @headers.key?("content-encoding")
+
           @_decoders = @headers.get("content-encoding").map do |encoding|
             Compression.registry(encoding).decoder
           end
@@ -40,6 +50,8 @@ module HTTPX
         end
 
         def write(chunk)
+          return super unless defined?(@_compressed_length)
+
           @_compressed_length -= chunk.bytesize
           chunk = decompress(chunk)
           super(chunk)
@@ -47,6 +59,9 @@ module HTTPX
 
         def close
           super
+
+          return unless defined?(@_decoders)
+
           @_decoders.each(&:close)
         end
 
@@ -70,6 +85,7 @@ module HTTPX
 
         def each(&blk)
           return enum_for(__method__) unless block_given?
+
           unless @buffer.size.zero?
             @buffer.rewind
             return @buffer.each(&blk)
@@ -97,6 +113,7 @@ module HTTPX
 
         def deflate(&blk)
           return unless @buffer.size.zero?
+
           @body.rewind
           @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
         end

data/lib/httpx/plugins/cookies.rb

@@ -1,60 +1,113 @@
 # frozen_string_literal: true
 
+require "forwardable"
+
 module HTTPX
   module Plugins
+    #
+    # This plugin implements a persistent cookie jar for the duration of a session.
+    #
+    # It also adds a *#cookies* helper, so that you can pre-fill the cookies of a session.
+    #
     module Cookies
+      using URIExtensions
+
+      def self.extra_options(options)
+        Class.new(options.class) do
+          def_option(:cookies) do |cookies|
+            return cookies if cookies.is_a?(Store)
+
+            Store.new(cookies)
+          end
+        end.new(options)
+      end
+
+      class Store
+        def initialize(cookies = nil)
+          @store = Hash.new { |hash, origin| hash[origin] = HTTP::CookieJar.new }
+          return unless cookies
+
+          cookies = cookies.split(/ *; */) if cookies.is_a?(String)
+          @default_cookies = cookies.map do |cookie, v|
+            if cookie.is_a?(HTTP::Cookie)
+              cookie
+            else
+              HTTP::Cookie.new(cookie.to_s, v.to_s)
+            end
+          end
+        end
+
+        def set(origin, cookies)
+          return unless cookies
+
+          @store[origin].parse(cookies, origin)
+        end
+
+        def [](uri)
+          store = @store[uri.origin]
+          @default_cookies.each do |cookie|
+            c = cookie.dup
+            c.domain ||= uri.authority
+            c.path ||= uri.path
+            store.add(c)
+          end if @default_cookies
+          store
+        end
+      end
+
       def self.load_dependencies(*)
         require "http/cookie"
       end
 
       module InstanceMethods
-        def cookies(cookies)
-          branch(default_options.with_cookies(cookies))
+        extend Forwardable
+
+        def_delegator :@options, :cookies
+
+        def initialize(options = {}, &blk)
+          super({ cookies: Store.new }.merge(options), &blk)
         end
-      end
 
-      module RequestMethods
-        def initialize(*)
-          super
-          @headers.cookies(@options.cookies, self)
+        def with_cookies(cookies)
+          branch(default_options.with_cookies(cookies))
         end
-      end
 
-      module HeadersMethods
-        def cookies(jar, request)
-          return unless jar
-          unless jar.is_a?(HTTP::CookieJar)
-            jar = jar.each_with_object(HTTP::CookieJar.new) do |(k, v), j|
-              cookie = k.is_a?(HTTP::Cookie) ? v : HTTP::Cookie.new(k.to_s, v.to_s)
-              cookie.domain = request.authority
-              cookie.path = request.path
-              j.add(cookie)
+        def wrap
+          return super unless block_given?
+
+          super do |session|
+            old_cookies_store = @options.cookies.dup
+            begin
+              yield session
+            ensure
+              @options = @options.with_cookies(old_cookies_store)
             end
           end
-          self["cookie"] = HTTP::Cookie.cookie_value(jar.cookies)
         end
-      end
 
-      module ResponseMethods
-        def cookie_jar
-          return @cookie_jar if defined?(@cookie_jar)
-          return nil unless headers.key?("set-cookie")
-          @cookie_jar ||= begin
-            jar = HTTP::CookieJar.new
-            jar.parse(headers["set-cookie"], @request.uri)
-            jar
-          end
+        private
+
+        def on_response(request, response)
+          @options.cookies.set(request.origin, response.headers["set-cookie"]) if response.respond_to?(:headers)
+
+          super
+        end
+
+        def build_request(*, _)
+          request = super
+          request.headers.set_cookie(@options.cookies[request.uri])
+          request
         end
-        alias_method :cookies, :cookie_jar
       end
 
-      module OptionsMethods
-        def self.included(klass)
-          super
-          klass.def_option(:cookies) do |cookies|
-            cookies.split(/ *; */) if cookies.is_a?(String)
-            cookies
-          end
+      module HeadersMethods
+        def set_cookie(jar)
+          return unless jar
+
+          cookie_value = HTTP::Cookie.cookie_value(jar.cookies)
+          return if cookie_value.empty?
+
+          add("cookie", cookie_value)
         end
       end
     end

data/lib/httpx/plugins/digest_authentication.rb

@@ -2,9 +2,23 @@
 
 module HTTPX
   module Plugins
+    #
+    # This plugin adds helper methods to implement HTTP Digest Auth
+    # https://tools.ietf.org/html/rfc7616
+    #
     module DigestAuthentication
       DigestError = Class.new(Error)
 
+      def self.extra_options(options)
+        Class.new(options.class) do
+          def_option(:digest) do |digest|
+            raise Error, ":digest must be a Digest" unless digest.is_a?(Digest)
+
+            digest
+          end
+        end.new(options)
+      end
+
       def self.load_dependencies(*)
         require "securerandom"
         require "digest"
@@ -12,37 +26,41 @@ module HTTPX
 
       module InstanceMethods
         def digest_authentication(user, password)
-          @_digest = Digest.new(user, password)
-          self
+          branch(default_options.with_digest(Digest.new(user, password)))
         end
+
         alias_method :digest_auth, :digest_authentication
 
-        def request(*args, keep_open: @keep_open, **options)
-          return super unless @_digest
-          begin
-            requests = __build_reqs(*args, **options)
-            probe_request = requests.first
-            prev_response = __send_reqs(*probe_request).first
+        def request(*args, **options)
+          requests = build_requests(*args, options)
+          probe_request = requests.first
+          digest = probe_request.options.digest
 
-            unless prev_response.status == 401
-              raise Error, "request doesn't require authentication (status: #{prev_response})"
-            end
+          return super unless digest
 
-            probe_request.transition(:idle)
-            responses = []
+          prev_response = wrap { send_requests(*probe_request, options).first }
 
-            requests.each do |request|
-              token = @_digest.generate_header(request, prev_response)
-              request.headers["authorization"] = "Digest #{token}"
-              response = __send_reqs(*request).first
-              responses << response
-              prev_response = response
+          raise Error, "request doesn't require authentication (status: #{prev_response.status})" unless prev_response.status == 401
+
+          probe_request.transition(:idle)
+
+          responses = []
+
+          while (request = requests.shift)
+            token = digest.generate_header(request, prev_response)
+            request.headers["authorization"] = "Digest #{token}"
+            response = if requests.empty?
+              send_requests(*request, options).first
+            else
+              wrap { send_requests(*request, options).first }
             end
-            return responses.first if responses.size == 1
-            responses
-          ensure
-            close unless keep_open
+            responses << response
+            prev_response = response
           end
+
+          return responses.first if responses.size == 1
+
+          responses
         end
       end
 
@@ -54,10 +72,10 @@ module HTTPX
         end
 
         def generate_header(request, response, _iis = false)
-          method = request.verb.to_s.upcase
+          meth = request.verb.to_s.upcase
           www = response.headers["www-authenticate"]
 
-          # TODO: assert if auth-type is Digest
+          # discard first token, it's Digest
           auth_info = www[/^(\w+) (.*)/, 2]
 
           uri = request.path
@@ -100,7 +118,7 @@ module HTTPX
           end
 
           ha1 = algorithm.hexdigest(a1)
-          ha2 = algorithm.hexdigest("#{method}:#{uri}")
+          ha2 = algorithm.hexdigest("#{meth}:#{uri}")
           request_digest = [ha1, nonce]
           request_digest.push(nc, cnonce, qop) if qop
           request_digest << ha2
@@ -136,6 +154,7 @@ module HTTPX
         end
       end
     end
+
     register_plugin :digest_authentication, DigestAuthentication
   end
 end

data/lib/httpx/plugins/follow_redirects.rb

@@ -3,76 +3,74 @@
 module HTTPX
   InsecureRedirectError = Class.new(Error)
   module Plugins
+    #
+    # This plugin adds support for following redirect (status 30X) responses.
+    #
+    # It has an upper bound of followed redirects (see *MAX_REDIRECTS*), after which it
+    # will return the last redirect response. It will **not** raise an exception.
+    #
+    # It also doesn't follow insecure redirects (https -> http) by default (see *follow_insecure_redirects*).
+    #
     module FollowRedirects
-      module InstanceMethods
-        MAX_REDIRECTS = 3
-        REDIRECT_STATUS = 300..399
+      MAX_REDIRECTS = 3
+      REDIRECT_STATUS = (300..399).freeze
 
-        def max_redirects(n)
-          branch(default_options.with_max_redirects(n.to_i))
-        end
+      def self.extra_options(options)
+        Class.new(options.class) do
+          def_option(:max_redirects) do |num|
+            num = Integer(num)
+            raise Error, ":max_redirects must be positive" unless num.positive?
 
-        def request(*args, **options)
-          # do not needlessly close channels
-          keep_open = @keep_open
-          @keep_open = true
-
-          max_redirects = @options.max_redirects || MAX_REDIRECTS
-          requests = __build_reqs(*args, **options)
-          responses = __send_reqs(*requests)
-
-          loop do
-            redirect_requests = []
-            indexes = responses.each_with_index.map do |response, index|
-              next unless REDIRECT_STATUS.include?(response.status)
-              request = requests[index]
-              retry_request = __build_redirect_req(request, response, options)
-              redirect_requests << retry_request
-              index
-            end.compact
-            break if redirect_requests.empty?
-            break if max_redirects <= 0
-            max_redirects -= 1
-
-            redirect_responses = __send_reqs(*redirect_requests)
-            indexes.each_with_index do |index, i2|
-              requests[index] = redirect_requests[i2]
-              responses[index] = redirect_responses[i2]
-            end
+            num
           end
 
-          return responses.first if responses.size == 1
-          responses
-        ensure
-          @keep_open = keep_open
+          def_option(:follow_insecure_redirects)
+        end.new(options)
+      end
+
+      module InstanceMethods
+        def max_redirects(n)
+          branch(default_options.with_max_redirects(n.to_i))
         end
 
         private
 
-        def fetch_response(request)
-          response = super
-          if response &&
-             REDIRECT_STATUS.include?(response.status) &&
-             !@options.follow_insecure_redirects
-            redirect_uri = __get_location_from_response(response)
-            if response.uri.scheme == "https" &&
-               redirect_uri.scheme == "http"
-              error = InsecureRedirectError.new(redirect_uri.to_s)
-              error.set_backtrace(caller)
-              response = ErrorResponse.new(error, @options)
-            end
+        def fetch_response(request, connections, options)
+          redirect_request = request.redirect_request
+          response = super(redirect_request, connections, options)
+          return unless response
+
+          max_redirects = redirect_request.max_redirects
+
+          return response unless REDIRECT_STATUS.include?(response.status)
+          return response unless max_redirects.positive?
+
+          retry_request = build_redirect_request(redirect_request, response, options)
+
+          request.redirect_request = retry_request
+
+          if !options.follow_insecure_redirects &&
+             response.uri.scheme == "https" &&
+             retry_request.uri.scheme == "http"
+            error = InsecureRedirectError.new(retry_request.uri.to_s)
+            error.set_backtrace(caller)
+            return ErrorResponse.new(error, options)
           end
-          response
+
+          connection = find_connection(retry_request, connections, options)
+          connection.send(retry_request)
+          nil
         end
 
-        def __build_redirect_req(request, response, options)
+        def build_redirect_request(request, response, options)
           redirect_uri = __get_location_from_response(response)
+          max_redirects = request.max_redirects
 
-          # TODO: integrate cookies in the next request
           # redirects are **ALWAYS** GET
           retry_options = options.merge(headers: request.headers,
-                                        body: request.body)
-          __build_req(:get, redirect_uri, retry_options)
+                                        body: request.body,
+                                        max_redirects: max_redirects - 1)
+          build_request(:get, redirect_uri, retry_options)
         end
 
         def __get_location_from_response(response)
@@ -82,16 +80,17 @@ module HTTPX
         end
       end
 
-      module OptionsMethods
+      module RequestMethods
         def self.included(klass)
-          super
-          klass.def_option(:max_redirects) do |num|
-            num = Integer(num)
-            raise Error, ":max_redirects must be positive" unless num.positive?
-            num
-          end
+          klass.__send__(:attr_writer, :redirect_request)
+        end
+
+        def redirect_request
+          @redirect_request || self
+        end
 
-          klass.def_option(:follow_insecure_redirects)
+        def max_redirects
+          @options.max_redirects || MAX_REDIRECTS
         end
       end
     end