httpx 0.15.4 → 0.18.0

data/lib/httpx/plugins/digest_authentication.rb

@@ -14,19 +14,23 @@ module HTTPX
 
       DigestError = Class.new(Error)
 
-      def self.extra_options(options)
-        Class.new(options.class) do
-          def_option(:digest, <<-OUT)
-            raise Error, ":digest must be a Digest" unless value.is_a?(#{Digest})
-
-            value
-          OUT
-        end.new(options).merge(max_concurrent_requests: 1)
+      class << self
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
+
+        def load_dependencies(*)
+          require "securerandom"
+          require "digest"
+        end
       end
 
-      def self.load_dependencies(*)
-        require "securerandom"
-        require "digest"
+      module OptionsMethods
+        def option_digest(value)
+          raise TypeError, ":digest must be a Digest" unless value.is_a?(Digest)
+
+          value
+        end
       end
 
       module InstanceMethods
@@ -36,12 +40,12 @@ module HTTPX
 
         alias_method :digest_auth, :digest_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             digest = request.options.digest
 
             if digest
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if digest && !probe_response.is_a?(ErrorResponse) &&
                  probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
@@ -52,12 +56,12 @@ module HTTPX
                 token = digest.generate_header(request, probe_response)
                 request.headers["authorization"] = "Digest #{token}"
 
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/expect.rb

@@ -10,26 +10,30 @@ module HTTPX
     module Expect
       EXPECT_TIMEOUT = 2
 
-      def self.no_expect_store
-        @no_expect_store ||= []
+      class << self
+        def no_expect_store
+          @no_expect_store ||= []
+        end
+
+        def extra_options(options)
+          options.merge(expect_timeout: EXPECT_TIMEOUT)
+        end
       end
 
-      def self.extra_options(options)
-        Class.new(options.class) do
-          def_option(:expect_timeout, <<-OUT)
-            seconds = Integer(value)
-            raise Error, ":expect_timeout must be positive" unless seconds.positive?
+      module OptionsMethods
+        def option_expect_timeout(value)
+          seconds = Integer(value)
+          raise TypeError, ":expect_timeout must be positive" unless seconds.positive?
 
-            seconds
-          OUT
+          seconds
+        end
 
-          def_option(:expect_threshold_size, <<-OUT)
-            bytes = Integer(value)
-            raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
+        def option_expect_threshold_size(value)
+          bytes = Integer(value)
+          raise TypeError, ":expect_threshold_size must be positive" unless bytes.positive?
 
-            bytes
-          OUT
-        end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
+          bytes
+        end
       end
 
       module RequestMethods
@@ -65,9 +69,14 @@ module HTTPX
       end
 
       module ConnectionMethods
-        def send(request)
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["expect"] == "100-continue"
+
           request.once(:expect) do
-            @timers.after(@options.expect_timeout) do
+            @timers.after(request.options.expect_timeout) do
+              # expect timeout expired
               if request.state == :expect && !request.expects?
                 Expect.no_expect_store << request.origin
                 request.headers.delete("expect")
@@ -75,7 +84,6 @@ module HTTPX
               end
             end
           end
-          super
         end
       end
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -17,17 +17,17 @@ module HTTPX
       MAX_REDIRECTS = 3
       REDIRECT_STATUS = (300..399).freeze
 
-      def self.extra_options(options)
-        Class.new(options.class) do
-          def_option(:max_redirects, <<-OUT)
-            num = Integer(value)
-            raise Error, ":max_redirects must be positive" if num.negative?
+      module OptionsMethods
+        def option_max_redirects(value)
+          num = Integer(value)
+          raise TypeError, ":max_redirects must be positive" if num.negative?
 
-            num
-          OUT
+          num
+        end
 
-          def_option(:follow_insecure_redirects)
-        end.new(options)
+        def option_follow_insecure_redirects(value)
+          value
+        end
       end
 
       module InstanceMethods

data/lib/httpx/plugins/grpc/call.rb

@@ -10,6 +10,7 @@ module HTTPX
         def initialize(response)
           @response = response
           @decoder = ->(z) { z }
+          @consumed = false
         end
 
         def inspect
@@ -25,7 +26,7 @@ module HTTPX
         end
 
         def trailing_metadata
-          return unless @response.body.closed?
+          return unless @consumed
 
           @response.trailing_metadata
         end
@@ -40,8 +41,10 @@ module HTTPX
               Message.stream(@response).each do |message|
                 y << @decoder.call(message)
               end
+              @consumed = true
             end
           else
+            @consumed = true
             @decoder.call(Message.unary(@response))
           end
         end

data/lib/httpx/plugins/grpc/message.rb

@@ -17,7 +17,7 @@ module HTTPX
 
         # lazy decodes a grpc stream response
         def stream(response, &block)
-          return enum_for(__method__, response) unless block_given?
+          return enum_for(__method__, response) unless block
 
           response.each do |frame|
             decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
@@ -57,7 +57,7 @@ module HTTPX
 
             yield data
 
-            message = message.byteslice(5 + size..-1)
+            message = message.byteslice((5 + size)..-1)
           end
         end
 

data/lib/httpx/plugins/grpc.rb

@@ -49,7 +49,6 @@ module HTTPX
       class << self
         def load_dependencies(*)
           require "stringio"
-          require "google/protobuf"
           require "httpx/plugins/grpc/message"
           require "httpx/plugins/grpc/call"
         end
@@ -61,36 +60,7 @@ module HTTPX
         end
 
         def extra_options(options)
-          Class.new(options.class) do
-            def_option(:grpc_service, <<-OUT)
-              String(value)
-            OUT
-
-            def_option(:grpc_compression, <<-OUT)
-              case value
-              when true, false
-                value
-              else
-                value.to_s
-              end
-            OUT
-
-            def_option(:grpc_rpcs, <<-OUT)
-              Hash[value]
-            OUT
-
-            def_option(:grpc_deadline, <<-OUT)
-              raise Error, ":grpc_deadline must be positive" unless value.positive?
-
-              value
-            OUT
-
-            def_option(:call_credentials, <<-OUT)
-              raise Error, ":call_credentials must respond to #call" unless value.respond_to?(:call)
-
-              value
-            OUT
-          end.new(options).merge(
+          options.merge(
             fallback_protocol: "h2",
             http2_settings: { wait_for_handshake: false },
             grpc_rpcs: {}.freeze,
@@ -100,6 +70,37 @@ module HTTPX
         end
       end
 
+      module OptionsMethods
+        def option_grpc_service(value)
+          String(value)
+        end
+
+        def option_grpc_compression(value)
+          case value
+          when true, false
+            value
+          else
+            value.to_s
+          end
+        end
+
+        def option_grpc_rpcs(value)
+          Hash[value]
+        end
+
+        def option_grpc_deadline(value)
+          raise TypeError, ":grpc_deadline must be positive" unless value.positive?
+
+          value
+        end
+
+        def option_call_credentials(value)
+          raise TypeError, ":call_credentials must respond to #call" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+
       module ResponseMethods
         attr_reader :trailing_metadata
 
@@ -140,9 +141,19 @@ module HTTPX
             deadline: @options.grpc_deadline,
           }.merge(opts)
 
-          with(grpc_rpcs: @options.grpc_rpcs.merge(
-            rpc_name.underscore => [rpc_name, input, output, rpc_opts]
-          ).freeze)
+          session_class = Class.new(self.class) do
+            class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+              def #{rpc_name}(input, **opts)              # def grpc_action(input, **opts)
+                rpc_execute("#{rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+              end                                         # end
+            OUT
+          end
+
+          session_class.new(@options.merge(
+                              grpc_rpcs: @options.grpc_rpcs.merge(
+                                rpc_name.underscore => [rpc_name, input, output, rpc_opts]
+                              ).freeze
+                            ))
         end
 
         def build_stub(origin, service: nil, compression: false)
@@ -150,7 +161,32 @@ module HTTPX
 
           origin = URI.parse("#{scheme}://#{origin}")
 
-          with(origin: origin, grpc_service: service, grpc_compression: compression)
+          session = self
+
+          if service && service.respond_to?(:rpc_descs)
+            # it's a grpc generic service
+            service.rpc_descs.each do |rpc_name, rpc_desc|
+              rpc_opts = {
+                marshal_method: rpc_desc.marshal_method,
+                unmarshal_method: rpc_desc.unmarshal_method,
+              }
+
+              input = rpc_desc.input
+              input = input.type if input.respond_to?(:type)
+
+              output = rpc_desc.output
+              if output.respond_to?(:type)
+                rpc_opts[:stream] = true
+                output = output.type
+              end
+
+              session = session.rpc(rpc_name, input, output, **rpc_opts)
+            end
+
+            service = service.service_name
+          end
+
+          session.with(origin: origin, grpc_service: service, grpc_compression: compression)
         end
 
         def execute(rpc_method, input,
@@ -166,7 +202,7 @@ module HTTPX
         private
 
         def rpc_execute(rpc_name, input, **opts)
-          rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name.to_s] || raise(Error, "#{rpc_name}: undefined service")
+          rpc_name, input_enc, output_enc, rpc_opts = @options.grpc_rpcs[rpc_name]
 
           exec_opts = rpc_opts.merge(opts)
 
@@ -230,16 +266,6 @@ module HTTPX
 
           build_request(:post, uri, headers: headers, body: body)
         end
-
-        def respond_to_missing?(meth, *, &blk)
-          @options.grpc_rpcs.key?(meth.to_s) || super
-        end
-
-        def method_missing(meth, *args, **kwargs, &blk)
-          return rpc_execute(meth, *args, **kwargs, &blk) if @options.grpc_rpcs.key?(meth.to_s)
-
-          super
-        end
       end
     end
     register_plugin :grpc, GRPC

data/lib/httpx/plugins/h2c.rb

@@ -24,15 +24,19 @@ module HTTPX
         def call(connection, request, response)
           connection.upgrade_to_h2c(request, response)
         end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
       end
 
       module InstanceMethods
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           upgrade_request, *remainder = requests
 
           return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
 
-          connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
+          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
           return super if connection && connection.upgrade_protocol == :h2c
 
@@ -42,7 +46,7 @@ module HTTPX
           upgrade_request.headers["upgrade"] = "h2c"
           upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
 
-          super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
+          super(upgrade_request, *remainder)
         end
       end
 

data/lib/httpx/plugins/internal_telemetry.rb

@@ -44,6 +44,11 @@ module HTTPX
           meter_elapsed_time("Session: initialized!!!")
         end
 
+        def close(*)
+          super
+          meter_elapsed_time("Session -> close")
+        end
+
         private
 
         def build_requests(*)
@@ -55,11 +60,6 @@ module HTTPX
           meter_elapsed_time("Session -> response") if response
           response
         end
-
-        def close(*)
-          super
-          meter_elapsed_time("Session -> close")
-        end
       end
 
       module RequestMethods
@@ -69,9 +69,9 @@ module HTTPX
         end
 
         def transition(nextstate)
-          state = @state
+          prev_state = @state
           super
-          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{prev_state}] -> #{@state}") if prev_state != @state
         end
       end
 
@@ -84,7 +84,7 @@ module HTTPX
         def transition(nextstate)
           state = @state
           super
-          meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
         end
       end
     end

data/lib/httpx/plugins/multipart/decoder.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX::Plugins
+  module Multipart
+    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
+
+    CRLF = "\r\n"
+
+    class FilePart < SimpleDelegator
+      attr_reader :original_filename, :content_type
+
+      def initialize(filename, content_type)
+        @original_filename = filename
+        @content_type = content_type
+        @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+        super(@file)
+      end
+    end
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    CONDISP = /Content-Disposition:\s*#{TOKEN}\s*/i.freeze
+    BROKEN_QUOTED = /^#{CONDISP}.*;\s*filename="(.*?)"(?:\s*$|\s*;\s*#{TOKEN}=)/i.freeze
+    BROKEN_UNQUOTED = /^#{CONDISP}.*;\s*filename=(#{TOKEN})/i.freeze
+    MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+    MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+    MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+    # Updated definitions from RFC 2231
+    ATTRIBUTE_CHAR = %r{[^ \t\v\n\r)(><@,;:\\"/\[\]?='*%]}.freeze
+    ATTRIBUTE = /#{ATTRIBUTE_CHAR}+/.freeze
+    SECTION = /\*[0-9]+/.freeze
+    REGULAR_PARAMETER_NAME = /#{ATTRIBUTE}#{SECTION}?/.freeze
+    REGULAR_PARAMETER = /(#{REGULAR_PARAMETER_NAME})=(#{VALUE})/.freeze
+    EXTENDED_OTHER_NAME = /#{ATTRIBUTE}\*[1-9][0-9]*\*/.freeze
+    EXTENDED_OTHER_VALUE = /%[0-9a-fA-F]{2}|#{ATTRIBUTE_CHAR}/.freeze
+    EXTENDED_OTHER_PARAMETER = /(#{EXTENDED_OTHER_NAME})=(#{EXTENDED_OTHER_VALUE}*)/.freeze
+    EXTENDED_INITIAL_NAME = /#{ATTRIBUTE}(?:\*0)?\*/.freeze
+    EXTENDED_INITIAL_VALUE = /[a-zA-Z0-9\-]*'[a-zA-Z0-9\-]*'#{EXTENDED_OTHER_VALUE}*/.freeze
+    EXTENDED_INITIAL_PARAMETER = /(#{EXTENDED_INITIAL_NAME})=(#{EXTENDED_INITIAL_VALUE})/.freeze
+    EXTENDED_PARAMETER = /#{EXTENDED_INITIAL_PARAMETER}|#{EXTENDED_OTHER_PARAMETER}/.freeze
+    DISPPARM = /;\s*(?:#{REGULAR_PARAMETER}|#{EXTENDED_PARAMETER})\s*/.freeze
+    RFC2183 = /^#{CONDISP}(#{DISPPARM})+$/i.freeze
+
+    class Decoder
+      BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+      WINDOW_SIZE = 2 << 14
+
+      def initialize(response)
+        @boundary = begin
+          m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+          raise Error, "no boundary declared in content-type header" unless m
+
+          m.strip
+        end
+        @buffer = "".b
+        @parts = {}
+        @intermediate_boundary = "--#{@boundary}"
+        @state = :idle
+      end
+
+      def call(response, _)
+        response.body.each do |chunk|
+          @buffer << chunk
+
+          parse
+        end
+
+        raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+        @parts
+      end
+
+      private
+
+      def parse
+        case @state
+        when :idle
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+          @state = :part_header
+        when :part_header
+          idx = @buffer.index("#{CRLF}#{CRLF}")
+
+          # raise Error, "couldn't parse part headers" unless idx
+          return unless idx
+
+          head = @buffer.byteslice(0..idx + 4 - 1)
+
+          @buffer = @buffer.byteslice(head.bytesize..-1)
+
+          content_type = head[MULTIPART_CONTENT_TYPE, 1]
+          if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+            name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+            name.gsub!(/\\(.)/, "\\1")
+            name
+          else
+            name = head[MULTIPART_CONTENT_ID, 1]
+          end
+
+          filename = get_filename(head)
+
+          name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+          @current = name
+
+          @parts[name] = if filename
+            FilePart.new(filename, content_type)
+          else
+            "".b
+          end
+
+          @state = :part_body
+        when :part_body
+          part = @parts[@current]
+
+          body_separator = if part.is_a?(FilePart)
+            "#{CRLF}#{CRLF}"
+          else
+            CRLF
+          end
+          idx = @buffer.index(body_separator)
+
+          if idx
+            payload = @buffer.byteslice(0..idx - 1)
+            @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+            part << payload
+            part.rewind if part.respond_to?(:rewind)
+            @state = :parse_boundary
+          else
+            part << @buffer
+            @buffer.clear
+          end
+        when :parse_boundary
+          raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+          @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+          if @buffer == "--"
+            @buffer.clear
+            @state = :done
+            return
+          elsif @buffer.start_with?(CRLF)
+            @buffer = @buffer.byteslice(2..-1)
+            @state = :part_header
+          else
+            return
+          end
+        when :done
+          raise Error, "parsing should have been over by now"
+        end until @buffer.empty?
+      end
+
+      def get_filename(head)
+        filename = nil
+        case head
+        when RFC2183
+          params = Hash[*head.scan(DISPPARM).flat_map(&:compact)]
+
+          if (filename = params["filename"])
+            filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+          elsif (filename = params["filename*"])
+            encoding, _, filename = filename.split("'", 3)
+          end
+        when BROKEN_QUOTED, BROKEN_UNQUOTED
+          filename = Regexp.last_match(1)
+        end
+
+        return unless filename
+
+        filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+        filename.scrub!
+
+        filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+        filename.force_encoding ::Encoding.find(encoding) if encoding
+
+        filename
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -17,7 +17,7 @@ module HTTPX
 
       elsif defined?(MimeMagic)
 
-        def call(file, *)
+        def call(file, _)
           mime = MimeMagic.by_magic(file)
           mime.type if mime
         end
@@ -25,7 +25,7 @@ module HTTPX
       elsif system("which file", out: File::NULL)
         require "open3"
 
-        def call(file, *)
+        def call(file, _)
           return if file.eof? # file command returns "application/x-empty" for empty files
 
           Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
@@ -56,7 +56,7 @@ module HTTPX
 
       else
 
-        def call(*); end
+        def call(_, _); end
 
       end
     end

data/lib/httpx/plugins/multipart/part.rb

@@ -8,7 +8,7 @@ module HTTPX
       def call(value)
         # take out specialized objects of the way
         if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
-          return [value, value.content_type, value.filename]
+          return value, value.content_type, value.filename
         end
 
         content_type = filename = nil
@@ -19,7 +19,7 @@ module HTTPX
           value = value[:body]
         end
 
-        value = value.open(:binmode => true) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
+        value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
         if value.is_a?(File)
           filename ||= File.basename(value.path)