httpx 0.10.2 → 0.12.0

data/lib/httpx/plugins/internal_telemetry.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # The InternalTelemetry plugin is for internal use only. It is therefore undocumented, and
+    # its use is disencouraged, as API compatiblity will **not be guaranteed**.
+    #
+    # The gist of it is: when debug_level of logger is enabled to 3 or greater, considered internal-only
+    # supported log levels, it'll be loaded by default.
+    #
+    # Against a specific point of time, which will be by default the session initialization, but can be set
+    # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
+    # using the monotonic clock.
+    module InternalTelemetry
+      module TrackTimeMethods
+        private
+
+        def elapsed_time
+          yield
+        ensure
+          meter_elapsed_time("#{self.class.superclass}##{caller_locations(1, 1)[0].label}")
+        end
+
+        def meter_elapsed_time(label)
+          $http_init_time ||= Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          prev_time = $http_init_time
+          after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+          # $http_init_time = after_time
+          elapsed = after_time - prev_time
+          warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
+        end
+      end
+
+      module InstanceMethods
+        def self.included(klass)
+          klass.prepend TrackTimeMethods
+          super
+        end
+
+        def initialize(*)
+          meter_elapsed_time("Session: initializing...")
+          super
+          meter_elapsed_time("Session: initialized!!!")
+        end
+
+        private
+
+        def build_requests(*)
+          elapsed_time { super }
+        end
+
+        def fetch_response(*)
+          response = super
+          meter_elapsed_time("Session -> response") if response
+          response
+        end
+
+        def close(*)
+          super
+          meter_elapsed_time("Session -> close")
+        end
+      end
+
+      module RequestMethods
+        def self.included(klass)
+          klass.prepend TrackTimeMethods
+          super
+        end
+
+        def transition(nextstate)
+          state = @state
+          super
+          meter_elapsed_time("Request[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
+        end
+      end
+
+      module ConnectionMethods
+        def self.included(klass)
+          klass.prepend TrackTimeMethods
+          super
+        end
+
+        def transition(nextstate)
+          state = @state
+          super
+          meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
+        end
+      end
+    end
+    register_plugin :internal_telemetry, InternalTelemetry
+  end
+end

data/lib/httpx/plugins/multipart.rb

@@ -10,53 +10,60 @@ module HTTPX
     # https://gitlab.com/honeyryderchuck/httpx/wikis/Multipart-Uploads
     #
     module Multipart
-      module FormTranscoder
-        module_function
-
-        class Encoder
-          extend Forwardable
-
-          def_delegator :@raw, :content_type
-
-          def_delegator :@raw, :to_s
+      MULTIPART_VALUE_COND = lambda do |value|
+        value.respond_to?(:read) ||
+          (value.respond_to?(:to_hash) &&
+            value.key?(:body) &&
+            (value.key?(:filename) || value.key?(:content_type)))
+      end
 
-          def_delegator :@raw, :read
+      class << self
+        def normalize_keys(key, value, &block)
+          Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
+        end
 
-          def initialize(form)
-            @raw = if multipart?(form)
-              HTTP::FormData::Multipart.new(Hash[form.flat_map { |k, v| Transcoder.enum_for(:normalize_keys, k, v).to_a }])
-            else
-              HTTP::FormData::Urlencoded.new(form, :encoder => Transcoder::Form.method(:encode))
+        def load_dependencies(*)
+          # :nocov:
+          begin
+            unless defined?(HTTP::FormData)
+              # in order not to break legacy code, we'll keep loading http/form_data for them.
+              require "http/form_data"
+              warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
+                "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
+                "If you'd like to stop seeing this message, require 'http/form_data' yourself."
             end
+          rescue LoadError
           end
+          # :nocov:
+          require "httpx/plugins/multipart/encoder"
+          require "httpx/plugins/multipart/part"
+          require "httpx/plugins/multipart/mime_type_detector"
+        end
 
-          def bytesize
-            @raw.content_length
-          end
+        def configure(*)
+          Transcoder.register("form", FormTranscoder)
+        end
+      end
 
-          private
+      module FormTranscoder
+        module_function
 
-          def multipart?(data)
-            data.any? do |_, v|
-              v.is_a?(HTTP::FormData::Part) ||
-                (v.respond_to?(:to_ary) && v.to_ary.any? { |e| e.is_a?(HTTP::FormData::Part) }) ||
-                (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| e.is_a?(HTTP::FormData::Part) })
-            end
+        def encode(form)
+          if multipart?(form)
+            Encoder.new(form)
+          else
+            Transcoder::Form::Encoder.new(form)
           end
         end
 
-        def encode(form)
-          Encoder.new(form)
+        def multipart?(data)
+          data.any? do |_, v|
+            MULTIPART_VALUE_COND.call(v) ||
+              (v.respond_to?(:to_ary) && v.to_ary.any?(&MULTIPART_VALUE_COND)) ||
+              (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| MULTIPART_VALUE_COND.call(e) })
+          end
         end
       end
-
-      def self.load_dependencies(*)
-        require "http/form_data"
-      end
-
-      def self.configure(*)
-        Transcoder.register("form", FormTranscoder)
-      end
     end
     register_plugin :multipart, Multipart
   end

data/lib/httpx/plugins/multipart/encoder.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module HTTPX::Plugins
+  module Multipart
+    class Encoder
+      attr_reader :bytesize
+
+      def initialize(form)
+        @boundary = ("-" * 21) << SecureRandom.hex(21)
+        @part_index = 0
+        @buffer = "".b
+
+        @form = form
+        @parts = to_parts(form)
+      end
+
+      def content_type
+        "multipart/form-data; boundary=#{@boundary}"
+      end
+
+      def read(length = nil, outbuf = nil)
+        data   = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+        data ||= "".b
+
+        read_chunks(data, length)
+
+        data unless length && data.empty?
+      end
+
+      def rewind
+        form = @form.each_with_object([]) do |(key, val), aux|
+          val = val.reopen(val.path, File::RDONLY) if val.is_a?(File) && val.closed?
+          val.rewind if val.respond_to?(:rewind)
+          aux << [key, val]
+        end
+        @form = form
+        @parts = to_parts(form)
+        @part_index = 0
+      end
+
+      private
+
+      def to_parts(form)
+        @bytesize = 0
+        params = form.each_with_object([]) do |(key, val), aux|
+          Multipart.normalize_keys(key, val) do |k, v|
+            next if v.nil?
+
+            value, content_type, filename = Part.call(v)
+
+            header = header_part(k, content_type, filename)
+            @bytesize += header.size
+            aux << header
+
+            @bytesize += value.size
+            aux << value
+
+            delimiter = StringIO.new("\r\n")
+            @bytesize += delimiter.size
+            aux << delimiter
+          end
+        end
+        final_delimiter = StringIO.new("--#{@boundary}--\r\n")
+        @bytesize += final_delimiter.size
+        params << final_delimiter
+
+        params
+      end
+
+      def header_part(key, content_type, filename)
+        header = "--#{@boundary}\r\n".b
+        header << "Content-Disposition: form-data; name=#{key.inspect}".b
+        header << "; filename=#{filename.inspect}" if filename
+        header << "\r\nContent-Type: #{content_type}\r\n\r\n"
+        StringIO.new(header)
+      end
+
+      def read_chunks(buffer, length = nil)
+        while @part_index < @parts.size
+          chunk = read_from_part(length)
+
+          next unless chunk
+
+          buffer << chunk.force_encoding(Encoding::BINARY)
+
+          next unless length
+
+          length -= chunk.bytesize
+
+          break if length.zero?
+        end
+      end
+
+      # if there's a current part to read from, tries to read a chunk.
+      def read_from_part(max_length = nil)
+        part = @parts[@part_index]
+
+        chunk = part.read(max_length, @buffer)
+
+        return chunk if chunk && !chunk.empty?
+
+        part.close if part.respond_to?(:close)
+
+        @part_index += 1
+
+        nil
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module MimeTypeDetector
+      module_function
+
+      DEFAULT_MIMETYPE = "application/octet-stream"
+
+      # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
+      if defined?(MIME::Types)
+
+        def call(_file, filename)
+          mime = MIME::Types.of(filename).first
+          mime.content_type if mime
+        end
+
+      elsif defined?(MimeMagic)
+
+        def call(file, *)
+          mime = MimeMagic.by_magic(file)
+          mime.type if mime
+        end
+
+      elsif system("which file", out: File::NULL)
+        require "open3"
+
+        def call(file, *)
+          return if file.eof? # file command returns "application/x-empty" for empty files
+
+          Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
+            begin
+              ::IO.copy_stream(file, stdin.binmode)
+            rescue Errno::EPIPE
+            end
+            file.rewind
+            stdin.close
+
+            status = thread.value
+
+            # call to file command failed
+            if status.nil? || !status.success?
+              $stderr.print(stderr.read)
+            else
+
+              output = stdout.read.strip
+
+              if output.include?("cannot open")
+                $stderr.print(output)
+              else
+                output
+              end
+            end
+          end
+        end
+
+      else
+
+        def call(*); end
+
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/part.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module Part
+      module_function
+
+      def call(value)
+        # take out specialized objects of the way
+        if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
+          return [value, value.content_type, value.filename]
+        end
+
+        content_type = filename = nil
+
+        if value.is_a?(Hash)
+          content_type = value[:content_type]
+          filename = value[:filename]
+          value = value[:body]
+        end
+
+        value = value.open(:binmode => true) if value.is_a?(Pathname)
+
+        if value.is_a?(File)
+          filename ||= File.basename(value.path)
+          content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
+          [value, content_type, filename]
+        else
+          [StringIO.new(value.to_s), "text/plain"]
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/proxy.rb

@@ -242,7 +242,7 @@ module HTTPX
     register_plugin :proxy, Proxy
   end
 
-  class ProxySSL < SSL
+  class ProxySSL < IO.registry["ssl"]
     def initialize(tcp, request_uri, options)
       @io = tcp.to_io
       super(request_uri, tcp.addresses, options)

data/lib/httpx/plugins/proxy/http.rb

@@ -81,7 +81,7 @@ module HTTPX
             request.uri.to_s
           end
 
-          def set_request_headers(request)
+          def set_protocol_headers(request)
             super
             proxy_params = @options.proxy
             request.headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?

data/lib/httpx/plugins/proxy/socks4.rb

@@ -16,6 +16,14 @@ module HTTPX
         Error = Socks4Error
 
         module ConnectionMethods
+          def interests
+            if @state == :connecting
+              return @write_buffer.empty? ? :r : :w
+            end
+
+            super
+          end
+
           private
 
           def transition(nextstate)

data/lib/httpx/plugins/proxy/socks5.rb

@@ -35,6 +35,14 @@ module HTTPX
             super || @state == :authenticating || @state == :negotiating
           end
 
+          def interests
+            if @state == :connecting || @state == :authenticating || @state == :negotiating
+              return @write_buffer.empty? ? :r : :w
+            end
+
+            super
+          end
+
           private
 
           def transition(nextstate)
@@ -159,9 +167,10 @@ module HTTPX
             packet = [VERSION, CONNECT, 0].pack("C*")
             begin
               ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
 
-              packet << [IPV4, ip.to_i].pack("CN")
+              ipcode = ip.ipv6? ? IPV6 : IPV4
+
+              packet << [ipcode].pack("C") << ip.hton
             rescue IPAddr::InvalidAddressError
               packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
             end