httpx-patched 1.6.2.1

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder::Multipart
+    class Encoder
+      attr_reader :bytesize
+
+      def initialize(form)
+        @boundary = ("-" * 21) << SecureRandom.hex(21)
+        @part_index = 0
+        @buffer = "".b
+
+        @form = form
+        @bytesize = 0
+        @parts = to_parts(form)
+      end
+
+      def content_type
+        "multipart/form-data; boundary=#{@boundary}"
+      end
+
+      def to_s
+        read || ""
+      ensure
+        rewind
+      end
+
+      def read(length = nil, outbuf = nil)
+        data   = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
+        data ||= "".b
+
+        read_chunks(data, length)
+
+        data unless length && data.empty?
+      end
+
+      def rewind
+        form = @form.each_with_object([]) do |(key, val), aux|
+          if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
+            # @type var val: File
+            val = val.reopen(val.path, File::RDONLY)
+          end
+          val.rewind if val.respond_to?(:rewind)
+          aux << [key, val]
+        end
+        @form = form
+        @bytesize = 0
+        @parts = to_parts(form)
+        @part_index = 0
+      end
+
+      private
+
+      def to_parts(form)
+        params = form.each_with_object([]) do |(key, val), aux|
+          Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
+            next if v.nil?
+
+            value, content_type, filename = Part.call(v)
+
+            header = header_part(k, content_type, filename)
+            @bytesize += header.size
+            aux << header
+
+            @bytesize += value.size
+            aux << value
+
+            delimiter = StringIO.new("\r\n")
+            @bytesize += delimiter.size
+            aux << delimiter
+          end
+        end
+        final_delimiter = StringIO.new("--#{@boundary}--\r\n")
+        @bytesize += final_delimiter.size
+        params << final_delimiter
+
+        params
+      end
+
+      def header_part(key, content_type, filename)
+        header = "--#{@boundary}\r\n".b
+        header << "Content-Disposition: form-data; name=#{key.inspect}".b
+        header << "; filename=#{filename.inspect}" if filename
+        header << "\r\nContent-Type: #{content_type}\r\n\r\n"
+        StringIO.new(header)
+      end
+
+      def read_chunks(buffer, length = nil)
+        while @part_index < @parts.size
+          chunk = read_from_part(length)
+
+          next unless chunk
+
+          buffer << chunk.force_encoding(Encoding::BINARY)
+
+          next unless length
+
+          length -= chunk.bytesize
+
+          break if length.zero?
+        end
+      end
+
+      # if there's a current part to read from, tries to read a chunk.
+      def read_from_part(max_length = nil)
+        part = @parts[@part_index]
+
+        chunk = part.read(max_length, @buffer)
+
+        return chunk if chunk && !chunk.empty?
+
+        part.close if part.respond_to?(:close)
+
+        @part_index += 1
+
+        nil
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/multipart/mime_type_detector.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder::Multipart
+    module MimeTypeDetector
+      module_function
+
+      DEFAULT_MIMETYPE = "application/octet-stream"
+
+      # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
+      if defined?(FileMagic)
+        MAGIC_NUMBER = 256 * 1024
+
+        def call(file, _)
+          return nil if file.eof? # FileMagic returns "application/x-empty" for empty files
+
+          mime = FileMagic.open(FileMagic::MAGIC_MIME_TYPE) do |filemagic|
+            filemagic.buffer(file.read(MAGIC_NUMBER))
+          end
+
+          file.rewind
+
+          mime
+        end
+      elsif defined?(Marcel)
+        def call(file, filename)
+          return nil if file.eof? # marcel returns "application/octet-stream" for empty files
+
+          Marcel::MimeType.for(file, name: filename)
+        end
+
+      elsif defined?(MimeMagic)
+
+        def call(file, _)
+          mime = MimeMagic.by_magic(file)
+          mime.type if mime
+        end
+
+      elsif system("which file", out: File::NULL)
+        require "open3"
+
+        def call(file, _)
+          return if file.eof? # file command returns "application/x-empty" for empty files
+
+          Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
+            begin
+              IO.copy_stream(file, stdin.binmode)
+            rescue Errno::EPIPE
+            end
+            file.rewind
+            stdin.close
+
+            status = thread.value
+
+            # call to file command failed
+            if status.nil? || !status.success?
+              $stderr.print(stderr.read)
+            else
+
+              output = stdout.read.strip
+
+              if output.include?("cannot open")
+                $stderr.print(output)
+              else
+                output
+              end
+            end
+          end
+        end
+
+      else
+
+        def call(_, _); end
+
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/multipart/part.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder::Multipart
+    module Part
+      module_function
+
+      def call(value)
+        # take out specialized objects of the way
+        if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
+          return value, value.content_type, value.filename
+        end
+
+        content_type = filename = nil
+
+        if value.is_a?(Hash)
+          content_type = value[:content_type]
+          filename = value[:filename]
+          value = value[:body]
+        end
+
+        value = value.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
+
+        if value.respond_to?(:path) && value.respond_to?(:read)
+          # either a File, a Tempfile, or something else which has to quack like a file
+          filename ||= File.basename(value.path)
+          content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
+          [value, content_type, filename]
+        else
+          [StringIO.new(value.to_s), content_type || "text/plain", filename]
+        end
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/multipart.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "multipart/encoder"
+require_relative "multipart/decoder"
+require_relative "multipart/part"
+require_relative "multipart/mime_type_detector"
+
+module HTTPX::Transcoder
+  module Multipart
+    MULTIPART_VALUE_COND = lambda do |value|
+      value.respond_to?(:read) ||
+        (value.respond_to?(:to_hash) &&
+          value.key?(:body) &&
+          (value.key?(:filename) || value.key?(:content_type)))
+    end
+
+    module_function
+
+    def multipart?(form_data)
+      form_data.any? do |_, v|
+        Multipart::MULTIPART_VALUE_COND.call(v) ||
+          (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
+          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+      end
+    end
+
+    def encode(form_data)
+      Encoder.new(form_data)
+    end
+  end
+end

data/lib/httpx/transcoder/utils/body_reader.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module HTTPX
+  module Transcoder
+    class BodyReader
+      def initialize(body)
+        @body = if body.respond_to?(:read)
+          body.rewind if body.respond_to?(:rewind)
+          body
+        elsif body.respond_to?(:each)
+          body.enum_for(:each)
+        else
+          StringIO.new(body.to_s)
+        end
+      end
+
+      def bytesize
+        return @body.bytesize if @body.respond_to?(:bytesize)
+
+        Float::INFINITY
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @body.read(length, outbuf) if @body.respond_to?(:read)
+
+        begin
+          chunk = @body.next
+          if outbuf
+            outbuf.clear.force_encoding(Encoding::BINARY)
+            outbuf << chunk
+          else
+            outbuf = chunk
+          end
+          outbuf unless length && outbuf.empty?
+        rescue StopIteration
+        end
+      end
+
+      def close
+        @body.close if @body.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "body_reader"
+
+module HTTPX
+  module Transcoder
+    class Deflater
+      attr_reader :content_type
+
+      def initialize(body)
+        @content_type = body.content_type
+        @body = BodyReader.new(body)
+        @closed = false
+      end
+
+      def bytesize
+        buffer_deflate!
+
+        @buffer.size
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @buffer.read(length, outbuf) if @buffer
+
+        return if @closed
+
+        chunk = @body.read(length)
+
+        compressed_chunk = deflate(chunk)
+
+        return unless compressed_chunk
+
+        if outbuf
+          outbuf.clear.force_encoding(Encoding::BINARY)
+          outbuf << compressed_chunk
+        else
+          compressed_chunk
+        end
+      end
+
+      def close
+        return if @closed
+
+        @buffer.close if @buffer
+
+        @body.close
+
+        @closed = true
+      end
+
+      def rewind
+        return unless @buffer
+
+        @buffer.rewind
+      end
+
+      private
+
+      # rubocop:disable Naming/MemoizedInstanceVariableName
+      def buffer_deflate!
+        return @buffer if defined?(@buffer)
+
+        buffer = Response::Buffer.new(
+          threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
+        )
+        IO.copy_stream(self, buffer)
+
+        buffer.rewind if buffer.respond_to?(:rewind)
+
+        @buffer = buffer
+      end
+      # rubocop:enable Naming/MemoizedInstanceVariableName
+    end
+  end
+end

data/lib/httpx/transcoder.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Transcoder
+    module_function
+
+    def normalize_keys(key, value, cond = nil, &block)
+      if cond && cond.call(value)
+        block.call(key.to_s, value)
+      elsif value.respond_to?(:to_ary)
+        if value.empty?
+          block.call("#{key}[]")
+        else
+          value.to_ary.each do |element|
+            normalize_keys("#{key}[]", element, cond, &block)
+          end
+        end
+      elsif value.respond_to?(:to_hash)
+        value.to_hash.each do |child_key, child_value|
+          normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
+        end
+      else
+        block.call(key.to_s, value)
+      end
+    end
+
+    # based on https://github.com/rack/rack/blob/d15dd728440710cfc35ed155d66a98dc2c07ae42/lib/rack/query_parser.rb#L82
+    def normalize_query(params, name, v, depth)
+      raise Error, "params depth surpasses what's supported" if depth <= 0
+
+      name =~ /\A[\[\]]*([^\[\]]+)\]*/
+      k = Regexp.last_match(1) || ""
+      after = Regexp.last_match ? Regexp.last_match.post_match : ""
+
+      if k.empty?
+        return Array(v) if !v.empty? && name == "[]"
+
+        return
+      end
+
+      case after
+      when ""
+        params[k] = v
+      when "["
+        params[name] = v
+      when "[]"
+        params[k] ||= []
+        raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
+
+        params[k] << v
+      when /^\[\]\[([^\[\]]+)\]$/, /^\[\](.+)$/
+        child_key = Regexp.last_match(1)
+        params[k] ||= []
+        raise Error, "expected Array (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Array)
+
+        if params[k].last.is_a?(Hash) && !params_hash_has_key?(params[k].last, child_key)
+          normalize_query(params[k].last, child_key, v, depth - 1)
+        else
+          params[k] << normalize_query({}, child_key, v, depth - 1)
+        end
+      else
+        params[k] ||= {}
+        raise Error, "expected Hash (got #{params[k].class}) for param '#{k}'" unless params[k].is_a?(Hash)
+
+        params[k] = normalize_query(params[k], after, v, depth - 1)
+      end
+
+      params
+    end
+
+    def params_hash_has_key?(hash, key)
+      return false if key.include?("[]")
+
+      key.split(/[\[\]]+/).inject(hash) do |h, part|
+        next h if part == ""
+        return false unless h.is_a?(Hash) && h.key?(part)
+
+        h[part]
+      end
+
+      true
+    end
+  end
+end
+
+require "httpx/transcoder/body"
+require "httpx/transcoder/form"
+require "httpx/transcoder/json"
+require "httpx/transcoder/chunker"
+require "httpx/transcoder/deflate"
+require "httpx/transcoder/gzip"

data/lib/httpx/utils.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Utils
+    using URIExtensions
+
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
+    FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
+
+    module_function
+
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def elapsed_time(monotonic_timestamp)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
+    end
+
+    # The value of this field can be either an HTTP-date or a number of
+    # seconds to delay after the response is received.
+    def parse_retry_after(retry_after)
+      # first: bet on it being an integer
+      Integer(retry_after)
+    rescue ArgumentError
+      # Then it's a datetime
+      time = Time.httpdate(retry_after)
+      time - Time.now
+    end
+
+    def get_filename(header, _prefix_regex = nil)
+      filename = nil
+      case header
+      when FILENAME_REGEX
+        filename = Regexp.last_match(1)
+        filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+      when FILENAME_EXTENSION_REGEX
+        filename = Regexp.last_match(1)
+        encoding, _, filename = filename.split("'", 3)
+      end
+
+      return unless filename
+
+      filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
+
+      filename.scrub!
+
+      filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
+
+      filename.force_encoding ::Encoding.find(encoding) if encoding
+
+      filename
+    end
+
+    URIParser = URI::RFC2396_Parser.new
+
+    def to_uri(uri)
+      return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+
+      uri = URI(URIParser.escape(uri))
+
+      non_ascii_hostname = URIParser.unescape(uri.host)
+
+      non_ascii_hostname.force_encoding(Encoding::UTF_8)
+
+      idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
+
+      uri.host = idna_hostname
+      uri.non_ascii_hostname = non_ascii_hostname
+      uri
+    end
+  end
+end

data/lib/httpx/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module HTTPX
+  VERSION = "1.6.2.1"
+end

data/lib/httpx.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "httpx/version"
+
+# Top-Level Namespace
+#
+module HTTPX
+  EMPTY = [].freeze
+  EMPTY_HASH = {}.freeze
+
+  # All plugins should be stored under this module/namespace. Can register and load
+  # plugins.
+  #
+  module Plugins
+    @plugins = {}
+    @plugins_mutex = Thread::Mutex.new
+
+    # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
+    # it from the load path under "httpx/plugins/" directory.
+    #
+    def self.load_plugin(name)
+      h = @plugins
+      m = @plugins_mutex
+      unless (plugin = m.synchronize { h[name] })
+        require "httpx/plugins/#{name}"
+        raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
+      end
+      plugin
+    end
+
+    # Registers a plugin (+mod+) in the central store indexed by +name+.
+    #
+    def self.register_plugin(name, mod)
+      h = @plugins
+      m = @plugins_mutex
+      m.synchronize { h[name] = mod }
+    end
+  end
+end
+
+require "httpx/extensions"
+
+require "httpx/errors"
+require "httpx/utils"
+require "httpx/punycode"
+require "httpx/domain_name"
+require "httpx/altsvc"
+require "httpx/callbacks"
+require "httpx/loggable"
+require "httpx/transcoder"
+require "httpx/timers"
+require "httpx/pool"
+require "httpx/headers"
+require "httpx/request"
+require "httpx/response"
+require "httpx/options"
+require "httpx/chainable"
+
+require "httpx/session"
+require "httpx/session_extensions"
+
+# load integrations when possible
+
+require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog::Tracing)
+require "httpx/adapters/sentry" if defined?(Sentry)
+require "httpx/adapters/webmock" if defined?(WebMock)

data/sig/altsvc.rbs

@@ -0,0 +1,33 @@
+module HTTPX
+  module AltSvc
+    module ConnectionMixin
+
+      def send: (Request request) -> void
+
+      def match?: (URI::Generic uri, Options options) -> bool
+
+      private
+
+      def match_altsvcs?: (URI::Generic uri) -> bool
+
+      def match_altsvc_options?: (URI::Generic uri, Options options) -> bool
+    end
+
+    type altsvc_params = Hash[String, untyped]
+
+    def self?.cached_altsvc: (String origin) -> Array[altsvc_params]
+
+    def self?.cached_altsvc_set: (String origin, altsvc_params) -> void
+
+    def self?.lookup: (String origin, Integer | Float ttl) -> Array[altsvc_params]
+
+    def self?.emit: (Request request, response response) { (http_uri alt_origin, String origin, altsvc_params alt_params) -> void } -> void
+
+    def self?.parse: (String altsvc) { (http_uri alt_origin, altsvc_params alt_params) -> void } -> void
+                   | (String altsvc) -> Enumerable[[http_uri, altsvc_params]]
+
+    def self?.parse_altsvc_scheme: (String alt_proto) -> String?
+
+    def self.parse_altsvc_origin: (string alt_proto, String alt_origin) -> http_uri?
+  end
+end

data/sig/buffer.rbs

@@ -0,0 +1,27 @@
+module HTTPX
+  class Buffer
+    extend Forwardable
+
+    include _ToS
+    include _ToStr
+
+    @buffer: String
+
+    attr_reader limit: Integer
+
+    def full?: () -> bool
+    def shift!: (Integer) -> void
+    def capacity: () -> Integer
+
+    # delegated
+    def <<: (String data) -> String
+    def empty?: () -> bool
+    def bytesize: () -> (Integer | Float)
+    def clear: () -> void
+    def replace: (string) -> void
+
+    private
+
+    def initialize: (Integer limit) -> untyped
+  end
+end