httpx 0.21.0 → 1.2.1

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX
+  module Transcoder
+    module Multipart
+      class FilePart < SimpleDelegator
+        attr_reader :original_filename, :content_type
+
+        def initialize(filename, content_type)
+          @original_filename = filename
+          @content_type = content_type
+          @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+          super(@file)
+        end
+      end
+
+      class Decoder
+        include HTTPX::Utils
+
+        CRLF = "\r\n"
+        BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+        MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+        MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+        MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+        WINDOW_SIZE = 2 << 14
+
+        def initialize(response)
+          @boundary = begin
+            m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+            raise Error, "no boundary declared in content-type header" unless m
+
+            m.strip
+          end
+          @buffer = "".b
+          @parts = {}
+          @intermediate_boundary = "--#{@boundary}"
+          @state = :idle
+        end
+
+        def call(response, *)
+          response.body.each do |chunk|
+            @buffer << chunk
+
+            parse
+          end
+
+          raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+          @parts
+        end
+
+        private
+
+        def parse
+          case @state
+          when :idle
+            raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+            @state = :part_header
+          when :part_header
+            idx = @buffer.index("#{CRLF}#{CRLF}")
+
+            # raise Error, "couldn't parse part headers" unless idx
+            return unless idx
+
+            head = @buffer.byteslice(0..idx + 4 - 1)
+
+            @buffer = @buffer.byteslice(head.bytesize..-1)
+
+            content_type = head[MULTIPART_CONTENT_TYPE, 1]
+            if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+              name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+              name.gsub!(/\\(.)/, "\\1")
+              name
+            else
+              name = head[MULTIPART_CONTENT_ID, 1]
+            end
+
+            filename = HTTPX::Utils.get_filename(head)
+
+            name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+            @current = name
+
+            @parts[name] = if filename
+              FilePart.new(filename, content_type)
+            else
+              "".b
+            end
+
+            @state = :part_body
+          when :part_body
+            part = @parts[@current]
+
+            body_separator = if part.is_a?(FilePart)
+              "#{CRLF}#{CRLF}"
+            else
+              CRLF
+            end
+            idx = @buffer.index(body_separator)
+
+            if idx
+              payload = @buffer.byteslice(0..idx - 1)
+              @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+              part << payload
+              part.rewind if part.respond_to?(:rewind)
+              @state = :parse_boundary
+            else
+              part << @buffer
+              @buffer.clear
+            end
+          when :parse_boundary
+            raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+            if @buffer == "--"
+              @buffer.clear
+              @state = :done
+              return
+            elsif @buffer.start_with?(CRLF)
+              @buffer = @buffer.byteslice(2..-1)
+              @state = :part_header
+            else
+              return
+            end
+          when :done
+            raise Error, "parsing should have been over by now"
+          end until @buffer.empty?
+        end
+      end
+    end
+  end
+end

data/lib/httpx/{plugins → transcoder}/multipart/encoder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-module HTTPX::Plugins
-  module Multipart
+module HTTPX
+  module Transcoder::Multipart
     class Encoder
       attr_reader :bytesize
 
@@ -43,7 +43,7 @@ module HTTPX::Plugins
       def to_parts(form)
         @bytesize = 0
         params = form.each_with_object([]) do |(key, val), aux|
-          Multipart.normalize_keys(key, val) do |k, v|
+          Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
             next if v.nil?
 
             value, content_type, filename = Part.call(v)

data/lib/httpx/{plugins → transcoder}/multipart/mime_type_detector.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  module Plugins::Multipart
+  module Transcoder::Multipart
     module MimeTypeDetector
       module_function
 

data/lib/httpx/{plugins → transcoder}/multipart/part.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  module Plugins::Multipart
+  module Transcoder::Multipart
     module Part
       module_function
 
@@ -21,7 +21,8 @@ module HTTPX
 
         value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
-        if value.is_a?(File)
+        if value.respond_to?(:path) && value.respond_to?(:read)
+          # either a File, a Tempfile, or something else which has to quack like a file
           filename ||= File.basename(value.path)
           content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
           [value, content_type, filename]

data/lib/httpx/transcoder/multipart.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "multipart/encoder"
+require_relative "multipart/decoder"
+require_relative "multipart/part"
+require_relative "multipart/mime_type_detector"
+
+module HTTPX::Transcoder
+  module Multipart
+    MULTIPART_VALUE_COND = lambda do |value|
+      value.respond_to?(:read) ||
+        (value.respond_to?(:to_hash) &&
+          value.key?(:body) &&
+          (value.key?(:filename) || value.key?(:content_type)))
+    end
+  end
+end

data/lib/httpx/transcoder/utils/body_reader.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module HTTPX
+  module Transcoder
+    class BodyReader
+      def initialize(body)
+        @body = if body.respond_to?(:read)
+          body.rewind if body.respond_to?(:rewind)
+          body
+        elsif body.respond_to?(:each)
+          body.enum_for(:each)
+        else
+          StringIO.new(body.to_s)
+        end
+      end
+
+      def bytesize
+        return @body.bytesize if @body.respond_to?(:bytesize)
+
+        Float::INFINITY
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @body.read(length, outbuf) if @body.respond_to?(:read)
+
+        begin
+          chunk = @body.next
+          if outbuf
+            outbuf.clear.force_encoding(Encoding::BINARY)
+            outbuf << chunk
+          else
+            outbuf = chunk
+          end
+          outbuf unless length && outbuf.empty?
+        rescue StopIteration
+        end
+      end
+
+      def close
+        @body.close if @body.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require_relative "body_reader"
+
+module HTTPX
+  module Transcoder
+    class Deflater
+      extend Forwardable
+
+      attr_reader :content_type
+
+      def initialize(body)
+        @content_type = body.content_type
+        @body = BodyReader.new(body)
+        @closed = false
+      end
+
+      def bytesize
+        buffer_deflate!
+
+        @buffer.size
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @buffer.read(length, outbuf) if @buffer
+
+        return if @closed
+
+        chunk = @body.read(length)
+
+        compressed_chunk = deflate(chunk)
+
+        return unless compressed_chunk
+
+        if outbuf
+          outbuf.clear.force_encoding(Encoding::BINARY)
+          outbuf << compressed_chunk
+        else
+          compressed_chunk
+        end
+      end
+
+      def close
+        return if @closed
+
+        @buffer.close if @buffer
+
+        @body.close
+
+        @closed = true
+      end
+
+      private
+
+      # rubocop:disable Naming/MemoizedInstanceVariableName
+      def buffer_deflate!
+        return @buffer if defined?(@buffer)
+
+        buffer = Response::Buffer.new(
+          threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
+        )
+        ::IO.copy_stream(self, buffer)
+
+        buffer.rewind
+
+        @buffer = buffer
+      end
+      # rubocop:enable Naming/MemoizedInstanceVariableName
+    end
+  end
+end

data/lib/httpx/transcoder/utils/inflater.rb

@@ -0,0 +1,19 @@
+module HTTPX
+  module Transcoder
+    class Inflater
+      def initialize(bytesize)
+        @bytesize = bytesize
+      end
+
+      def call(chunk)
+        buffer = @inflater.inflate(chunk)
+        @bytesize -= chunk.bytesize
+        if @bytesize <= 0
+          buffer << @inflater.finish
+          @inflater.close
+        end
+        buffer
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/xml.rb

@@ -6,8 +6,6 @@ require "uri"
 
 module HTTPX::Transcoder
   module Xml
-    using HTTPX::RegexpExtensions
-
     module_function
 
     MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
@@ -38,7 +36,6 @@ module HTTPX::Transcoder
     begin
       require "nokogiri"
 
-      # rubocop:disable Lint/DuplicateMethods
       def decode(response)
         content_type = response.content_type.mime_type
 
@@ -51,7 +48,5 @@ module HTTPX::Transcoder
         raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
       end
     end
-    # rubocop:enable Lint/DuplicateMethods
   end
-  register "xml", Xml
 end

data/lib/httpx/transcoder.rb

@@ -2,14 +2,10 @@
 
 module HTTPX
   module Transcoder
-    extend Registry
-
-    using RegexpExtensions unless Regexp.method_defined?(:match?)
-
     module_function
 
     def normalize_keys(key, value, cond = nil, &block)
-      if (cond && cond.call(value))
+      if cond && cond.call(value)
         block.call(key.to_s, value)
       elsif value.respond_to?(:to_ary)
         if value.empty?
@@ -73,7 +69,7 @@ module HTTPX
     end
 
     def params_hash_has_key?(hash, key)
-      return false if /\[\]/.match?(key)
+      return false if key.include?("[]")
 
       key.split(/[\[\]]+/).inject(hash) do |h, part|
         next h if part == ""
@@ -92,3 +88,5 @@ require "httpx/transcoder/form"
 require "httpx/transcoder/json"
 require "httpx/transcoder/xml"
 require "httpx/transcoder/chunker"
+require "httpx/transcoder/deflate"
+require "httpx/transcoder/gzip"

data/lib/httpx/utils.rb

@@ -4,6 +4,11 @@ module HTTPX
   module Utils
     using URIExtensions
 
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
+    FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
+
     module_function
 
     def now
@@ -25,31 +30,46 @@ module HTTPX
       time - Time.now
     end
 
-    if RUBY_VERSION < "2.3"
-
-      def to_uri(uri)
-        URI(uri)
+    def get_filename(header, _prefix_regex = nil)
+      filename = nil
+      case header
+      when FILENAME_REGEX
+        filename = Regexp.last_match(1)
+        filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+      when FILENAME_EXTENSION_REGEX
+        filename = Regexp.last_match(1)
+        encoding, _, filename = filename.split("'", 3)
       end
 
-    else
+      return unless filename
 
-      URIParser = URI::RFC2396_Parser.new
+      filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
 
-      def to_uri(uri)
-        return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+      filename.scrub!
 
-        uri = URI(URIParser.escape(uri))
+      filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
 
-        non_ascii_hostname = URIParser.unescape(uri.host)
+      filename.force_encoding ::Encoding.find(encoding) if encoding
 
-        non_ascii_hostname.force_encoding(Encoding::UTF_8)
+      filename
+    end
 
-        idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
+    URIParser = URI::RFC2396_Parser.new
 
-        uri.host = idna_hostname
-        uri.non_ascii_hostname = non_ascii_hostname
-        uri
-      end
+    def to_uri(uri)
+      return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+
+      uri = URI(URIParser.escape(uri))
+
+      non_ascii_hostname = URIParser.unescape(uri.host)
+
+      non_ascii_hostname.force_encoding(Encoding::UTF_8)
+
+      idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
+
+      uri.host = idna_hostname
+      uri.non_ascii_hostname = non_ascii_hostname
+      uri
     end
   end
 end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.21.0"
+  VERSION = "1.2.1"
 end

data/lib/httpx.rb

@@ -11,7 +11,6 @@ require "httpx/domain_name"
 require "httpx/altsvc"
 require "httpx/callbacks"
 require "httpx/loggable"
-require "httpx/registry"
 require "httpx/transcoder"
 require "httpx/timers"
 require "httpx/pool"
@@ -21,7 +20,6 @@ require "httpx/response"
 require "httpx/options"
 require "httpx/chainable"
 
-require "mutex_m"
 # Top-Level Namespace
 #
 module HTTPX
@@ -32,16 +30,17 @@ module HTTPX
   #
   module Plugins
     @plugins = {}
-    @plugins.extend(Mutex_m)
+    @plugins_mutex = Thread::Mutex.new
 
     # Loads a plugin based on a name. If the plugin hasn't been loaded, tries to load
     # it from the load path under "httpx/plugins/" directory.
     #
     def self.load_plugin(name)
       h = @plugins
-      unless (plugin = h.synchronize { h[name] })
+      m = @plugins_mutex
+      unless (plugin = m.synchronize { h[name] })
         require "httpx/plugins/#{name}"
-        raise "Plugin #{name} hasn't been registered" unless (plugin = h.synchronize { h[name] })
+        raise "Plugin #{name} hasn't been registered" unless (plugin = m.synchronize { h[name] })
       end
       plugin
     end
@@ -50,20 +49,19 @@ module HTTPX
     #
     def self.register_plugin(name, mod)
       h = @plugins
-      h.synchronize { h[name] = mod }
+      m = @plugins_mutex
+      m.synchronize { h[name] = mod }
     end
   end
 
-  # :nocov:
-  def self.const_missing(const_name)
-    super unless const_name == :Client
-    warn "DEPRECATION WARNING: the class #{self}::Client is deprecated. Use #{self}::Session instead."
-    Session
-  end
-  # :nocov:
-
   extend Chainable
 end
 
 require "httpx/session"
 require "httpx/session_extensions"
+
+# load integrations when possible
+
+require "httpx/adapters/datadog" if defined?(DDTrace) || defined?(Datadog)
+require "httpx/adapters/sentry" if defined?(Sentry)
+require "httpx/adapters/webmock" if defined?(WebMock)

data/sig/altsvc.rbs

@@ -0,0 +1,33 @@
+module HTTPX
+  module AltSvc
+    module ConnectionMixin
+
+      def send: (Request request) -> void
+
+      def match?: (URI::Generic uri, Options options) -> bool
+
+      private
+
+      def match_altsvcs?: (URI::Generic uri) -> bool
+
+      def match_altsvc_options?: (URI::Generic uri, Options options) -> bool
+    end
+
+    type altsvc_params = Hash[String, untyped]
+
+    def self?.cached_altsvc: (String origin) -> Array[altsvc_params]
+
+    def self?.cached_altsvc_set: (String origin, altsvc_params) -> void
+
+    def self?.lookup: (String origin, Integer | Float ttl) -> Array[altsvc_params]
+
+    def self?.emit: (Request request, response response) { (http_uri alt_origin, String origin, altsvc_params alt_params) -> void } -> void
+
+    def self?.parse: (String altsvc) { (http_uri alt_origin, altsvc_params alt_params) -> void } -> void
+                   | (String altsvc) -> Enumerable[[http_uri, altsvc_params]]
+
+    def self?.parse_altsvc_scheme: (String alt_proto) -> String?
+
+    def self.parse_altsvc_origin: (string alt_proto, String alt_origin) -> http_uri?
+  end
+end

data/sig/buffer.rbs

@@ -11,6 +11,7 @@ module HTTPX
 
     def full?: () -> bool
     def shift!: (Integer) -> void
+    def capacity: () -> Integer
 
     # delegated
     def <<: (string data) -> String

data/sig/callbacks.rbs

@@ -4,9 +4,9 @@ module HTTPX
   end
 
   module Callbacks
-    def on: (Symbol) { (*untyped) -> void } -> void
-    def once: (Symbol) { (*untyped) -> void } -> void
-    def only: (Symbol) { (*untyped) -> void } -> void
+    def on: (Symbol) { (*untyped) -> void } -> self
+    def once: (Symbol) { (*untyped) -> void } -> self
+    def only: (Symbol) { (*untyped) -> void } -> self
     def emit: (Symbol, *untyped) -> void
 
     def callbacks_for?: (Symbol) -> bool

data/sig/chainable.rbs

@@ -2,9 +2,9 @@ module HTTPX
   module Chainable
     def request: (*Request, **untyped) -> Array[response]
                | (Request, **untyped) -> response
-               | (verb | string, uri | [uri], **untyped) -> response
-               | (Array[[verb | string, uri] | [verb | string, uri, options]], **untyped) -> Array[response]
-               | (verb | string, _Each[uri | [uri, options]], **untyped) -> Array[response]
+               | (verb, uri | [uri], **untyped) -> response
+               | (Array[[verb, uri] | [verb, uri, options]], **untyped) -> Array[response]
+               | (verb, _Each[uri | [uri, options]], **untyped) -> Array[response]
 
     def accept: (String) -> Session
     def wrap: () { (Session) -> void } -> void
@@ -12,18 +12,17 @@ module HTTPX
     def with: (options) -> Session
             | (options) { (Session) -> void } -> void
 
-    def plugin: (:authentication, ?options) -> Plugins::sessionAuthentication
-              | (:basic_authentication, ?options) -> Plugins::sessionBasicAuth
-              | (:digest_authentication, ?options) -> Plugins::sessionDigestAuth
-              | (:ntlm_authentication, ?options) -> Plugins::sessionNTLMAuth
+    def plugin: (:auth, ?options) -> Plugins::sessionAuthorization
+              | (:basic_auth, ?options) -> Plugins::sessionBasicAuth
+              | (:digest_auth, ?options) -> Plugins::sessionDigestAuth
+              | (:ntlm_auth, ?options) -> Plugins::sessionNTLMAuth
               | (:aws_sdk_authentication, ?options) -> Plugins::sessionAwsSdkAuthentication
-              | (:compression, ?options) -> Session
+              | (:brotli, ?options) -> Session
               | (:cookies, ?options) -> Plugins::sessionCookies
               | (:expect, ?options) -> Session
               | (:follow_redirects, ?options) -> Plugins::sessionFollowRedirects
               | (:upgrade, ?options) -> Session
               | (:h2c, ?options) -> Session
-              | (:multipart, ?options) -> Session
               | (:persistent, ?options) -> Plugins::sessionPersistent
               | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
@@ -34,6 +33,8 @@ module HTTPX
               | (:grpc, ?options) -> Plugins::grpcSession
               | (:response_cache, ?options) -> Plugins::sessionResponseCache
               | (:circuit_breaker, ?options) -> Plugins::sessionCircuitBreaker
+              | (:oauth, ?options) -> Plugins::sessionOAuth
+              | (:callbacks, ?options) -> Plugins::sessionCallbacks
               | (Symbol | Module, ?options) { (Class) -> void } -> Session
               | (Symbol | Module, ?options) -> Session
 

data/sig/connection/http1.rbs

@@ -10,8 +10,9 @@ module HTTPX
     attr_reader pending: Array[Request]
     attr_reader requests: Array[Request]
 
+    attr_accessor max_concurrent_requests: Integer
+
     @options: Options
-    @max_concurrent_requests: Integer
     @max_requests: Integer
     @parser: Parser::HTTP1
     @buffer: Buffer
@@ -53,19 +54,19 @@ module HTTPX
 
     def ping: () -> void
 
-    def timeout: () -> Numeric
+    def timeout: () -> Numeric?
 
     private
 
     def initialize: (Buffer, options) -> untyped
 
-    def manage_connection: (Response) -> void
+    def manage_connection: (Request request, Response response) -> void
 
     def disable: () -> void
 
     def disable_pipelining: () -> void
 
-    def set_protocol_headers: (Request) -> _Each[[String, String]]
+    def set_protocol_headers: (Request request) -> _Each[[String, String]]
 
     def handle: (Request request) -> void