httpx 0.20.0 → 1.3.1

data/lib/httpx/transcoder/gzip.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "uri"
+require "stringio"
+require "zlib"
+
+module HTTPX
+  module Transcoder
+    module GZIP
+      class Deflater < Transcoder::Deflater
+        def initialize(body)
+          @compressed_chunk = "".b
+          super
+        end
+
+        def deflate(chunk)
+          @deflater ||= Zlib::GzipWriter.new(self)
+
+          if chunk.nil?
+            unless @deflater.closed?
+              @deflater.flush
+              @deflater.close
+              compressed_chunk
+            end
+          else
+            @deflater.write(chunk)
+            compressed_chunk
+          end
+        end
+
+        private
+
+        def write(chunk)
+          @compressed_chunk << chunk
+        end
+
+        def compressed_chunk
+          @compressed_chunk.dup
+        ensure
+          @compressed_chunk.clear
+        end
+      end
+
+      class Inflater
+        def initialize(bytesize)
+          @inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+          @bytesize = bytesize
+        end
+
+        def call(chunk)
+          buffer = @inflater.inflate(chunk)
+          @bytesize -= chunk.bytesize
+          if @bytesize <= 0
+            buffer << @inflater.finish
+            @inflater.close
+          end
+          buffer
+        end
+      end
+
+      module_function
+
+      def encode(body)
+        Deflater.new(body)
+      end
+
+      def decode(response, bytesize: nil)
+        bytesize ||= response.headers.key?("content-length") ? response.headers["content-length"].to_i : Float::INFINITY
+        Inflater.new(bytesize)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/json.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "json"
 
 module HTTPX::Transcoder
   module JSON
-    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+)?json\b}i.freeze
-
-    using HTTPX::RegexpExtensions unless Regexp.method_defined?(:match?)
-
     module_function
 
+    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+|hal\+)?json\b}i.freeze
+
     class Encoder
       extend Forwardable
 
@@ -19,7 +16,7 @@ module HTTPX::Transcoder
       def_delegator :@raw, :bytesize
 
       def initialize(json)
-        @raw = ::JSON.dump(json)
+        @raw = JSON.json_dump(json)
         @charset = @raw.encoding.name.downcase
       end
 
@@ -37,8 +34,24 @@ module HTTPX::Transcoder
 
       raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
 
-      ::JSON.method(:parse)
+      method(:json_load)
+    end
+
+    # rubocop:disable Style/SingleLineMethods
+    if defined?(MultiJson)
+      def json_load(*args); MultiJson.load(*args); end
+      def json_dump(*args); MultiJson.dump(*args); end
+    elsif defined?(Oj)
+      def json_load(response, *args); Oj.load(response.to_s, *args); end
+      def json_dump(*args); Oj.dump(*args); end
+    elsif defined?(Yajl)
+      def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
+      def json_dump(*args); Yajl::Encoder.encode(*args); end
+    else
+      require "json"
+      def json_load(*args); ::JSON.parse(*args); end
+      def json_dump(*args); ::JSON.dump(*args); end
     end
+    # rubocop:enable Style/SingleLineMethods
   end
-  register "json", JSON
 end

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require "delegate"
+
+module HTTPX
+  module Transcoder
+    module Multipart
+      class FilePart < SimpleDelegator
+        attr_reader :original_filename, :content_type
+
+        def initialize(filename, content_type)
+          @original_filename = filename
+          @content_type = content_type
+          @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+          super(@file)
+        end
+      end
+
+      class Decoder
+        include HTTPX::Utils
+
+        CRLF = "\r\n"
+        BOUNDARY_RE = /;\s*boundary=([^;]+)/i.freeze
+        MULTIPART_CONTENT_TYPE = /Content-Type: (.*)#{CRLF}/ni.freeze
+        MULTIPART_CONTENT_DISPOSITION = /Content-Disposition:.*;\s*name=(#{VALUE})/ni.freeze
+        MULTIPART_CONTENT_ID = /Content-ID:\s*([^#{CRLF}]*)/ni.freeze
+        WINDOW_SIZE = 2 << 14
+
+        def initialize(response)
+          @boundary = begin
+            m = response.headers["content-type"].to_s[BOUNDARY_RE, 1]
+            raise Error, "no boundary declared in content-type header" unless m
+
+            m.strip
+          end
+          @buffer = "".b
+          @parts = {}
+          @intermediate_boundary = "--#{@boundary}"
+          @state = :idle
+        end
+
+        def call(response, *)
+          response.body.each do |chunk|
+            @buffer << chunk
+
+            parse
+          end
+
+          raise Error, "invalid or unsupported multipart format" unless @buffer.empty?
+
+          @parts
+        end
+
+        private
+
+        def parse
+          case @state
+          when :idle
+            raise Error, "payload does not start with boundary" unless @buffer.start_with?("#{@intermediate_boundary}#{CRLF}")
+
+            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize + 2..-1)
+
+            @state = :part_header
+          when :part_header
+            idx = @buffer.index("#{CRLF}#{CRLF}")
+
+            # raise Error, "couldn't parse part headers" unless idx
+            return unless idx
+
+            head = @buffer.byteslice(0..idx + 4 - 1)
+
+            @buffer = @buffer.byteslice(head.bytesize..-1)
+
+            content_type = head[MULTIPART_CONTENT_TYPE, 1]
+            if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
+              name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
+              name.gsub!(/\\(.)/, "\\1")
+              name
+            else
+              name = head[MULTIPART_CONTENT_ID, 1]
+            end
+
+            filename = HTTPX::Utils.get_filename(head)
+
+            name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+
+            @current = name
+
+            @parts[name] = if filename
+              FilePart.new(filename, content_type)
+            else
+              "".b
+            end
+
+            @state = :part_body
+          when :part_body
+            part = @parts[@current]
+
+            body_separator = if part.is_a?(FilePart)
+              "#{CRLF}#{CRLF}"
+            else
+              CRLF
+            end
+            idx = @buffer.index(body_separator)
+
+            if idx
+              payload = @buffer.byteslice(0..idx - 1)
+              @buffer = @buffer.byteslice(idx + body_separator.bytesize..-1)
+              part << payload
+              part.rewind if part.respond_to?(:rewind)
+              @state = :parse_boundary
+            else
+              part << @buffer
+              @buffer.clear
+            end
+          when :parse_boundary
+            raise Error, "payload does not start with boundary" unless @buffer.start_with?(@intermediate_boundary)
+
+            @buffer = @buffer.byteslice(@intermediate_boundary.bytesize..-1)
+
+            if @buffer == "--"
+              @buffer.clear
+              @state = :done
+              return
+            elsif @buffer.start_with?(CRLF)
+              @buffer = @buffer.byteslice(2..-1)
+              @state = :part_header
+            else
+              return
+            end
+          when :done
+            raise Error, "parsing should have been over by now"
+          end until @buffer.empty?
+        end
+      end
+    end
+  end
+end

data/lib/httpx/{plugins → transcoder}/multipart/encoder.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-module HTTPX::Plugins
-  module Multipart
+module HTTPX
+  module Transcoder::Multipart
     class Encoder
       attr_reader :bytesize
 
@@ -19,7 +19,7 @@ module HTTPX::Plugins
       end
 
       def read(length = nil, outbuf = nil)
-        data   = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+        data   = String(outbuf).clear.force_encoding(Encoding::BINARY) if outbuf
         data ||= "".b
 
         read_chunks(data, length)
@@ -43,7 +43,7 @@ module HTTPX::Plugins
       def to_parts(form)
         @bytesize = 0
         params = form.each_with_object([]) do |(key, val), aux|
-          Multipart.normalize_keys(key, val) do |k, v|
+          Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
             next if v.nil?
 
             value, content_type, filename = Part.call(v)

data/lib/httpx/{plugins → transcoder}/multipart/mime_type_detector.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  module Plugins::Multipart
+  module Transcoder::Multipart
     module MimeTypeDetector
       module_function
 

data/lib/httpx/{plugins → transcoder}/multipart/part.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  module Plugins::Multipart
+  module Transcoder::Multipart
     module Part
       module_function
 
@@ -21,7 +21,8 @@ module HTTPX
 
         value = value.open(File::RDONLY) if Object.const_defined?(:Pathname) && value.is_a?(Pathname)
 
-        if value.is_a?(File)
+        if value.respond_to?(:path) && value.respond_to?(:read)
+          # either a File, a Tempfile, or something else which has to quack like a file
           filename ||= File.basename(value.path)
           content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
           [value, content_type, filename]

data/lib/httpx/transcoder/multipart.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "multipart/encoder"
+require_relative "multipart/decoder"
+require_relative "multipart/part"
+require_relative "multipart/mime_type_detector"
+
+module HTTPX::Transcoder
+  module Multipart
+    MULTIPART_VALUE_COND = lambda do |value|
+      value.respond_to?(:read) ||
+        (value.respond_to?(:to_hash) &&
+          value.key?(:body) &&
+          (value.key?(:filename) || value.key?(:content_type)))
+    end
+  end
+end

data/lib/httpx/transcoder/utils/body_reader.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module HTTPX
+  module Transcoder
+    class BodyReader
+      def initialize(body)
+        @body = if body.respond_to?(:read)
+          body.rewind if body.respond_to?(:rewind)
+          body
+        elsif body.respond_to?(:each)
+          body.enum_for(:each)
+        else
+          StringIO.new(body.to_s)
+        end
+      end
+
+      def bytesize
+        return @body.bytesize if @body.respond_to?(:bytesize)
+
+        Float::INFINITY
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @body.read(length, outbuf) if @body.respond_to?(:read)
+
+        begin
+          chunk = @body.next
+          if outbuf
+            outbuf.clear.force_encoding(Encoding::BINARY)
+            outbuf << chunk
+          else
+            outbuf = chunk
+          end
+          outbuf unless length && outbuf.empty?
+        rescue StopIteration
+        end
+      end
+
+      def close
+        @body.close if @body.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require_relative "body_reader"
+
+module HTTPX
+  module Transcoder
+    class Deflater
+      extend Forwardable
+
+      attr_reader :content_type
+
+      def initialize(body)
+        @content_type = body.content_type
+        @body = BodyReader.new(body)
+        @closed = false
+      end
+
+      def bytesize
+        buffer_deflate!
+
+        @buffer.size
+      end
+
+      def read(length = nil, outbuf = nil)
+        return @buffer.read(length, outbuf) if @buffer
+
+        return if @closed
+
+        chunk = @body.read(length)
+
+        compressed_chunk = deflate(chunk)
+
+        return unless compressed_chunk
+
+        if outbuf
+          outbuf.clear.force_encoding(Encoding::BINARY)
+          outbuf << compressed_chunk
+        else
+          compressed_chunk
+        end
+      end
+
+      def close
+        return if @closed
+
+        @buffer.close if @buffer
+
+        @body.close
+
+        @closed = true
+      end
+
+      private
+
+      # rubocop:disable Naming/MemoizedInstanceVariableName
+      def buffer_deflate!
+        return @buffer if defined?(@buffer)
+
+        buffer = Response::Buffer.new(
+          threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
+        )
+        ::IO.copy_stream(self, buffer)
+
+        buffer.rewind
+
+        @buffer = buffer
+      end
+      # rubocop:enable Naming/MemoizedInstanceVariableName
+    end
+  end
+end

data/lib/httpx/transcoder/utils/inflater.rb

@@ -0,0 +1,19 @@
+module HTTPX
+  module Transcoder
+    class Inflater
+      def initialize(bytesize)
+        @bytesize = bytesize
+      end
+
+      def call(chunk)
+        buffer = @inflater.inflate(chunk)
+        @bytesize -= chunk.bytesize
+        if @bytesize <= 0
+          buffer << @inflater.finish
+          @inflater.close
+        end
+        buffer
+      end
+    end
+  end
+end

data/lib/httpx/transcoder/xml.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "forwardable"
+require "uri"
+
+module HTTPX::Transcoder
+  module Xml
+    module_function
+
+    MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
+
+    class Encoder
+      def initialize(xml)
+        @raw = xml
+      end
+
+      def content_type
+        charset = @raw.respond_to?(:encoding) ? @raw.encoding.to_s.downcase : "utf-8"
+        "application/xml; charset=#{charset}"
+      end
+
+      def bytesize
+        @raw.to_s.bytesize
+      end
+
+      def to_s
+        @raw.to_s
+      end
+    end
+
+    def encode(xml)
+      Encoder.new(xml)
+    end
+
+    begin
+      require "nokogiri"
+
+      def decode(response)
+        content_type = response.content_type.mime_type
+
+        raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
+
+        Nokogiri::XML.method(:parse)
+      end
+    rescue LoadError
+      def decode(_response)
+        raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
+      end
+    end
+  end
+end

data/lib/httpx/transcoder.rb

@@ -2,14 +2,10 @@
 
 module HTTPX
   module Transcoder
-    extend Registry
-
-    using RegexpExtensions unless Regexp.method_defined?(:match?)
-
     module_function
 
     def normalize_keys(key, value, cond = nil, &block)
-      if (cond && cond.call(value))
+      if cond && cond.call(value)
         block.call(key.to_s, value)
       elsif value.respond_to?(:to_ary)
         if value.empty?
@@ -73,7 +69,7 @@ module HTTPX
     end
 
     def params_hash_has_key?(hash, key)
-      return false if /\[\]/.match?(key)
+      return false if key.include?("[]")
 
       key.split(/[\[\]]+/).inject(hash) do |h, part|
         next h if part == ""
@@ -90,4 +86,7 @@ end
 require "httpx/transcoder/body"
 require "httpx/transcoder/form"
 require "httpx/transcoder/json"
+require "httpx/transcoder/xml"
 require "httpx/transcoder/chunker"
+require "httpx/transcoder/deflate"
+require "httpx/transcoder/gzip"

data/lib/httpx/utils.rb

@@ -4,6 +4,11 @@ module HTTPX
   module Utils
     using URIExtensions
 
+    TOKEN = %r{[^\s()<>,;:\\"/\[\]?=]+}.freeze
+    VALUE = /"(?:\\"|[^"])*"|#{TOKEN}/.freeze
+    FILENAME_REGEX = /\s*filename=(#{VALUE})/.freeze
+    FILENAME_EXTENSION_REGEX = /\s*filename\*=(#{VALUE})/.freeze
+
     module_function
 
     def now
@@ -25,31 +30,46 @@ module HTTPX
       time - Time.now
     end
 
-    if RUBY_VERSION < "2.3"
-
-      def to_uri(uri)
-        URI(uri)
+    def get_filename(header, _prefix_regex = nil)
+      filename = nil
+      case header
+      when FILENAME_REGEX
+        filename = Regexp.last_match(1)
+        filename = Regexp.last_match(1) if filename =~ /^"(.*)"$/
+      when FILENAME_EXTENSION_REGEX
+        filename = Regexp.last_match(1)
+        encoding, _, filename = filename.split("'", 3)
       end
 
-    else
+      return unless filename
 
-      URIParser = URI::RFC2396_Parser.new
+      filename = URI::DEFAULT_PARSER.unescape(filename) if filename.scan(/%.?.?/).all? { |s| /%[0-9a-fA-F]{2}/.match?(s) }
 
-      def to_uri(uri)
-        return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+      filename.scrub!
 
-        uri = URI(URIParser.escape(uri))
+      filename = filename.gsub(/\\(.)/, '\1') unless /\\[^\\"]/.match?(filename)
 
-        non_ascii_hostname = URIParser.unescape(uri.host)
+      filename.force_encoding ::Encoding.find(encoding) if encoding
 
-        non_ascii_hostname.force_encoding(Encoding::UTF_8)
+      filename
+    end
 
-        idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
+    URIParser = URI::RFC2396_Parser.new
 
-        uri.host = idna_hostname
-        uri.non_ascii_hostname = non_ascii_hostname
-        uri
-      end
+    def to_uri(uri)
+      return URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
+
+      uri = URI(URIParser.escape(uri))
+
+      non_ascii_hostname = URIParser.unescape(uri.host)
+
+      non_ascii_hostname.force_encoding(Encoding::UTF_8)
+
+      idna_hostname = Punycode.encode_hostname(non_ascii_hostname)
+
+      uri.host = idna_hostname
+      uri.non_ascii_hostname = non_ascii_hostname
+      uri
     end
   end
 end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.20.0"
+  VERSION = "1.3.1"
 end