httparty 0.21.0 → 0.24.3

data/lib/httparty/request/streaming_multipart_body.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module HTTParty
+  class Request
+    class StreamingMultipartBody
+      NEWLINE = "\r\n"
+      CHUNK_SIZE = 64 * 1024 # 64 KB chunks
+
+      def initialize(parts, boundary)
+        @parts = parts
+        @boundary = boundary
+        @part_index = 0
+        @state = :header
+        @current_file = nil
+        @header_buffer = nil
+        @header_offset = 0
+        @footer_sent = false
+      end
+
+      def size
+        @size ||= calculate_size
+      end
+
+      def read(length = nil, outbuf = nil)
+        outbuf = outbuf ? outbuf.replace(''.b) : ''.b
+
+        return read_all(outbuf) if length.nil?
+
+        while outbuf.bytesize < length
+          chunk = read_chunk(length - outbuf.bytesize)
+          break if chunk.nil?
+          outbuf << chunk
+        end
+
+        outbuf.empty? ? nil : outbuf
+      end
+
+      def rewind
+        @part_index = 0
+        @state = :header
+        @current_file = nil
+        @header_buffer = nil
+        @header_offset = 0
+        @footer_sent = false
+        @parts.each do |_key, value, _is_file|
+          value.rewind if value.respond_to?(:rewind)
+        end
+      end
+
+      private
+
+      def read_all(outbuf)
+        while (chunk = read_chunk(CHUNK_SIZE))
+          outbuf << chunk
+        end
+        outbuf.empty? ? nil : outbuf
+      end
+
+      def read_chunk(max_length)
+        loop do
+          return nil if @part_index >= @parts.size && @footer_sent
+
+          if @part_index >= @parts.size
+            @footer_sent = true
+            return "--#{@boundary}--#{NEWLINE}".b
+          end
+
+          key, value, is_file = @parts[@part_index]
+
+          case @state
+          when :header
+            chunk = read_header_chunk(key, value, is_file, max_length)
+            return chunk if chunk
+
+          when :body
+            chunk = read_body_chunk(value, is_file, max_length)
+            return chunk if chunk
+
+          when :newline
+            @state = :header
+            @part_index += 1
+            return NEWLINE.b
+          end
+        end
+      end
+
+      def read_header_chunk(key, value, is_file, max_length)
+        if @header_buffer.nil?
+          @header_buffer = build_part_header(key, value, is_file)
+          @header_offset = 0
+        end
+
+        remaining = @header_buffer.bytesize - @header_offset
+        if remaining > 0
+          chunk_size = [remaining, max_length].min
+          chunk = @header_buffer.byteslice(@header_offset, chunk_size)
+          @header_offset += chunk_size
+          return chunk
+        end
+
+        @header_buffer = nil
+        @header_offset = 0
+        @state = :body
+        nil
+      end
+
+      def read_body_chunk(value, is_file, max_length)
+        if is_file
+          chunk = read_file_chunk(value, max_length)
+          if chunk
+            return chunk
+          else
+            @current_file = nil
+            @state = :newline
+            return nil
+          end
+        else
+          @state = :newline
+          return value.to_s.b
+        end
+      end
+
+      def read_file_chunk(file, max_length)
+        chunk_size = [max_length, CHUNK_SIZE].min
+        chunk = file.read(chunk_size)
+        return nil if chunk.nil?
+        chunk.force_encoding(Encoding::BINARY) if chunk.respond_to?(:force_encoding)
+        chunk
+      end
+
+      def build_part_header(key, value, is_file)
+        header = "--#{@boundary}#{NEWLINE}".b
+        header << %(Content-Disposition: form-data; name="#{key}").b
+        if is_file
+          header << %(; filename="#{file_name(value).gsub(/["\r\n]/, replacement_table)}").b
+          header << NEWLINE.b
+          header << "Content-Type: #{content_type(value)}#{NEWLINE}".b
+        else
+          header << NEWLINE.b
+        end
+        header << NEWLINE.b
+        header
+      end
+
+      def calculate_size
+        total = 0
+        @parts.each do |key, value, is_file|
+          total += build_part_header(key, value, is_file).bytesize
+          total += content_size(value, is_file)
+          total += NEWLINE.bytesize
+        end
+        total += "--#{@boundary}--#{NEWLINE}".bytesize
+        total
+      end
+
+      def content_size(value, is_file)
+        if is_file
+          if value.respond_to?(:size)
+            value.size
+          elsif value.respond_to?(:stat)
+            value.stat.size
+          else
+            value.read.bytesize.tap { value.rewind }
+          end
+        else
+          value.to_s.b.bytesize
+        end
+      end
+
+      def content_type(object)
+        return object.content_type if object.respond_to?(:content_type)
+        require 'mini_mime'
+        mime = MiniMime.lookup_by_filename(object.path)
+        mime ? mime.content_type : 'application/octet-stream'
+      end
+
+      def file_name(object)
+        object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
+      end
+
+      def replacement_table
+        @replacement_table ||= {
+          '"'  => '%22',
+          "\r" => '%0D',
+          "\n" => '%0A'
+        }.freeze
+      end
+    end
+  end
+end

data/lib/httparty/request.rb

@@ -113,6 +113,8 @@ module HTTParty
         new_uri = path.clone
       end
 
+      validate_uri_safety!(new_uri) unless redirect
+
       # avoid double query string on redirects [#12]
       unless redirect
         new_uri.query = query_string(new_uri)
@@ -153,24 +155,28 @@ module HTTParty
       chunked_body = nil
       current_http = http
 
-      self.last_response = current_http.request(@raw_request) do |http_response|
-        if block
-          chunks = []
+      begin
+        self.last_response = current_http.request(@raw_request) do |http_response|
+          if block
+            chunks = []
 
-          http_response.read_body do |fragment|
-            encoded_fragment = encode_text(fragment, http_response['content-type'])
-            chunks << encoded_fragment if !options[:stream_body]
-            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
-          end
+            http_response.read_body do |fragment|
+              encoded_fragment = encode_text(fragment, http_response['content-type'])
+              chunks << encoded_fragment if !options[:stream_body]
+              block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
+            end
 
-          chunked_body = chunks.join
+            chunked_body = chunks.join
+          end
         end
-      end
 
-      handle_host_redirection if response_redirects?
-      result = handle_unauthorized
-      result ||= handle_response(chunked_body, &block)
-      result
+        handle_host_redirection if response_redirects?
+        result = handle_unauthorized
+        result ||= handle_response(chunked_body, &block)
+        result
+      rescue *COMMON_NETWORK_ERRORS => e
+        raise options[:foul] ? HTTParty::NetworkError.new("#{e.class}: #{e.message}") : e
+      end
     end
 
     def handle_unauthorized(&block)
@@ -241,8 +247,17 @@ module HTTParty
         if body.multipart?
           content_type = "multipart/form-data; boundary=#{body.boundary}"
           @raw_request['Content-Type'] = content_type
+        elsif options[:body].respond_to?(:to_hash) && !@raw_request['Content-Type']
+          @raw_request['Content-Type'] = 'application/x-www-form-urlencoded'
+        end
+
+        if body.streaming? && options[:stream_body] == true
+          stream = body.to_stream
+          @raw_request.body_stream = stream
+          @raw_request['Content-Length'] = stream.size.to_s
+        else
+          @raw_request.body = body.call
         end
-        @raw_request.body = body.call
       end
 
       @raw_request.instance_variable_set(:@decode_content, decompress_content?)
@@ -295,24 +310,7 @@ module HTTParty
 
     def handle_response(raw_body, &block)
       if response_redirects?
-        options[:limit] -= 1
-        if options[:logger]
-          logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
-          logger.format(self, last_response)
-        end
-        self.path = last_response['location']
-        self.redirect = true
-        if last_response.class == Net::HTTPSeeOther
-          unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
-            self.http_method = Net::HTTP::Get
-          end
-        elsif last_response.code != '307' && last_response.code != '308'
-          unless options[:maintain_method_across_redirects]
-            self.http_method = Net::HTTP::Get
-          end
-        end
-        capture_cookies(last_response)
-        perform(&block)
+        handle_redirection(&block)
       else
         raw_body ||= last_response.body
 
@@ -331,10 +329,34 @@ module HTTParty
       end
     end
 
+    def handle_redirection(&block)
+      options[:limit] -= 1
+      if options[:logger]
+        logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
+        logger.format(self, last_response)
+      end
+      self.path       = last_response['location']
+      self.redirect   = true
+      if last_response.class == Net::HTTPSeeOther
+        unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
+          self.http_method = Net::HTTP::Get
+        end
+      elsif last_response.code != '307' && last_response.code != '308'
+        unless options[:maintain_method_across_redirects]
+          self.http_method = Net::HTTP::Get
+        end
+      end
+      if http_method == Net::HTTP::Get
+        clear_body
+      end
+      capture_cookies(last_response)
+      perform(&block)
+    end
+
     def handle_host_redirection
       check_duplicate_location_header
       redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
-      return if redirect_path.relative? || path.host == redirect_path.host
+      return if redirect_path.relative? || path.host == redirect_path.host || uri.host == redirect_path.host
       @changed_hosts = true
     end
 
@@ -362,6 +384,14 @@ module HTTParty
       parser.call(body, format)
     end
 
+    # Some Web Application Firewalls reject incoming GET requests that have a body
+    # if we redirect, and the resulting verb is GET then we will clear the body that
+    # may be left behind from the initiating request
+    def clear_body
+      options[:body] = nil
+      @raw_request.body = nil
+    end
+
     def capture_cookies(response)
       return unless response['Set-Cookie']
       cookies_hash = HTTParty::CookieHash.new
@@ -414,5 +444,23 @@ module HTTParty
         assume_utf16_is_big_endian: assume_utf16_is_big_endian
       ).call
     end
+
+    def validate_uri_safety!(new_uri)
+      return if options[:skip_uri_validation]
+
+      configured_base_uri = options[:base_uri]
+      return unless configured_base_uri
+
+      normalized_base = options[:uri_adapter].parse(
+        HTTParty.normalize_base_uri(configured_base_uri)
+      )
+
+      return if new_uri.host == normalized_base.host
+
+      raise UnsafeURIError,
+        "Requested URI '#{new_uri}' has host '#{new_uri.host}' but the " \
+        "configured base_uri '#{normalized_base}' has host '#{normalized_base.host}'. " \
+        "This request could send credentials to an unintended server."
+    end
   end
 end

data/lib/httparty/response.rb

@@ -67,12 +67,12 @@ module HTTParty
     end
 
     # Support old multiple_choice? method from pre 2.0.0 era.
-    if ::RUBY_VERSION >= '2.0.0' && ::RUBY_PLATFORM != 'java'
+    if ::RUBY_PLATFORM != 'java'
       alias_method :multiple_choice?, :multiple_choices?
     end
 
     # Support old status codes method from pre 2.6.0 era.
-    if ::RUBY_VERSION >= '2.6.0' && ::RUBY_PLATFORM != 'java'
+    if ::RUBY_PLATFORM != 'java'
       alias_method :gateway_time_out?,                :gateway_timeout?
       alias_method :request_entity_too_large?,        :payload_too_large?
       alias_method :request_time_out?,                :request_timeout?
@@ -133,13 +133,27 @@ module HTTParty
     end
 
     def throw_exception
-      if @request.options[:raise_on] && @request.options[:raise_on].include?(code)
+      raise_on = @request.options[:raise_on]
+      return if raise_on.nil?
+
+      matchers = raise_on.is_a?(Range) ? [raise_on] : [*raise_on]
+
+      if matchers.any? { |matcher| code_matches?(matcher) }
         ::Kernel.raise ::HTTParty::ResponseError.new(@response), "Code #{code} - #{body}"
       end
     end
 
     private
 
+    def code_matches?(matcher)
+      case matcher
+      when Range
+        matcher.cover?(code) || matcher.cover?(code.to_s)
+      else
+        /#{matcher}/.match?(code.to_s)
+      end
+    end
+
     def warn_about_nil_deprecation
       trace_line = caller.reject { |line| line.include?('httparty') }.first
       warning = "[DEPRECATION] HTTParty will no longer override `response#nil?`. " \

data/lib/httparty/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTParty
-  VERSION = '0.21.0'
+  VERSION = '0.24.3'
 end

data/lib/httparty.rb

@@ -3,11 +3,6 @@
 require 'pathname'
 require 'net/http'
 require 'uri'
-require 'zlib'
-require 'multi_xml'
-require 'mini_mime'
-require 'json'
-require 'csv'
 
 require 'httparty/module_inheritable_attributes'
 require 'httparty/cookie_hash'
@@ -46,7 +41,7 @@ module HTTParty
   # [:+local_host+:] Local address to bind to before connecting.
   # [:+local_port+:] Local port to bind to before connecting.
   # [:+body_stream+:] Allow streaming to a REST server to specify a body_stream.
-  # [:+stream_body+:] Allow for streaming large files without loading them into memory.
+  # [:+stream_body+:] When downloading with a block, avoids accumulating the response in memory. When uploading files, streams the request body to reduce memory usage (opt-in).
   # [:+multipart+:] Force content-type to be multipart
   #
   # There are also another set of options with names corresponding to various class methods. The methods in question are those that let you set a class-wide default, and the options override the defaults on a request-by-request basis. Those options are:
@@ -67,6 +62,16 @@ module HTTParty
   # * :+ssl_ca_path+: see HTTParty::ClassMethods.ssl_ca_path.
 
   module ClassMethods
+    # Turns on or off the foul option.
+    #
+    #   class Foo
+    #     include HTTParty
+    #     foul true
+    #   end
+    def foul(bool)
+      default_options[:foul] = bool
+    end
+
     # Turns on logging
     #
     #   class Foo
@@ -79,14 +84,15 @@ module HTTParty
       default_options[:log_format] = format
     end
 
-    # Raises HTTParty::ResponseError if response's code matches this statuses
+    # Raises HTTParty::ResponseError if the response status matches a code,
+    # range, or regular expression string.
     #
     #   class Foo
     #     include HTTParty
-    #     raise_on [404, 500]
+    #     raise_on [404, 500..599, '3[0-9]*']
     #   end
     def raise_on(codes = [])
-      default_options[:raise_on] = *codes
+      default_options[:raise_on] = codes.is_a?(Range) ? [codes] : [*codes]
     end
 
     # Allows setting http proxy information to be used
@@ -591,6 +597,13 @@ module HTTParty
       perform_request Net::HTTP::Unlock, path, options, &block
     end
 
+    def build_request(http_method, path, options = {})
+      options = ModuleInheritableAttributes.hash_deep_dup(default_options).merge(options)
+      HeadersProcessor.new(headers, options).call
+      process_cookies(options)
+      Request.new(http_method, path, options)
+    end
+
     attr_reader :default_options
 
     private
@@ -606,10 +619,7 @@ module HTTParty
     end
 
     def perform_request(http_method, path, options, &block) #:nodoc:
-      options = ModuleInheritableAttributes.hash_deep_dup(default_options).merge(options)
-      HeadersProcessor.new(headers, options).call
-      process_cookies(options)
-      Request.new(http_method, path, options).perform(&block)
+      build_request(http_method, path, options).perform(&block)
     end
 
     def process_cookies(options) #:nodoc:
@@ -676,6 +686,10 @@ module HTTParty
   def self.options(*args, &block)
     Basement.options(*args, &block)
   end
+
+  def self.build_request(*args, &block)
+    Basement.build_request(*args, &block)
+  end
 end
 
 require 'httparty/hash_conversions'

data/script/release

@@ -18,9 +18,9 @@ gem_name=httparty
 rm -rf $gem_name-*.gem
 gem build -q $gem_name.gemspec
 
-# Make sure we're on the master branch.
-(git branch | grep -q '* master') || {
-  echo "Only release from the master branch."
+# Make sure we're on the main branch.
+(git branch | grep -q '* main') || {
+  echo "Only release from the main branch."
   exit 1
 }
 
@@ -39,4 +39,4 @@ git fetch -t origin
 
 # Tag it and bag it.
 gem push $gem_name-*.gem && git tag "$tag" &&
-  git push origin master && git push origin "$tag"
+  git push origin main && git push origin "$tag"

metadata

@@ -1,16 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: httparty
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.24.3
 platform: ruby
 authors:
 - John Nunemaker
 - Sandro Turriate
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-30 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_xml
   requirement: !ruby/object:Gem::Requirement
@@ -48,6 +61,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -76,6 +90,7 @@ files:
 - examples/microsoft_graph.rb
 - examples/multipart.rb
 - examples/nokogiri_html_parser.rb
+- examples/party_foul_mode.rb
 - examples/peer_cert.rb
 - examples/rescue_json.rb
 - examples/rubyurl.rb
@@ -102,6 +117,7 @@ files:
 - lib/httparty/request.rb
 - lib/httparty/request/body.rb
 - lib/httparty/request/multipart_boundary.rb
+- lib/httparty/request/streaming_multipart_body.rb
 - lib/httparty/response.rb
 - lib/httparty/response/headers.rb
 - lib/httparty/response_fragment.rb
@@ -114,7 +130,8 @@ files:
 homepage: https://github.com/jnunemaker/httparty
 licenses:
 - MIT
-metadata: {}
+metadata:
+  changelog_uri: https://github.com/jnunemaker/httparty/releases
 post_install_message: When you HTTParty, you must party hard!
 rdoc_options: []
 require_paths:
@@ -123,15 +140,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Makes http fun! Also, makes consuming restful web services dead easy.
 test_files: []