httparty-responsibly 0.17.0.r1

data/lib/httparty/logger/logger.rb

@@ -0,0 +1,28 @@
+require 'httparty/logger/apache_formatter'
+require 'httparty/logger/curl_formatter'
+require 'httparty/logger/logstash_formatter'
+
+module HTTParty
+  module Logger
+    def self.formatters
+      @formatters ||= {
+        :curl => Logger::CurlFormatter,
+        :apache => Logger::ApacheFormatter,
+        :logstash => Logger::LogstashFormatter,
+      }
+    end
+
+    def self.add_formatter(name, formatter)
+      raise HTTParty::Error.new("Log Formatter with name #{name} already exists") if formatters.include?(name)
+      formatters.merge!(name.to_sym => formatter)
+    end
+
+    def self.build(logger, level, formatter)
+      level ||= :info
+      formatter ||= :apache
+
+      logger_klass = formatters[formatter] || Logger::ApacheFormatter
+      logger_klass.new(logger, level)
+    end
+  end
+end

data/lib/httparty/logger/logstash_formatter.rb

@@ -0,0 +1,59 @@
+module HTTParty
+  module Logger
+    class LogstashFormatter #:nodoc:
+      TAG_NAME = HTTParty.name
+
+      attr_accessor :level, :logger
+
+      def initialize(logger, level)
+        @logger = logger
+        @level  = level.to_sym
+      end
+
+      def format(request, response)
+        @request = request
+        @response = response
+
+        logger.public_send level, logstash_message
+      end
+
+      private
+
+      attr_reader :request, :response
+
+      def logstash_message
+        {
+          '@timestamp' => current_time,
+          '@version' => 1,
+          'content_length' => content_length || '-',
+          'http_method' => http_method,
+          'message' => message,
+          'path' => path,
+          'response_code' => response.code,
+          'severity' => level,
+          'tags' => [TAG_NAME],
+        }.to_json
+      end
+
+      def message
+        "[#{TAG_NAME}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
+      end
+
+      def current_time
+        Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
+      end
+
+      def http_method
+        @http_method ||= request.http_method.name.split("::").last.upcase
+      end
+
+      def path
+        @path ||= request.path.to_s
+      end
+
+      def content_length
+        @content_length ||= response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
+      end
+    end
+  end
+end

data/lib/httparty/module_inheritable_attributes.rb

@@ -0,0 +1,56 @@
+module HTTParty
+  module ModuleInheritableAttributes #:nodoc:
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    # borrowed from Rails 3.2 ActiveSupport
+    def self.hash_deep_dup(hash)
+      duplicate = hash.dup
+
+      duplicate.each_pair do |key, value|
+        if value.is_a?(Hash)
+          duplicate[key] = hash_deep_dup(value)
+        elsif value.is_a?(Proc)
+          duplicate[key] = value.dup
+        else
+          duplicate[key] = value
+        end
+      end
+
+      duplicate
+    end
+
+    module ClassMethods #:nodoc:
+      def mattr_inheritable(*args)
+        @mattr_inheritable_attrs ||= [:mattr_inheritable_attrs]
+        @mattr_inheritable_attrs += args
+
+        args.each do |arg|
+          module_eval %(class << self; attr_accessor :#{arg} end)
+        end
+
+        @mattr_inheritable_attrs
+      end
+
+      def inherited(subclass)
+        super
+        @mattr_inheritable_attrs.each do |inheritable_attribute|
+          ivar = "@#{inheritable_attribute}"
+          subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
+
+          if instance_variable_get(ivar).respond_to?(:merge)
+            method = <<-EOM
+              def self.#{inheritable_attribute}
+                duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
+                #{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
+              end
+            EOM
+
+            subclass.class_eval method
+          end
+        end
+      end
+    end
+  end
+end

data/lib/httparty/net_digest_auth.rb

@@ -0,0 +1,136 @@
+require 'digest/md5'
+require 'net/http'
+
+module Net
+  module HTTPHeader
+    def digest_auth(username, password, response)
+      authenticator = DigestAuthenticator.new(
+        username,
+        password,
+        @method,
+        @path,
+        response
+      )
+
+      authenticator.authorization_header.each do |v|
+        add_field('Authorization', v)
+      end
+
+      authenticator.cookie_header.each do |v|
+        add_field('Cookie', v)
+      end
+    end
+
+    class DigestAuthenticator
+      def initialize(username, password, method, path, response_header)
+        @username = username
+        @password = password
+        @method   = method
+        @path     = path
+        @response = parse(response_header)
+        @cookies  = parse_cookies(response_header)
+      end
+
+      def authorization_header
+        @cnonce = md5(random)
+        header = [
+          %(Digest username="#{@username}"),
+          %(realm="#{@response['realm']}"),
+          %(nonce="#{@response['nonce']}"),
+          %(uri="#{@path}"),
+          %(response="#{request_digest}")
+        ]
+
+        header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
+
+        if qop_present?
+          fields = [
+            %(cnonce="#{@cnonce}"),
+            %(qop="#{@response['qop']}"),
+            "nc=00000001"
+          ]
+          fields.each { |field| header << field }
+        end
+
+        header << %(opaque="#{@response['opaque']}") if opaque_present?
+        header
+      end
+
+      def cookie_header
+        @cookies
+      end
+
+      private
+
+      def parse(response_header)
+        header = response_header['www-authenticate']
+
+        header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
+
+        header =~ /Digest (.*)/
+        params = {}
+        if $1
+          non_quoted = $1.gsub(/(\w+)="(.*?)"/) { params[$1] = $2 }
+          non_quoted.gsub(/(\w+)=([^,]*)/) { params[$1] = $2 }
+        end
+        params
+      end
+
+      def parse_cookies(response_header)
+        return [] unless response_header['Set-Cookie']
+
+        cookies = response_header['Set-Cookie'].split('; ')
+
+        cookies.reduce([]) do |ret, cookie|
+          ret << cookie
+          ret
+        end
+
+        cookies
+      end
+
+      def opaque_present?
+        @response.key?('opaque') && !@response['opaque'].empty?
+      end
+
+      def qop_present?
+        @response.key?('qop') && !@response['qop'].empty?
+      end
+
+      def random
+        format "%x", (Time.now.to_i + rand(65535))
+      end
+
+      def request_digest
+        a = [md5(a1), @response['nonce'], md5(a2)]
+        a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
+        md5(a.join(":"))
+      end
+
+      def md5(str)
+        Digest::MD5.hexdigest(str)
+      end
+
+      def algorithm_present?
+        @response.key?('algorithm') && !@response['algorithm'].empty?
+      end
+
+      def use_md5_sess?
+        algorithm_present? && @response['algorithm'] == 'MD5-sess'
+      end
+
+      def a1
+        a1_user_realm_pwd =  [@username, @response['realm'], @password].join(':')
+        if use_md5_sess?
+          [ md5(a1_user_realm_pwd), @response['nonce'], @cnonce ].join(':')
+        else
+          a1_user_realm_pwd
+        end
+      end
+
+      def a2
+        [@method, @path].join(":")
+      end
+    end
+  end
+end

data/lib/httparty/parser.rb

@@ -0,0 +1,150 @@
+module HTTParty
+  # The default parser used by HTTParty, supports xml, json, html, csv and
+  # plain text.
+  #
+  # == Custom Parsers
+  #
+  # If you'd like to do your own custom parsing, subclassing HTTParty::Parser
+  # will make that process much easier. There are a few different ways you can
+  # utilize HTTParty::Parser as a superclass.
+  #
+  # @example Intercept the parsing for all formats
+  #   class SimpleParser < HTTParty::Parser
+  #     def parse
+  #       perform_parsing
+  #     end
+  #   end
+  #
+  # @example Add the atom format and parsing method to the default parser
+  #   class AtomParsingIncluded < HTTParty::Parser
+  #     SupportedFormats.merge!(
+  #       {"application/atom+xml" => :atom}
+  #     )
+  #
+  #     def atom
+  #       perform_atom_parsing
+  #     end
+  #   end
+  #
+  # @example Only support the atom format
+  #   class ParseOnlyAtom < HTTParty::Parser
+  #     SupportedFormats = {"application/atom+xml" => :atom}
+  #
+  #     def atom
+  #       perform_atom_parsing
+  #     end
+  #   end
+  #
+  # @abstract Read the Custom Parsers section for more information.
+  class Parser
+    SupportedFormats = {
+      'text/xml'                    => :xml,
+      'application/xml'             => :xml,
+      'application/json'            => :json,
+      'application/vnd.api+json'    => :json,
+      'application/hal+json'        => :json,
+      'text/json'                   => :json,
+      'application/javascript'      => :plain,
+      'text/javascript'             => :plain,
+      'text/html'                   => :html,
+      'text/plain'                  => :plain,
+      'text/csv'                    => :csv,
+      'application/csv'             => :csv,
+      'text/comma-separated-values' => :csv
+    }
+
+    # The response body of the request
+    # @return [String]
+    attr_reader :body
+
+    # The intended parsing format for the request
+    # @return [Symbol] e.g. :json
+    attr_reader :format
+
+    # Instantiate the parser and call {#parse}.
+    # @param [String] body the response body
+    # @param [Symbol] format the response format
+    # @return parsed response
+    def self.call(body, format)
+      new(body, format).parse
+    end
+
+    # @return [Hash] the SupportedFormats hash
+    def self.formats
+      const_get(:SupportedFormats)
+    end
+
+    # @param [String] mimetype response MIME type
+    # @return [Symbol]
+    # @return [nil] mime type not supported
+    def self.format_from_mimetype(mimetype)
+      formats[formats.keys.detect {|k| mimetype.include?(k)}]
+    end
+
+    # @return [Array<Symbol>] list of supported formats
+    def self.supported_formats
+      formats.values.uniq
+    end
+
+    # @param [Symbol] format e.g. :json, :xml
+    # @return [Boolean]
+    def self.supports_format?(format)
+      supported_formats.include?(format)
+    end
+
+    def initialize(body, format)
+      @body = body
+      @format = format
+    end
+
+    # @return [Object] the parsed body
+    # @return [nil] when the response body is nil, an empty string, spaces only or "null"
+    def parse
+      return nil if body.nil?
+      return nil if body == "null"
+      return nil if body.valid_encoding? && body.strip.empty?
+      if body.valid_encoding? && body.encoding == Encoding::UTF_8
+        @body = body.gsub(/\A#{UTF8_BOM}/, '')
+      end
+      if supports_format?
+        parse_supported_format
+      else
+        body
+      end
+    end
+
+    protected
+
+    def xml
+      MultiXml.parse(body)
+    end
+
+    UTF8_BOM = "\xEF\xBB\xBF".freeze
+
+    def json
+      JSON.parse(body, :quirks_mode => true, :allow_nan => true)
+    end
+
+    def csv
+      CSV.parse(body)
+    end
+
+    def html
+      body
+    end
+
+    def plain
+      body
+    end
+
+    def supports_format?
+      self.class.supports_format?(format)
+    end
+
+    def parse_supported_format
+      send(format)
+    rescue NoMethodError => e
+      raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
+    end
+  end
+end

data/lib/httparty/request.rb

@@ -0,0 +1,386 @@
+require 'erb'
+
+module HTTParty
+  class Request #:nodoc:
+    SupportedHTTPMethods = [
+      Net::HTTP::Get,
+      Net::HTTP::Post,
+      Net::HTTP::Patch,
+      Net::HTTP::Put,
+      Net::HTTP::Delete,
+      Net::HTTP::Head,
+      Net::HTTP::Options,
+      Net::HTTP::Move,
+      Net::HTTP::Copy,
+      Net::HTTP::Mkcol,
+      Net::HTTP::Lock,
+      Net::HTTP::Unlock,
+    ]
+
+    SupportedURISchemes  = ['http', 'https', 'webcal', nil]
+
+    NON_RAILS_QUERY_STRING_NORMALIZER = proc do |query|
+      Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
+        if value.nil?
+          key.to_s
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.map {|v| "#{key}=#{ERB::Util.url_encode(v.to_s)}"}
+        else
+          HashConversions.to_params(key => value)
+        end
+      end.flatten.join('&')
+    end
+
+    JSON_API_QUERY_STRING_NORMALIZER = proc do |query|
+      Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
+        if value.nil?
+          key.to_s
+        elsif value.respond_to?(:to_ary)
+          values = value.to_ary.map{|v| ERB::Util.url_encode(v.to_s)}
+          "#{key}=#{values.join(',')}"
+        else
+          HashConversions.to_params(key => value)
+        end
+      end.flatten.join('&')
+    end
+
+    attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
+    attr_reader :path
+
+    def initialize(http_method, path, o = {})
+      @changed_hosts = false
+      @credentials_sent = false
+
+      self.http_method = http_method
+      self.options = {
+        limit: o.delete(:no_follow) ? 1 : 5,
+        assume_utf16_is_big_endian: true,
+        default_params: {},
+        follow_redirects: true,
+        parser: Parser,
+        uri_adapter: URI,
+        connection_adapter: ConnectionAdapter
+      }.merge(o)
+      self.path = path
+      set_basic_auth_from_uri
+    end
+
+    def path=(uri)
+      uri_adapter = options[:uri_adapter]
+
+      @path = if uri.is_a?(uri_adapter)
+        uri
+      elsif String.try_convert(uri)
+        uri_adapter.parse uri
+      else
+        raise ArgumentError,
+          "bad argument (expected #{uri_adapter} object or URI string)"
+      end
+    end
+
+    def request_uri(uri)
+      if uri.respond_to? :request_uri
+        uri.request_uri
+      else
+        uri.path
+      end
+    end
+
+    def uri
+      if redirect && path.relative? && path.path[0] != "/"
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, "")
+
+        path.path = "/#{path.path}" if last_uri_host[-1] != "/"
+        path.path = last_uri_host + path.path
+      end
+
+      if path.relative? && path.host
+        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}")
+      elsif path.relative?
+        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}")
+      else
+        new_uri = path.clone
+      end
+
+      # avoid double query string on redirects [#12]
+      unless redirect
+        new_uri.query = query_string(new_uri)
+      end
+
+      unless SupportedURISchemes.include? new_uri.scheme
+        raise UnsupportedURIScheme, "'#{new_uri}' Must be HTTP, HTTPS or Generic"
+      end
+
+      @last_uri = new_uri
+    end
+
+    def base_uri
+      if redirect
+        base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
+        base_uri += ":#{@last_uri.port}" if @last_uri.port != 80
+        base_uri
+      else
+        options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
+      end
+    end
+
+    def format
+      options[:format] || (format_from_mimetype(last_response['content-type']) if last_response)
+    end
+
+    def parser
+      options[:parser]
+    end
+
+    def connection_adapter
+      options[:connection_adapter]
+    end
+
+    def perform(&block)
+      validate
+      setup_raw_request
+      chunked_body = nil
+      current_http = http
+
+      self.last_response = current_http.request(@raw_request) do |http_response|
+        if block
+          chunks = []
+
+          http_response.read_body do |fragment|
+            encoded_fragment = encode_text(fragment, http_response['content-type'])
+            chunks << encoded_fragment if !options[:stream_body]
+            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
+          end
+
+          chunked_body = chunks.join
+        end
+      end
+
+      handle_host_redirection if response_redirects?
+      result = handle_unauthorized
+      result ||= handle_response(chunked_body, &block)
+      result
+    end
+
+    def handle_unauthorized(&block)
+      return unless digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
+      return if @credentials_sent
+      @credentials_sent = true
+      perform(&block)
+    end
+
+    def raw_body
+      @raw_request.body
+    end
+
+    private
+
+    def http
+      connection_adapter.call(uri, options)
+    end
+
+    def credentials
+      (options[:basic_auth] || options[:digest_auth]).to_hash
+    end
+
+    def username
+      credentials[:username]
+    end
+
+    def password
+      credentials[:password]
+    end
+
+    def normalize_query(query)
+      if query_string_normalizer
+        query_string_normalizer.call(query)
+      else
+        HashConversions.to_params(query)
+      end
+    end
+
+    def query_string_normalizer
+      options[:query_string_normalizer]
+    end
+
+    def setup_raw_request
+      @raw_request = http_method.new(request_uri(uri))
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
+
+      if options[:headers].respond_to?(:to_hash)
+        headers_hash = options[:headers].to_hash
+
+        @raw_request.initialize_http_header(headers_hash)
+        # If the caller specified a header of 'Accept-Encoding', assume they want to
+        # deal with encoding of content. Disable the internal logic in Net:HTTP
+        # that handles encoding, if the platform supports it.
+        if @raw_request.respond_to?(:decode_content) && (headers_hash.key?('Accept-Encoding') || headers_hash.key?('accept-encoding'))
+          # Using the '[]=' sets decode_content to false
+          @raw_request['accept-encoding'] = @raw_request['accept-encoding']
+        end
+      end
+
+      if options[:body]
+        body = Body.new(
+          options[:body],
+          query_string_normalizer: query_string_normalizer,
+          force_multipart: options[:multipart]
+        )
+
+        if body.multipart?
+          content_type = "multipart/form-data; boundary=#{body.boundary}"
+          @raw_request['Content-Type'] = content_type
+        end
+        @raw_request.body = body.call
+      end
+
+      if options[:basic_auth] && send_authorization_header?
+        @raw_request.basic_auth(username, password)
+        @credentials_sent = true
+      end
+      setup_digest_auth if digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
+    end
+
+    def digest_auth?
+      !!options[:digest_auth]
+    end
+
+    def response_unauthorized?
+      !!last_response && last_response.code == '401'
+    end
+
+    def response_has_digest_auth_challenge?
+      !last_response['www-authenticate'].nil? && last_response['www-authenticate'].length > 0
+    end
+
+    def setup_digest_auth
+      @raw_request.digest_auth(username, password, last_response)
+    end
+
+    def query_string(uri)
+      query_string_parts = []
+      query_string_parts << uri.query unless uri.query.nil?
+
+      if options[:query].respond_to?(:to_hash)
+        query_string_parts << normalize_query(options[:default_params].merge(options[:query].to_hash))
+      else
+        query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
+        query_string_parts << options[:query] unless options[:query].nil?
+      end
+
+      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
+      query_string_parts.size > 0 ? query_string_parts.join('&') : nil
+    end
+
+    def assume_utf16_is_big_endian
+      options[:assume_utf16_is_big_endian]
+    end
+
+    def handle_response(body, &block)
+      if response_redirects?
+        options[:limit] -= 1
+        if options[:logger]
+          logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
+          logger.format(self, last_response)
+        end
+        self.path = last_response['location']
+        self.redirect = true
+        if last_response.class == Net::HTTPSeeOther
+          unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
+            self.http_method = Net::HTTP::Get
+          end
+        elsif last_response.code != '307' && last_response.code != '308'
+          unless options[:maintain_method_across_redirects]
+            self.http_method = Net::HTTP::Get
+          end
+        end
+        capture_cookies(last_response)
+        perform(&block)
+      else
+        body ||= last_response.body
+        body = body.nil? ? body : encode_text(body, last_response['content-type'])
+        Response.new(self, last_response, lambda { parse_response(body) }, body: body)
+      end
+    end
+
+    def handle_host_redirection
+      check_duplicate_location_header
+      redirect_path = options[:uri_adapter].parse last_response['location']
+      return if redirect_path.relative? || path.host == redirect_path.host
+      @changed_hosts = true
+    end
+
+    def check_duplicate_location_header
+      location = last_response.get_fields('location')
+      if location.is_a?(Array) && location.count > 1
+        raise DuplicateLocationHeader.new(last_response)
+      end
+    end
+
+    def send_authorization_header?
+      !@changed_hosts
+    end
+
+    def response_redirects?
+      case last_response
+      when Net::HTTPNotModified # 304
+        false
+      when Net::HTTPRedirection
+        options[:follow_redirects] && last_response.key?('location')
+      end
+    end
+
+    def parse_response(body)
+      parser.call(body, format)
+    end
+
+    def capture_cookies(response)
+      return unless response['Set-Cookie']
+      cookies_hash = HTTParty::CookieHash.new
+      cookies_hash.add_cookies(options[:headers].to_hash['Cookie']) if options[:headers] && options[:headers].to_hash['Cookie']
+      response.get_fields('Set-Cookie').each { |cookie| cookies_hash.add_cookies(cookie) }
+
+      options[:headers] ||= {}
+      options[:headers]['Cookie'] = cookies_hash.to_cookie_string
+    end
+
+    # Uses the HTTP Content-Type header to determine the format of the
+    # response It compares the MIME type returned to the types stored in the
+    # SupportedFormats hash
+    def format_from_mimetype(mimetype)
+      if mimetype && parser.respond_to?(:format_from_mimetype)
+        parser.format_from_mimetype(mimetype)
+      end
+    end
+
+    def validate
+      raise HTTParty::RedirectionTooDeep.new(last_response), 'HTTP redirects too deep' if options[:limit].to_i <= 0
+      raise ArgumentError, 'only get, post, patch, put, delete, head, and options methods are supported' unless SupportedHTTPMethods.include?(http_method)
+      raise ArgumentError, ':headers must be a hash' if options[:headers] && !options[:headers].respond_to?(:to_hash)
+      raise ArgumentError, 'only one authentication method, :basic_auth or :digest_auth may be used at a time' if options[:basic_auth] && options[:digest_auth]
+      raise ArgumentError, ':basic_auth must be a hash' if options[:basic_auth] && !options[:basic_auth].respond_to?(:to_hash)
+      raise ArgumentError, ':digest_auth must be a hash' if options[:digest_auth] && !options[:digest_auth].respond_to?(:to_hash)
+      raise ArgumentError, ':query must be hash if using HTTP Post' if post? && !options[:query].nil? && !options[:query].respond_to?(:to_hash)
+    end
+
+    def post?
+      Net::HTTP::Post == http_method
+    end
+
+    def set_basic_auth_from_uri
+      if path.userinfo
+        username, password = path.userinfo.split(':')
+        options[:basic_auth] = {username: username, password: password}
+        @credentials_sent = true
+      end
+    end
+
+    def encode_text(text, content_type)
+      TextEncoder.new(
+        text,
+        content_type: content_type,
+        assume_utf16_is_big_endian: assume_utf16_is_big_endian
+      ).call
+    end
+  end
+end