rest 2.7.2 → 3.0.0

data/lib/rest/wrappers/internal_client/internal/mimes.rb

@@ -0,0 +1,54 @@
+module Rest
+  module InternalClient
+
+    module Mimes
+
+      def self.mime_for(path)
+        ret = Mimes.mime_for_ext(path)
+        if ret
+          return ret
+        end
+        return 'text/plain'
+      end
+
+      # takes something like json and returns application/json.
+      # If no conversion, returns ext passed in
+      def self.mime_for_ext(path)
+        ret = nil
+        dot = path.rindex('.')
+        if dot
+          ext = path[dot+1..path.length]
+        else
+          ext = path
+        end
+        # feel free to add more mime-types anyone.
+        case ext
+          when 'xml'
+            ret = 'application/xml'
+          when 'gif'
+            ret = 'image/gif'
+          when 'jpg', 'jpeg'
+            ret = 'image/jpeg'
+          when 'json'
+            ret = 'application/json'
+          when 'html'
+            ret = 'text/html'
+          when 'mp3'
+            ret = 'audio/mpeg'
+          when 'pdf'
+            ret = 'application/pdf'
+        end
+        return ret
+      end
+
+      def self.mime_for_or_not(ext)
+        ret = Mimes.mime_for_ext(ext)
+        if ret
+          return ret
+        end
+        return ext
+      end
+    end
+
+  end
+end

data/lib/rest/wrappers/internal_client/internal/net_http_ext.rb

@@ -0,0 +1,55 @@
+# module Net
+#   class HTTP
+#
+#     # Adding the patch method if it doesn't exist (rest-client issue: https://github.com/archiloque/rest-client/issues/79)
+#     if !defined?(Net::HTTP::Patch)
+#       # Code taken from this commit: https://github.com/ruby/ruby/commit/ab70e53ac3b5102d4ecbe8f38d4f76afad29d37d#lib/net/http.rb
+#       class Protocol
+#         # Sends a PATCH request to the +path+ and gets a response,
+#         # as an HTTPResponse object.
+#         def patch(path, data, initheader = nil, dest = nil, &block) # :yield: +body_segment+
+#           send_entity(path, data, initheader, dest, Patch, &block)
+#         end
+#
+#         # Executes a request which uses a representation
+#         # and returns its body.
+#         def send_entity(path, data, initheader, dest, type, &block)
+#           res = nil
+#           request(type.new(path, initheader), data) { |r|
+#             r.read_body dest, &block
+#             res = r
+#           }
+#           unless @newimpl
+#             res.value
+#             return res, res.body
+#           end
+#           res
+#         end
+#       end
+#
+#       class Patch < HTTPRequest
+#         METHOD = 'PATCH'
+#         REQUEST_HAS_BODY = true
+#         RESPONSE_HAS_BODY = true
+#       end
+#     end
+#
+#     #
+#     # Replace the request method in Net::HTTP to sniff the body type
+#     # and set the stream if appropriate
+#     #
+#     # Taken from:
+#     # http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
+#
+#     alias __request__ request
+#
+#     def request(req, body=nil, &block)
+#       if body != nil && body.respond_to?(:read)
+#         req.body_stream = body
+#         return __request__(req, nil, &block)
+#       else
+#         return __request__(req, body, &block)
+#       end
+#     end
+#   end
+# end

data/lib/rest/wrappers/internal_client/internal/payload.rb

@@ -0,0 +1,237 @@
+require 'tempfile'
+require 'stringio'
+
+module Rest
+  module InternalClient
+    module Payload
+      extend self
+
+      def generate(params)
+        if params.is_a?(String)
+          Base.new(params)
+        elsif params.is_a?(Hash)
+          if params.delete(:multipart) == true || has_file?(params)
+            Multipart.new(params)
+          else
+            UrlEncoded.new(params)
+          end
+        elsif params.respond_to?(:read)
+          Streamed.new(params)
+        else
+          nil
+        end
+      end
+
+      def has_file?(params)
+        params.any? do |_, v|
+          case v
+            when Hash
+              has_file?(v)
+            when Array
+              has_file_array?(v)
+            else
+              v.respond_to?(:path) && v.respond_to?(:read)
+          end
+        end
+      end
+
+      def has_file_array?(params)
+        params.any? do |v|
+          case v
+            when Hash
+              has_file?(v)
+            when Array
+              has_file_array?(v)
+            else
+              v.respond_to?(:path) && v.respond_to?(:read)
+          end
+        end
+      end
+
+      class Base
+        def initialize(params)
+          build_stream(params)
+        end
+
+        def build_stream(params)
+          @stream = StringIO.new(params)
+          @stream.seek(0)
+        end
+
+        def read(bytes=nil)
+          @stream.read(bytes)
+        end
+
+        alias :to_s :read
+
+        # Flatten parameters by converting hashes of hashes to flat hashes
+        # {keys1 => {keys2 => value}} will be transformed into [keys1[key2], value]
+        def flatten_params(params, parent_key = nil)
+          result = []
+          params.each do |key, value|
+            calculated_key = parent_key ? "#{parent_key}[#{handle_key(key)}]" : handle_key(key)
+            if value.is_a? Hash
+              result += flatten_params(value, calculated_key)
+            elsif value.is_a? Array
+              result += flatten_params_array(value, calculated_key)
+            else
+              result << [calculated_key, value]
+            end
+          end
+          result
+        end
+
+        def flatten_params_array value, calculated_key
+          result = []
+          value.each do |elem|
+            if elem.is_a? Hash
+              result += flatten_params(elem, calculated_key)
+            elsif elem.is_a? Array
+              result += flatten_params_array(elem, calculated_key)
+            else
+              result << ["#{calculated_key}[]", elem]
+            end
+          end
+          result
+        end
+
+        def headers
+          {'Content-Length' => size.to_s}
+        end
+
+        def size
+          @stream.size
+        end
+
+        alias :length :size
+
+        def close
+          @stream.close unless @stream.closed?
+        end
+
+        def inspect
+          result = to_s.inspect
+          @stream.seek(0)
+          result
+        end
+
+        def short_inspect
+          (size > 500 ? "#{size} byte(s) length" : inspect)
+        end
+
+      end
+
+      class Streamed < Base
+        def build_stream(params = nil)
+          @stream = params
+        end
+
+        def size
+          if @stream.respond_to?(:size)
+            @stream.size
+          elsif @stream.is_a?(IO)
+            @stream.stat.size
+          end
+        end
+
+        alias :length :size
+      end
+
+      class UrlEncoded < Base
+        def build_stream(params = nil)
+          @stream = StringIO.new(flatten_params(params).collect do |entry|
+                                   "#{entry[0]}=#{handle_key(entry[1])}"
+                                 end.join("&"))
+          @stream.seek(0)
+        end
+
+        # for UrlEncoded escape the keys
+        def handle_key key
+          parser.escape(key.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+        end
+
+        def headers
+          super.merge({'Content-Type' => 'application/x-www-form-urlencoded'})
+        end
+
+        private
+        def parser
+          URI.const_defined?(:Parser) ? URI::Parser.new : URI
+        end
+      end
+
+      class Multipart < Base
+        EOL = "\r\n"
+
+        def build_stream(params)
+          b = "--#{boundary}"
+
+          @stream = Tempfile.new("InternalClient.Stream.#{rand(1000)}")
+          @stream.binmode
+          @stream.write(b + EOL)
+
+          if params.is_a? Hash
+            x = flatten_params(params)
+          else
+            x = params
+          end
+
+          last_index = x.length - 1
+          x.each_with_index do |a, index|
+            k, v = * a
+            if v.respond_to?(:read) && v.respond_to?(:path)
+              create_file_field(@stream, k, v)
+            else
+              create_regular_field(@stream, k, v)
+            end
+            @stream.write(EOL + b)
+            @stream.write(EOL) unless last_index == index
+          end
+          @stream.write('--')
+          @stream.write(EOL)
+          @stream.seek(0)
+        end
+
+        def create_regular_field(s, k, v)
+          s.write("Content-Disposition: form-data; name=\"#{k}\"")
+          s.write(EOL)
+          s.write(EOL)
+          s.write(v)
+        end
+
+        def create_file_field(s, k, v)
+          begin
+            s.write("Content-Disposition: form-data;")
+            s.write(" name=\"#{k}\";") unless (k.nil? || k=='')
+            s.write(" filename=\"#{v.respond_to?(:original_filename) ? v.original_filename : File.basename(v.path)}\"#{EOL}")
+            s.write("Content-Type: #{v.respond_to?(:content_type) ? v.content_type : Mimes.mime_for(v.path)}#{EOL}")
+            s.write(EOL)
+            while data = v.read(8124)
+              s.write(data)
+            end
+          ensure
+            v.close if v.respond_to?(:close)
+          end
+        end
+
+
+        def boundary
+          @boundary ||= rand(1_000_000).to_s
+        end
+
+        # for Multipart do not escape the keys
+        def handle_key key
+          key
+        end
+
+        def headers
+          super.merge({'Content-Type' => %Q{multipart/form-data; boundary=#{boundary}}})
+        end
+
+        def close
+          @stream.close!
+        end
+      end
+    end
+  end
+end

data/lib/rest/wrappers/internal_client/internal/raw_response.rb

@@ -0,0 +1,36 @@
+module Rest
+  module InternalClient
+    # The response from InternalClient on a raw request looks like a string, but is
+    # actually one of these.  99% of the time you're making a rest call all you
+    # care about is the body, but on the occassion you want to fetch the
+    # headers you can:
+    #
+    #   InternalClient.get('http://example.com').headers[:content_type]
+    #
+    # In addition, if you do not use the response as a string, you can access
+    # a Tempfile object at res.file, which contains the path to the raw
+    # downloaded request body.
+    class RawResponse
+
+      include AbstractResponse
+
+      attr_reader :file
+
+      def initialize tempfile, net_http_res, args
+        @net_http_res = net_http_res
+        @args = args
+        @file = tempfile
+      end
+
+      def to_s
+        @file.open
+        @file.read
+      end
+
+      def size
+        File.size file
+      end
+
+    end
+  end
+end

data/lib/rest/wrappers/internal_client/internal/request.rb

@@ -0,0 +1,322 @@
+require 'tempfile'
+require 'cgi'
+require 'netrc'
+
+module Rest
+  module InternalClient
+    # This class is used internally by InternalClient to send the request, but you can also
+    # call it directly if you'd like to use a method not supported by the
+    # main API.  For example:
+    #
+    #   InternalClient::Request.execute(:method => :head, :url => 'http://example.com')
+    #
+    # Mandatory parameters:
+    # * :method
+    # * :url
+    # Optional parameters (have a look at ssl and/or uri for some explanations):
+    # * :headers a hash containing the request headers
+    # * :cookies will replace possible cookies in the :headers
+    # * :user and :password for basic auth, will be replaced by a user/password available in the :url
+    # * :block_response call the provided block with the HTTPResponse as parameter
+    # * :raw_response return a low-level RawResponse instead of a Response
+    # * :max_redirects maximum number of redirections (default to 10)
+    # * :verify_ssl enable ssl verification, possible values are constants from OpenSSL::SSL
+    # * :timeout and :open_timeout passing in -1 will disable the timeout by setting the corresponding net timeout values to nil
+    # * :ssl_client_cert, :ssl_client_key, :ssl_ca_file, :ssl_ca_path
+    # * :ssl_version specifies the SSL version for the underlying Net::HTTP connection (defaults to 'SSLv23')
+    class Request
+
+      attr_reader :method, :url, :headers, :cookies,
+                  :payload, :user, :password, :timeout, :max_redirects,
+                  :open_timeout, :raw_response, :verify_ssl, :ssl_client_cert,
+                  :ssl_client_key, :ssl_ca_file, :processed_headers, :args,
+                  :ssl_version, :ssl_ca_path
+
+      def self.execute(args, & block)
+        new(args).execute(& block)
+      end
+
+      def initialize args
+        @method = args[:method] or raise ArgumentError, "must pass :method"
+        @headers = args[:headers] || {}
+        if args[:url]
+          @url = process_url_params(args[:url], headers)
+        else
+          raise ArgumentError, "must pass :url"
+        end
+        @cookies = @headers.delete(:cookies) || args[:cookies] || {}
+        @payload = Payload.generate(args[:payload])
+        @user = args[:user]
+        @password = args[:password]
+        @timeout = args[:timeout]
+        @open_timeout = args[:open_timeout]
+        @block_response = args[:block_response]
+        @raw_response = args[:raw_response] || false
+        @verify_ssl = args[:verify_ssl] || false
+        @ssl_client_cert = args[:ssl_client_cert] || nil
+        @ssl_client_key = args[:ssl_client_key] || nil
+        @ssl_ca_file = args[:ssl_ca_file] || nil
+        @ssl_ca_path = args[:ssl_ca_path] || nil
+        @ssl_version = args[:ssl_version] || 'SSLv23'
+        @tf = nil # If you are a raw request, this is your tempfile
+        @max_redirects = args[:max_redirects] || 10
+        @processed_headers = make_headers headers
+        @args = args
+      end
+
+      def execute & block
+        uri = parse_url_with_auth(url)
+        transmit uri, net_http_request_class(method).new(uri.request_uri, processed_headers), payload, & block
+      ensure
+        payload.close if payload
+      end
+
+      # Extract the query parameters and append them to the url
+      def process_url_params url, headers
+        url_params = {}
+        headers.delete_if do |key, value|
+          if 'params' == key.to_s.downcase && value.is_a?(Hash)
+            url_params.merge! value
+            true
+          else
+            false
+          end
+        end
+        unless url_params.empty?
+          query_string = url_params.collect { |k, v| "#{k.to_s}=#{CGI::escape(v.to_s)}" }.join('&')
+          url + "?#{query_string}"
+        else
+          url
+        end
+      end
+
+      def make_headers user_headers
+        unless @cookies.empty?
+          user_headers[:cookie] = @cookies.map { |(key, val)| "#{key.to_s}=#{CGI::unescape(val.to_s)}" }.sort.join('; ')
+        end
+        headers = stringify_headers(default_headers).merge(stringify_headers(user_headers))
+        headers.merge!(@payload.headers) if @payload
+        headers
+      end
+
+      def net_http_class
+        if InternalClient.proxy
+          proxy_uri = URI.parse(InternalClient.proxy)
+          Net::HTTP::Proxy(proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password)
+        else
+          Net::HTTP
+        end
+      end
+
+      def net_http_request_class(method)
+        Net::HTTP.const_get(method.to_s.capitalize)
+      end
+
+      def parse_url(url)
+        url = "http://#{url}" unless url.match(/^http/)
+        URI.parse(url)
+      end
+
+      def parse_url_with_auth(url)
+        uri = parse_url(url)
+        @user = CGI.unescape(uri.user) if uri.user
+        @password = CGI.unescape(uri.password) if uri.password
+        if !@user && !@password
+          @user, @password = Netrc.read[uri.host]
+        end
+        uri
+      end
+
+      def process_payload(p=nil, parent_key=nil)
+        unless p.is_a?(Hash)
+          p
+        else
+          @headers[:content_type] ||= 'application/x-www-form-urlencoded'
+          p.keys.map do |k|
+            key = parent_key ? "#{parent_key}[#{k}]" : k
+            if p[k].is_a? Hash
+              process_payload(p[k], key)
+            else
+              value = parser.escape(p[k].to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+              "#{key}=#{value}"
+            end
+          end.join("&")
+        end
+      end
+
+      def transmit uri, req, payload, & block
+        setup_credentials req
+
+        net = net_http_class.new(uri.host, uri.port)
+        net.use_ssl = uri.is_a?(URI::HTTPS)
+        net.ssl_version = @ssl_version
+        err_msg = nil
+        if (@verify_ssl == false) || (@verify_ssl == OpenSSL::SSL::VERIFY_NONE)
+          net.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        elsif @verify_ssl.is_a? Integer
+          net.verify_mode = @verify_ssl
+          net.verify_callback = lambda do |preverify_ok, ssl_context|
+            if (!preverify_ok) || ssl_context.error != 0
+              err_msg = "SSL Verification failed -- Preverify: #{preverify_ok}, Error: #{ssl_context.error_string} (#{ssl_context.error})"
+              return false
+            end
+            true
+          end
+        end
+        net.cert = @ssl_client_cert if @ssl_client_cert
+        net.key = @ssl_client_key if @ssl_client_key
+        net.ca_file = @ssl_ca_file if @ssl_ca_file
+        net.ca_path = @ssl_ca_path if @ssl_ca_path
+        net.read_timeout = @timeout if @timeout
+        net.open_timeout = @open_timeout if @open_timeout
+
+        # disable the timeout if the timeout value is -1
+        net.read_timeout = nil if @timeout == -1
+        net.open_timeout = nil if @open_timeout == -1
+
+        InternalClient.before_execution_procs.each do |before_proc|
+          before_proc.call(req, args)
+        end
+
+        log_request
+
+        net.start do |http|
+          if @block_response
+            http.request(req, payload ? payload.to_s : nil, & @block_response)
+          else
+            res = http.request(req, payload ? payload.to_s : nil) { |http_response| fetch_body(http_response) }
+            log_response res
+            process_result res, & block
+          end
+        end
+      rescue OpenSSL::SSL::SSLError => e
+        if err_msg
+          raise SSLCertificateNotVerified.new(err_msg)
+        else
+          raise e
+        end
+      rescue EOFError
+        raise InternalClient::ServerBrokeConnection
+      rescue Timeout::Error
+        raise InternalClient::RequestTimeout
+      end
+
+      def setup_credentials(req)
+        req.basic_auth(user, password) if user
+      end
+
+      def fetch_body(http_response)
+        if @raw_response
+          # Taken from Chef, which as in turn...
+          # Stolen from http://www.ruby-forum.com/topic/166423
+          # Kudos to _why!
+          @tf = Tempfile.new("rest-client")
+          size, total = 0, http_response.header['Content-Length'].to_i
+          http_response.read_body do |chunk|
+            @tf.write chunk
+            size += chunk.size
+            if InternalClient.log
+              if size == 0
+                InternalClient.log << "#{@method} #{@url} done (0 length file\n)"
+              elsif total == 0
+                InternalClient.log << "#{@method} #{@url} (zero content length)\n"
+              else
+                InternalClient.log << "#{@method} #{@url} %d%% done (%d of %d)\n" % [(size * 100) / total, size, total]
+              end
+            end
+          end
+          @tf.close
+          @tf
+        else
+          http_response.read_body
+        end
+        http_response
+      end
+
+      def process_result res, & block
+        if @raw_response
+          # We don't decode raw requests
+          response = RawResponse.new(@tf, res, args)
+        else
+          response = Response.create(Request.decode(res['content-encoding'], res.body), res, args)
+        end
+
+        if block_given?
+          block.call(response, self, res, & block)
+        else
+          response.return!(self, res, & block)
+        end
+
+      end
+
+      def self.decode content_encoding, body
+        if (!body) || body.empty?
+          body
+        elsif content_encoding == 'gzip'
+          Zlib::GzipReader.new(StringIO.new(body)).read
+        elsif content_encoding == 'deflate'
+          begin
+            Zlib::Inflate.new.inflate body
+          rescue Zlib::DataError
+            # No luck with Zlib decompression. Let's try with raw deflate,
+            # like some broken web servers do.
+            Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate body
+          end
+        else
+          body
+        end
+      end
+
+      def log_request
+        if InternalClient.log
+          out = []
+          out << "InternalClient.#{method} #{url.inspect}"
+          out << payload.short_inspect if payload
+          out << processed_headers.to_a.sort.map { |(k, v)| [k.inspect, v.inspect].join("=>") }.join(", ")
+          InternalClient.log << out.join(', ') + "\n"
+        end
+      end
+
+      def log_response res
+        if InternalClient.log
+          size = @raw_response ? File.size(@tf.path) : (res.body.nil? ? 0 : res.body.size)
+          InternalClient.log << "# => #{res.code} #{res.class.to_s.gsub(/^Net::HTTP/, '')} | #{(res['Content-type'] || '').gsub(/;.*$/, '')} #{size} bytes\n"
+        end
+      end
+
+      # Return a hash of headers whose keys are capitalized strings
+      def stringify_headers headers
+        headers.inject({}) do |result, (key, value)|
+          if key.is_a? Symbol
+            key = key.to_s.split(/_/).map { |w| w.capitalize }.join('-')
+          end
+          if 'CONTENT-TYPE' == key.upcase
+            target_value = value.to_s
+            result[key] = Mimes.mime_for_or_not target_value
+          elsif 'ACCEPT' == key.upcase
+            # Accept can be composed of several comma-separated values
+            if value.is_a? Array
+              target_values = value
+            else
+              target_values = value.to_s.split ','
+            end
+            result[key] = target_values.map { |ext| Mimes.mime_for_or_not(ext.to_s.strip) }.join(', ')
+          else
+            result[key] = value.to_s
+          end
+          result
+        end
+      end
+
+      def default_headers
+        {:accept => '*/*; q=0.5, application/xml', :accept_encoding => 'gzip, deflate'}
+      end
+
+      private
+      def parser
+        URI.const_defined?(:Parser) ? URI::Parser.new : URI
+      end
+
+    end
+  end
+end