ruby-requests 0.0.1.a1

data/lib/requests/adapters.rb

@@ -0,0 +1,124 @@
+require 'net/http'
+require 'openssl'
+
+
+##
+# Initial implementation here will be very simple and lack features like
+# connection pooling.
+
+module Requests
+    class BaseAdapter
+        attr_reader :logger
+        private :logger
+
+        def initialize
+            @logger = Requests.logger
+        end
+
+        # :nocov:
+        # rubocop:disable Lint/UnusedMethodArgument
+        def send_request(_request, stream: false, timeout: nil, verify: true,
+                         cert: nil, proxies: {})
+            raise(NotImplementedError, "send_request must be implemented by " \
+                                       "subclasses of #{self.class.name}")
+        end
+        # rubocop:enable Lint/UnusedMethodArgument
+
+        def close
+            raise(NotImplementedError, "close must be implemented by " \
+                                       "subclasses of #{self.class.name}")
+        end
+        # :nocov:
+
+        def determine_proxy(uri, proxies)
+            scheme_host = "#{uri.scheme}://#{uri.host}"
+            proxy = proxies[scheme_host] || proxies[uri.scheme]
+            proxy ? URI(proxy) : nil
+        end
+    end
+
+    class HTTPAdapter < BaseAdapter
+        def send_request(request, stream: false, timeout: nil, verify: true,
+                         cert: nil, proxies: {})
+            logger.info("Request URI: #{request.uri.inspect}")
+
+            if stream
+                raise(NotImplementedError,
+                      "streaming response body is not yet implemented")
+            end
+
+            proxy = determine_proxy(request.uri, proxies)
+            http = create_http(request.uri, proxy, verify, cert)
+
+            if timeout.is_a?(Array)
+                http.open_timeout, http.read_timeout = timeout
+            else
+                http.open_timeout = timeout
+                http.read_timeout = timeout
+            end
+
+            # :nocov:
+            if ENV['REQUESTS_HTTP_DEBUG'] == 'true'
+                logger.debug("Setting Net::HTTP debug output to stdout")
+                http.set_debug_output($stdout)
+            end
+            # :nocov:
+
+            begin
+                resp = http.start do
+                    http.request(request.raw)
+                end
+            rescue Net::OpenTimeout => e
+                raise ConnectionTimeout.new('Connection timed out.',
+                                            from_error: e, request: request)
+            rescue Net::ReadTimeout => e
+                raise ReadTimeout.new('Read timed out.',
+                                      from_error: e, request: request)
+            rescue Net::HTTPServerException => e
+                if e.message == '407 "Proxy Authentication Required"'
+                    resp = Response.from_net_http_response(e.response,
+                                                           request)
+                    raise ProxyAuthError.new(e.message,
+                                             from_error: e,
+                                             request: request,
+                                             response: resp)
+                else
+                    raise e
+                end
+            end
+
+            body_size = resp.body ? resp.body.bytesize : 0
+            logger.debug("Response body: #{body_size} bytes")
+
+            # TODO: handle redirects
+
+            response = Response.from_net_http_response(resp, request)
+            logger.info("Response: #{response.inspect}")
+            response
+        end
+
+        def create_http(uri, proxy, verify, _cert)
+            if proxy
+                logger.info("Using proxy: #{proxy.scheme}://#{proxy.host}:" \
+                            "#{proxy.port}")
+                http = Net::HTTP.new(uri.host, uri.port, proxy.host, proxy.port)
+                http.proxy_user = proxy.user
+                http.proxy_pass = proxy.password
+            else
+                http = Net::HTTP.new(uri.host, uri.port, nil, nil)
+            end
+
+            http.use_ssl = (uri.scheme == 'https')
+
+            if verify == false
+                http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            elsif verify.is_a?(String)
+                http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+                http.ca_file = verify
+            else
+                http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            end
+            http
+        end
+    end
+end

data/lib/requests/cookies.rb

@@ -0,0 +1,98 @@
+require 'http-cookie'
+
+
+module Requests
+    class Cookie < HTTP::Cookie
+    end
+
+    class CookieJar < HTTP::CookieJar
+        ##
+        # Return Hash of cookies from this jar for a given uri.
+        # If provided no argument, it returns a Hash of all cookie
+        # key/value pairs in the jar, but this can lose information
+        # if the same cookie key was stored for different URIs.
+        #
+        # @param uri [URI String] optional
+
+        def to_hash(uri=nil)
+            hash = {}
+            cookies(uri).each do |c|
+                hash[c.name] = c.value
+            end
+            hash
+        end
+
+        ##
+        # Given another CookieJar, copy the cookies with this one
+
+        def merge!(jar)
+            jar.each do |cookie|
+                add(cookie.dup)
+            end
+            self
+        end
+
+        ##
+        # Given an array of Set-Cookie header values, parse each
+        # and add them to this cookie jar.
+        #
+        # @param header [Array]
+        # @param uri [URI String] URI these cookies belong to
+
+        def set_cookies_from_header(header, uri)
+            header.each do |value|
+                parse(value, uri)
+            end
+            self
+        end
+
+        def self.from_hash(hash, domain)
+            jar = new
+            hash.each do |key, value|
+                cookie = Cookie.new(key, value, domain: domain,
+                                    path: '/', for_domain: true)
+                jar.add(cookie)
+            end
+            jar
+        end
+    end
+
+    module CookiesMixin
+        ##
+        # The following methods depend on the instance variable
+        # @uri being defined
+
+        def cookies=(cookies)
+            if cookies.is_a?(Hash)
+                domain = @uri.hostname.downcase
+                @cookies = CookieJar.from_hash(cookies, domain)
+            else
+                @cookies = cookies
+            end
+        end
+
+        def cookies
+            @cookies ||= CookieJar.new
+            @cookies
+        end
+
+        def cookies_hash
+            @cookies.to_hash(@uri)
+        end
+
+        def cookie_header
+            Cookie.cookie_value(@cookies.cookies(@uri))
+        end
+
+        ##
+        # Given an array of Set-Cookie header values, add them
+        # to @cookies
+
+        def set_cookies_from_header(header, uri)
+            return if header.nil? || header.empty?
+            @cookies ||= CookieJar.new
+            @cookies.set_cookies_from_header(header, uri)
+            @cookies
+        end
+    end
+end

data/lib/requests/exceptions.rb

@@ -0,0 +1,32 @@
+require 'net/http'
+
+
+module Requests
+    class RequestException < RuntimeError
+        attr_reader :request, :response, :message, :from_error
+
+        def initialize(msg, from_error: nil, request: nil, response: nil)
+            super(msg)
+            @message = msg
+            @from_error = from_error
+            @request = request
+            @response = response
+        end
+
+        def inspect
+            "#<#{self.class.name} #{@message}>"
+        end
+    end
+
+    class TimeoutError < RequestException
+    end
+
+    class ConnectionTimeout < TimeoutError
+    end
+
+    class ReadTimeout < TimeoutError
+    end
+
+    class ProxyAuthError < RequestException
+    end
+end

data/lib/requests/http_methods.rb

@@ -0,0 +1,36 @@
+module Requests
+    module HttpMethods
+        def get(url, params: {}, **kwargs)
+            kwargs[:allow_redirects] = kwargs.fetch(:allow_redirects, true)
+            request('get', url, params: params, **kwargs)
+        end
+
+        def post(url, params: {}, data: nil, json: nil, **kwargs)
+            request('post', url, params: params, data: data,
+                    json: json, **kwargs)
+        end
+
+        def put(url, params: {}, data: nil, json: nil, **kwargs)
+            request('put', url, params: params, data: data,
+                    json: json, **kwargs)
+        end
+
+        def delete(url, **kwargs)
+            request('delete', url, **kwargs)
+        end
+
+        def head(url, **kwargs)
+            kwargs[:allow_redirects] = kwargs.fetch(:allow_redirects, false)
+            request('head', url, **kwargs)
+        end
+
+        def options(url, **kwargs)
+            kwargs[:allow_redirects] = kwargs.fetch(:allow_redirects, true)
+            request('options', url, **kwargs)
+        end
+
+        def patch(url, **kwargs)
+            request('patch', url, **kwargs)
+        end
+    end
+end

data/lib/requests/logger.rb

@@ -0,0 +1,37 @@
+require 'logger'
+
+
+module Requests
+    class LogFormatter
+        FORMAT = "%<severity>s [%<time>s] %<msg>s\n".freeze
+
+        def call(severity, time, _, msg)
+            FORMAT % {:severity => severity.ljust(5),
+                      :time => format_time(time),
+                      :msg => msg}
+        end
+
+        def format_time(time)
+            time.iso8601
+        end
+    end
+
+    class Logger < ::Logger
+        DEFAULT_LOG_DEV = STDOUT
+        DEFAULT_FORMATTER = LogFormatter.new
+
+        attr_accessor :enabled
+
+        def initialize(logdev, shift_age=0, shift_size=1048576, level: DEBUG,
+                       enabled: true)
+            super(logdev, shift_age, shift_size)
+            @enabled = enabled
+            self.formatter = formatter || DEFAULT_FORMATTER
+            self.level = level
+        end
+
+        def add(severity, message=nil, progname=nil)
+            super(severity, message, progname) if @enabled
+        end
+    end
+end

data/lib/requests/request.rb

@@ -0,0 +1,94 @@
+require 'json'
+require 'net/http'
+require 'uri'
+
+
+module Requests
+    class Request
+        include CookiesMixin
+        include Utils::HeadersMixin
+
+        attr_accessor :method, :url, :uri, :files, :data, :params, :auth,
+                      :json, :body
+        attr_reader :logger, :raw
+        private :logger
+
+        def initialize(method, url, headers: {}, files: nil, data: nil,
+                       params: {}, auth: nil, cookies: nil, json: nil)
+            @logger = Requests.logger
+            @method = method
+            @url = url
+            @uri = URI(@url)
+            @headers = headers
+
+            @files = files
+            if @files
+                raise(NotImplementedError,
+                      "files keyword param is not yet implemented")
+            end
+
+            @data = data
+            @params = params
+            @auth = auth
+            self.cookies = cookies
+            @json = json
+        end
+
+        def inspect
+            "#<Requests::Request #{@method}>"
+        end
+
+        def prepare
+            if @params && !@params.empty?
+                @uri.query = URI.encode_www_form(@params)
+            end
+            set_body_and_content_type
+            @raw = build_request
+            set_request_auth(@raw)
+            self.headers = @raw.to_hash
+        end
+
+        def set_body_and_content_type
+            if @json
+                @headers['Content-Type'] = 'application/json'
+                @body = @json.to_json
+            elsif @data.is_a?(String)
+                @headers['Content-Type'] ||= 'text/plain'
+                @body = @data
+            elsif @data.is_a?(Array)
+                @headers['Content-Type'] ||= 'application/x-www-form-urlencoded'
+                @body = URI.encode_www_form(Hash[@data])
+            elsif @data.is_a?(Hash)
+                @headers['Content-Type'] ||= 'application/x-www-form-urlencoded'
+                @body = URI.encode_www_form(@data)
+            end
+        end
+
+        def build_request
+            req = http_method_class.new(uri.request_uri)
+            req.body = body
+            @headers['cookie'] = cookie_header if !cookies.empty?
+
+            @headers.each do |key, value|
+                req[key] = value
+            end
+            req
+        end
+
+        ##
+        # @param request: Net::HTTPRequest object to set authentication on
+        #                 Currently only basic auth is supported
+
+        def set_request_auth(request)
+            if @auth && @auth.is_a?(Array)
+                user, pass = @auth
+                request.basic_auth(user, pass)
+            end
+            request
+        end
+
+        def http_method_class
+            Requests::Utils.http_method_class(@method)
+        end
+    end
+end

data/lib/requests/response.rb

@@ -0,0 +1,93 @@
+require 'json'
+require 'zlib'
+
+
+module Requests
+    class Response
+        include CookiesMixin
+        include Utils::HeadersMixin
+
+        attr_accessor :status_code, :body, :request, :reason, :cookie_jar,
+                      :uri, :raw
+
+        attr_reader :logger, :raw_headers
+        private :logger
+
+        def initialize(status_code:, reason:, headers:, body:, uri:,
+                       request:, raw: nil)
+            @logger = Requests.logger
+            @status_code = status_code
+            @reason = reason
+            @raw_headers = Requests::Utils::InsensitiveDict.new(headers)
+            self.headers = headers
+            @body = body ? decode_content(body) : ''
+            @uri = URI(uri)
+            @request = request
+            @raw = raw
+            set_cookies_from_header(@raw_headers['set-cookie'], @request.uri)
+            fix_string_encoding
+        end
+
+        def inspect
+            "#<Requests::Response #{@status_code} #{@reason}>"
+        end
+
+        def url
+            uri.to_s
+        end
+
+        def text
+            body
+        end
+
+        def json
+            JSON.parse(body)
+        end
+
+        def decode_content(body)
+            if content_encoding == 'identity'
+                body
+            elsif content_encoding == 'gzip'
+                str_io = StringIO.new(body)
+                gz = Zlib::GzipReader.new(str_io)
+                gz.read
+            elsif content_encoding == 'deflate'
+                Zlib::Inflate.inflate(body)
+            end
+        end
+
+        def content_encoding
+            @headers['content-encoding'] || 'identity'
+        end
+
+        def fix_string_encoding
+            charset = charset_from_content_type
+
+            if charset
+                begin
+                    encoding = Encoding.find(charset)
+                    logger.debug("Forcing encoding for response body " \
+                                 "using charset: #{charset.inspect}")
+                    @body.force_encoding(encoding)
+                rescue ArgumentError
+                    logger.error("No such encoding: #{charset.inspect}")
+                end
+            end
+        end
+
+        def charset_from_content_type
+            Requests::Utils.charset_from_content_type(headers['Content-Type'])
+        end
+
+        def self.from_net_http_response(resp, request)
+            body = resp.body rescue nil
+            self.new(status_code: resp.code.to_i,
+                     reason: resp.message,
+                     headers: resp.to_hash,
+                     body: body,
+                     uri: request.uri,
+                     request: request,
+                     raw: resp)
+        end
+    end
+end