ruby-requests 0.0.1.a1

data/lib/requests/session.rb

@@ -0,0 +1,120 @@
+module Requests
+    class Session
+        include HttpMethods
+
+        attr_accessor :headers, :cookies, :auth, :proxies, :params, :verify,
+                      :cert, :stream, :max_redirects, :trust_env
+        attr_reader :logger
+        private :logger
+
+        DEFAULT_REDIRECT_LIMIT = 30
+
+        def initialize
+            @logger = Requests.logger
+            @headers = Utils.default_headers
+            @cookies = CookieJar.new
+            @proxies = {}
+            @params = {}
+            @verify = true
+            @stream = false
+            @trust_env = true
+            @max_redirects = DEFAULT_REDIRECT_LIMIT
+        end
+
+        def request(method, url, params: {}, data: nil, headers: {},
+                    cookies: {}, files: nil, proxies: {}, stream: false,
+                    auth: nil, timeout: nil, json: nil, allow_redirects: true,
+                    verify: true)
+            headers = @headers.merge(headers)
+            req = Request.new(method.upcase,
+                              url,
+                              headers: headers,
+                              files: files,
+                              params: params,
+                              data: data,
+                              json: json,
+                              cookies: cookies,
+                              auth: auth)
+
+            prepare_request(req)
+            settings = merge_environment_settings(
+                req.url, proxies, stream, verify, cert
+            )
+            send_kwargs = {
+                :timeout => timeout,
+                :allow_redirects => allow_redirects,
+            }.merge(settings)
+            send_request(req, **send_kwargs)
+        end
+
+        def prepare_request(request)
+            merged_cookies = CookieJar.new
+            merged_cookies.merge!(@cookies)
+            merged_cookies.merge!(request.cookies)
+
+            request.headers = merge_setting(request.headers, @headers,
+                                            Utils::InsensitiveDict)
+            request.params = merge_setting(request.params, @params)
+            request.auth = merge_setting(request.auth, @auth)
+            request.cookies = merged_cookies
+            request.prepare
+            request
+        end
+
+        def send_request(request, **kwargs)
+            kwargs[:stream] = kwargs.fetch(:stream, @stream)
+            kwargs[:verify] = kwargs.fetch(:verify, @verify)
+            kwargs[:cert] = kwargs.fetch(:cert, @cert)
+            kwargs[:proxies] = kwargs.fetch(:proxies, @proxies)
+
+            ## Use the following line once implementing redirects:
+            # allow_redirects = kwargs.delete(:allow_redirects) || true
+            kwargs.delete(:allow_redirects)
+
+            response = adapter.send_request(request, **kwargs)
+            @cookies.merge!(response.cookies)
+            response
+        end
+
+        def merge_environment_settings(_url, proxies, stream, verify, cert)
+            # TODO: Implement actually using environment settings
+            proxies = merge_setting(proxies, @proxies)
+            stream = merge_setting(stream, @stream)
+            verify = merge_setting(verify, @verify)
+            cert = merge_setting(cert, @cert)
+
+            {verify: verify,
+             proxies: proxies,
+             stream: stream,
+             cert: cert}
+        end
+
+        ##
+        # Determine appropriate setting for a given request, taking into account
+        # the explicit setting on that request, and the setting in the session.
+        # If a is a Hash or InsensitiveDict they will be merged together using
+        # `dict_class`
+
+        def merge_setting(request_setting, session_setting, dict_class=Hash)
+            return request_setting if session_setting.nil?
+            return session_setting if request_setting.nil?
+
+            map_types = [Hash, Requests::Utils::InsensitiveDict]
+            if !(map_types.include?(request_setting.class) &&
+                 map_types.include?(session_setting.class))
+                return request_setting
+            end
+
+            merged_setting = dict_class.new.merge(session_setting)
+            merged_setting.merge!(request_setting)
+
+            nil_keys = merged_setting.select { |_k, v| v.nil? }.map(&:first)
+            nil_keys.each { |key| merged_setting.delete(key) }
+            merged_setting
+        end
+
+        def adapter
+            HTTPAdapter.new
+        end
+    end
+end

data/lib/requests/utils.rb

@@ -0,0 +1,162 @@
+require 'forwardable'
+
+
+module Requests
+    module Utils
+        ##
+        # Dictionary that stores keys as case-insensitive strings.
+        # Only Strings can be used as keys in this structure; attempts
+        # to use anything else as a key will raise a TypeError.
+
+        class InsensitiveDict
+            attr_accessor :store, :dict
+
+            extend Forwardable
+            include Enumerable
+
+            def_delegators :@dict, :each
+
+            def initialize(init_dict={})
+                @store = {}
+                @dict = {}
+                init_dict.each do |k, v|
+                    self[k] = v
+                end
+            end
+
+            ##
+            # Can compare to another InsensitiveDict or to  a regular Hash.
+
+            def ==(compare)
+                compare.each do |k, v|
+                    return false if !k.is_a?(String) || self[k.downcase] != v
+                end
+                true
+            end
+
+            def []=(k, v)
+                check_key_type(k)
+                @store[k.downcase] = [k, v]
+                @dict[k.downcase] = v
+            end
+
+            def [](k)
+                check_key_type(k)
+                @dict[k.downcase]
+            end
+
+            def merge!(hash)
+                hash.each do |k, v|
+                    self[k] = v
+                end
+                self
+            end
+
+            def merge(new)
+                hash = to_h
+                new.each do |k, v|
+                    hash[k] = v
+                end
+                hash
+            end
+
+            def to_h
+                hash = {}
+                @store.each_value do |i|
+                    orig, value = i
+                    hash[orig] = value
+                end
+                hash
+            end
+
+            def inspect
+                to_h.inspect
+            end
+
+            def to_s
+                to_h.to_s
+            end
+
+            def delete(key)
+                if key.is_a?(String)
+                    key = key.downcase
+                    @store.delete(key)
+                    @dict.delete(key)
+                end
+            end
+
+            private def check_key_type(key)
+                if !key.is_a?(String)
+                    err = "Key for #{self.class.name} object " \
+                          "must be String, not #{key.class.name}"
+                    raise(TypeError, err)
+                end
+            end
+        end
+
+        def self.default_user_agent
+            "ruby-requests/#{Requests::VERSION}"
+        end
+
+        def self.default_accept_encoding
+            'gzip, deflate'
+        end
+
+        def self.default_headers
+            headers = {
+                'User-Agent' => default_user_agent,
+                'Accept-Encoding' => default_accept_encoding,
+                'Accept' => '*/*',
+                'Connection' => 'keep-alive',
+            }
+            InsensitiveDict.new(headers)
+        end
+
+        module HeadersMixin
+            ##
+            # Set @headers to InsensitiveDict keyed by String with Strings as
+            # values. When passed the result of Net::HTTPResponse.to_hash,
+            # which is a Hash keyed by String with Arrays as values, it will
+            # join the elements in the Array by ', ' as the header value.
+            # 'Set-Cookie' is handled specially because a comma is valid in a
+            # cookie definition, so it is joined by '; ' instead
+
+            def headers=(hash)
+                @headers = Utils::InsensitiveDict.new
+                hash.each do |k, v|
+                    delimiter = ', '
+                    @headers[k] = v.is_a?(Array) ? v.join(delimiter) : v
+                end
+            end
+
+            def headers
+                @headers
+            end
+        end
+
+        ##
+        # Given a string indicating an HTTP method, return the
+        # corresponding Net::HTTP class
+
+        def self.http_method_class(method)
+            {
+                'GET' => Net::HTTP::Get,
+                'POST' => Net::HTTP::Post,
+                'PUT' => Net::HTTP::Put,
+                'DELETE' => Net::HTTP::Delete,
+                'HEAD' => Net::HTTP::Head,
+                'OPTIONS' => Net::HTTP::Options,
+                'PATCH' => Net::HTTP::Patch,
+            }[method.upcase]
+        end
+
+        def self.charset_from_content_type(header)
+            charset = nil
+            if header
+                match = header.match('charset=([^\s;]+)')
+                charset = match.captures[0] if match
+            end
+            charset
+        end
+    end
+end

data/lib/requests/version.rb

@@ -0,0 +1,4 @@
+module Requests
+    VERSION = '0.0.1.a1'
+    VERSION_INFO = Gem::Version.new(VERSION)
+end

data/ruby-requests.gemspec

@@ -0,0 +1,23 @@
+require File.expand_path('lib/requests/version', __dir__)
+
+
+Gem::Specification.new do |s|
+    s.name = 'ruby-requests'
+    s.version = Requests::VERSION
+    s.authors = ['Daniel Hones']
+    s.description = "A library for HTTP requests that aims to provide " \
+                    "the same API as the Requests library in Python"
+    s.summary = 'A Ruby version of the Requests library for Python'
+
+    s.license = 'MIT'
+    s.homepage = 'https://gitlab.com/danielhones/ruby_requests'
+
+    s.files = `git ls-files`.split("\n")
+    s.test_files = `git ls-files -- {spec}/**/*`.split("\n")
+    s.required_ruby_version = ">= 2.3.0"
+
+    s.add_dependency('http-cookie', '~> 1.0')
+    s.add_development_dependency('rspec', '~> 3.7')
+    s.add_development_dependency('rubocop', '~> 0.54.0')
+    s.add_development_dependency('simplecov', '~> 0.15')
+end

data/spec/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+    proxy:
+        image: sameersbn/squid
+        volumes:
+            - ./functional/proxies/squid_conf:/etc/squid3
+            - ./functional/proxies/squid_logs:/var/log/squid3
+        ports:
+            - "3128:3128"
+
+    ruby:
+        image: ruby
+        command: /code/spec/docker_test_entrypoint.sh
+        environment:
+            - REPO_ROOT_DIR=/code
+            - RUNNING_IN_DOCKER=true
+            - PROXY_HOST=proxy
+            - PROXY_PORT=3128
+        volumes:
+            - ../:/code
+        depends_on:
+            - "proxy"

data/spec/docker_test_entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# This is the script used as the entrypoint for running proxy
+# tests from within a docker container as part of docker-compose
+
+cd $REPO_ROOT_DIR
+gem install -g Gemfile
+rspec spec/
+result=$?
+
+echo "ENTRYPOINT: Rspec returned, exiting with code $result"
+exit $result

data/spec/docker_tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+this_dir="$(dirname $0)"
+docker_compose_file="$this_dir/docker-compose.yml"
+
+docker-compose -f "$docker_compose_file" up --exit-code-from ruby

data/spec/functional/proxies/proxies_spec.rb

@@ -0,0 +1,84 @@
+require_relative '../../spec_helper'
+
+##
+# These tests will run from inside the docker container named
+# "ruby", set up by the docker-compose called in spec/docker_tests.sh
+
+RSpec.describe 'proxies', :if => running_in_docker do
+    Requests.enable_dev_mode
+
+    let(:proxy) { ENV['PROXY_HOST'] + ":" + ENV['PROXY_PORT'] }
+
+    # These are hard-coded in the squid_conf/passwords file which gets
+    # placed on the squid proxy container:
+    let(:proxy_user) { "foobar" }
+    let(:proxy_pass) { "SecretPass" }
+
+    before(:each) do
+        # The squid_logs directory is shared in a volume with both
+        # the proxy and ruby docker containers:
+        @proxy_log_file = File.join(__dir__, 'squid_logs/access.log')
+        if File.exist?(@proxy_log_file)
+            @proxy_log = File.open(@proxy_log_file)
+            @proxy_log.read
+        end
+        proxy_creds = "http://#{proxy_user}:#{proxy_pass}@#{proxy}"
+        proxy_no_creds = "http://#{proxy}"
+        @proxies = {'http' => proxy_creds, 'https' => proxy_creds}
+        @proxies_no_creds = {'http' => proxy_no_creds,
+                             'https' => proxy_no_creds}
+    end
+
+    after(:each) do
+        @proxy_log.close if @proxy_log
+    end
+
+    context "HTTP" do
+        before(:each) do
+            @url = HTTPBIN_URLS['HTTP'] + '/anything'
+        end
+
+        it 'through proxy' do
+            r = Requests.get(@url, proxies: @proxies)
+            sleep(1)  # wait for log to be written to file
+            new_log_entries = @proxy_log.readlines
+            expect(r.status_code).to eq(200)
+            msg = /GET #{@url} #{proxy_user}/
+            expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
+        end
+
+        it "through proxy, wrong credentials returns 407 response" do
+            r = Requests.get(@url, proxies: @proxies_no_creds)
+            sleep(1)  # wait for log to be written to file
+            new_log_entries = @proxy_log.readlines
+            expect(r.status_code).to eq(407)
+            msg = /TCP_DENIED\/407 .* GET #{@url}/
+            expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
+        end
+    end
+
+    context "HTTPS" do
+        before(:each) do
+            @url = HTTPBIN_URLS['HTTPS'] + '/anything'
+        end
+
+        it 'through proxy' do
+            r = Requests.get(@url, proxies: @proxies)
+            sleep(1)  # wait for log to be written to file
+            new_log_entries = @proxy_log.readlines
+            expect(r.status_code).to eq(200)
+            msg = /CONNECT #{URI(@url).host}/
+            expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
+        end
+
+        it "through proxy, wrong credentials raises ProxyAuthError" do
+            expect {
+                Requests.get(@url, proxies: @proxies_no_creds)
+            }.to raise_error(Requests::ProxyAuthError)
+            sleep(1)  # wait for log to be written to file
+            new_log_entries = @proxy_log.readlines
+            msg = /TCP_DENIED\/407 .* CONNECT #{URI(@url).host}/
+            expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
+        end
+    end
+end

data/spec/functional/proxies/squid_conf/passwords

@@ -0,0 +1,2 @@
+# foobar:SecretPass
+foobar:$apr1$09.iONid$31VssE/7OUnEyu95osiAe0