gemirro 2.0.0 → 3.0.0

data/lib/gemirro/http.rb

@@ -2,61 +2,135 @@
 
 module Gemirro
   ##
-  # The Http class is responsible for executing GET request
-  # to a specific url and return an response as an HTTP::Message
-  #
-  # @!attribute [r] client
-  #  @return [HTTPClient]
+  # The Http class is responsible for executing GET requests against a
+  # given URL, following redirects and reusing keep-alive connections.
   #
   class Http
-    attr_accessor :client
-
     ##
-    # Requests the given HTTP resource.
-    #
-    # @param [String] url
-    # @return [HTTP::Message]
+    # Raised when a request comes back with a non-successful,
+    # non-redirect HTTP status.
     #
-    def self.get(url)
-      response = client.get(url, follow_redirect: true)
+    class Error < StandardError; end
 
-      raise HTTPClient::BadResponseError, response.reason unless HTTP::Status.successful?(response.status)
+    MAX_REDIRECTS = 10
 
-      response
-    end
+    class << self
+      ##
+      # Requests the given HTTP resource, following redirects.
+      #
+      # @param [String] url
+      # @return [Net::HTTPResponse]
+      #
+      def get(url)
+        fetch(URI.parse(url), MAX_REDIRECTS)
+      end
 
-    ##
-    # @return [HTTPClient]
-    #
-    def self.client
-      client ||= HTTPClient.new
-      config = Utils.configuration
-      if defined?(config.upstream_user)
-        user = config.upstream_user
-        password = config.upstream_password
-        domain = config.upstream_domain
-        client.set_auth(domain, user, password)
+      ##
+      # Closes and forgets every connection cached for the current
+      # Thread. Mostly useful for tests.
+      #
+      def reset!
+        pool.each_value { |connection| connection.finish if connection.started? }
+        pool.clear
+      end
+
+      private
+
+      ##
+      # @param [URI] uri
+      # @param [Integer] redirects_left
+      # @return [Net::HTTPResponse]
+      #
+      def fetch(uri, redirects_left)
+        response = connection_for(uri).request(build_request(uri))
+
+        case response
+        when Net::HTTPRedirection
+          raise Error, "Too many redirects for #{uri}" if redirects_left <= 0
+
+          fetch(URI.parse(response['location']), redirects_left - 1)
+        when Net::HTTPSuccess
+          response
+        else
+          raise Error, "#{response.code} #{response.message}"
+        end
+      end
+
+      ##
+      # @param [URI] uri
+      # @return [Net::HTTP::Get]
+      #
+      def build_request(uri)
+        request = Net::HTTP::Get.new(uri)
+        config = Utils.configuration
+        request.basic_auth(config.upstream_user, config.upstream_password) if defined?(config.upstream_user)
+
+        request
       end
 
-      if defined?(config.proxy)
-        proxy = config.proxy
-        client.proxy = (proxy)
+      ##
+      # Returns a started, keep-alive connection for the given URI,
+      # reusing one already opened by the current Thread when possible.
+      #
+      # @param [URI] uri
+      # @return [Net::HTTP]
+      #
+      def connection_for(uri)
+        key = "#{uri.scheme}://#{uri.host}:#{uri.port}"
+        pool[key] ||= build_connection(uri)
       end
 
-      # Use my own ca file for self signed cert
-      if defined?(config.rootca)
-        abort "The configuration file #{config.rootca} does not exist" unless File.file?(config.rootca)
-        client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        client.ssl_config.set_trust_ca(config.rootca)
-      elsif defined?(config.verify_mode)
-        client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE unless config.verify_mode
+      ##
+      # Connection pool for the current Thread. Kept per-Thread so
+      # sockets are never shared/interleaved across concurrent requests.
+      #
+      # @return [Hash]
+      #
+      def pool
+        Thread.current[:gemirro_http_pool] ||= {}
       end
 
-      # Enforce base auth
-      if defined?(config.basic_auth) && config.basic_auth
-        client.www_auth.basic_auth.force_auth = (true)
+      ##
+      # @param [URI] uri
+      # @return [Net::HTTP]
+      #
+      def build_connection(uri)
+        config = Utils.configuration
+        http = Net::HTTP.new(uri.host, uri.port, *proxy_args(config))
+        http.use_ssl = uri.scheme == 'https'
+        configure_ssl(http, config)
+
+        http.start
+      end
+
+      ##
+      # @param [Net::HTTP] http
+      # @param [Gemirro::Configuration] config
+      #
+      def configure_ssl(http, config)
+        if defined?(config.rootca)
+          abort "The configuration file #{config.rootca} does not exist" unless File.file?(config.rootca)
+
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          http.ca_file = config.rootca
+        elsif defined?(config.verify_mode) && !config.verify_mode
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+      end
+
+      ##
+      # Splits a `proxy 'http://user:pass@host:port'` configuration
+      # value into the args expected by `Net::HTTP.new`.
+      #
+      # @param [Gemirro::Configuration] config
+      # @return [Array]
+      #
+      def proxy_args(config)
+        return [] unless defined?(config.proxy) && config.proxy
+
+        proxy_uri = URI.parse(config.proxy)
+        [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
       end
-      @client = client
     end
   end
 end

data/lib/gemirro/indexer.rb

@@ -119,7 +119,7 @@ module Gemirro
       source_host = Gemirro.configuration.source.host
       Utils.logger.info("Download from source #{source_host}/#{file}")
       resp = Http.get("#{source_host}/#{File.basename(file)}")
-      return unless resp.code == 200
+      return unless resp.code.to_i == 200
 
       resp.body
     end

data/lib/gemirro/mirror_file.rb

@@ -27,10 +27,8 @@ module Gemirro
     #
     def write(content)
       FileUtils.mkdir_p(File.dirname(@path))
-      handle = File.open(@path, 'w')
-
-      handle.write(content)
-      handle.close
+      File.write(@path, content)
+      nil
     end
 
     ##
@@ -39,12 +37,7 @@ module Gemirro
     # @return [String]
     #
     def read
-      handle  = File.open(@path, 'r')
-      content = handle.read
-
-      handle.close
-
-      content
+      File.read(@path)
     end
   end
 end

data/lib/gemirro/server.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'sinatra/base'
-require 'thin'
+require 'puma'
 require 'uri'
 require 'addressable/uri'
 require 'base64'
@@ -33,6 +33,7 @@ module Gemirro
       config.server.port = '2000' if config.server.port.nil?
 
       set :static, true
+      set :server, 'puma'
 
       set :views, Gemirro::Configuration.views_directory
       set :port, config.server.port

data/lib/gemirro/version.rb

@@ -2,5 +2,5 @@
 
 # Gemirro Version
 module Gemirro
-  VERSION = '2.0.0'
+  VERSION = '3.0.0'
 end

data/lib/gemirro.rb

@@ -4,14 +4,15 @@ require 'builder'
 require 'confstruct'
 require 'digest/sha2'
 require 'fileutils'
-require 'httpclient'
 require 'json'
 require 'logger'
+require 'net/http'
+require 'openssl'
+require 'optparse'
 require 'parallel'
 require 'rubygems'
 require 'rubygems/indexer'
 require 'rubygems/user_interaction'
-require 'slop'
 require 'stringio'
 require 'tempfile'
 require 'zlib'
@@ -34,8 +35,3 @@ require 'gemirro/versions_fetcher'
 require 'gemirro/gems_fetcher'
 
 require 'gemirro/cli'
-require 'gemirro/cli/index'
-require 'gemirro/cli/init'
-require 'gemirro/cli/list'
-require 'gemirro/cli/server'
-require 'gemirro/cli/update'

data/spec/gemirro/cli_spec.rb

@@ -1,7 +1,6 @@
 require 'spec_helper'
 require 'gemirro/cli'
 require 'gemirro/mirror_file'
-require 'slop'
 
 # Gemirro tests
 module Gemirro
@@ -10,21 +9,47 @@ module Gemirro
     describe 'CLI' do
       include FakeFS::SpecHelpers
 
-      it 'should return options' do
-        options = CLI.options
-        expect(options).to be_a(::Slop)
-        expect(options.config[:strict]).to be_truthy
-        expect(options.config[:banner])
-          .to eq('Usage: gemirro [COMMAND] [OPTIONS]')
-        expect(options.to_s)
-          .to match(/-v, --version(\s+)Shows the current version/)
-        expect(options.to_s)
-          .to match(/-h, --help(\s+)Display this help message./)
+      it 'should print version information for -v' do
+        expect { CLI.run(['-v']) }.to output(/gemirro v.* on ruby/).to_stdout
+      end
+
+      it 'should print version information for --version' do
+        expect { CLI.run(['--version']) }.to output(/gemirro v.* on ruby/).to_stdout
+      end
+
+      it 'should print usage for -h' do
+        expect { CLI.run(['-h']) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
+
+      it 'should print usage for --help' do
+        expect { CLI.run(['--help']) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
+
+      it 'should print usage when no argument is given' do
+        expect { CLI.run([]) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
 
-        version = options.fetch_option(:v)
-        expect(version.short).to eq('v')
-        expect(version.long).to eq('version')
-        expect { version.call }.to output(/gemirro v.* on ruby/).to_stdout
+      it 'should list the available commands in the usage banner' do
+        output = capture_stdout { CLI.run(['-h']) }
+
+        CLI::COMMANDS.each do |name, description|
+          expect(output).to match(/#{name}(\s+)#{Regexp.escape(description)}/)
+        end
+      end
+
+      it 'should set $PROGRAM_NAME' do
+        $PROGRAM_NAME = 'rspec'
+        CLI.run(['-v'])
+        expect($PROGRAM_NAME).to eq('gemirro')
+      end
+
+      it 'should print an error and exit for an unknown command' do
+        expect { CLI.run(['nope']) }.to output(/Unknown command: nope/).to_stdout.and raise_error(SystemExit)
+      end
+
+      it 'should print an option error for an unknown flag' do
+        expect { CLI.run(['list', '--nope']) }
+          .to output(/invalid option: --nope/).to_stdout.and raise_error(SystemExit)
       end
 
       it 'should retrieve version information' do
@@ -45,6 +70,15 @@ module Gemirro
         file.write('test')
         expect { CLI.load_configuration('config.rb') }.to raise_error LoadError
       end
+
+      def capture_stdout
+        original = $stdout
+        $stdout = StringIO.new
+        yield
+        $stdout.string
+      ensure
+        $stdout = original
+      end
     end
   end
 end

data/spec/gemirro/gems_fetcher_spec.rb

@@ -47,7 +47,7 @@ module Gemirro
       version = ::Gem::Version.new('0.0.1')
       Utils.configuration.ignore_gem('gemirro', '0.0.1', 'ruby')
       allow(@source).to receive(:fetch_gem)
-        .once.with('gemirro', version).and_raise(ArgumentError)
+        .once.with('gemirro-0.0.1.gem').and_raise(ArgumentError)
       allow(Utils.logger).to receive(:error)
         .once.with(/Failed to retrieve/)
       allow(Utils.logger).to receive(:debug)

data/spec/gemirro/http_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
-require 'httpclient'
+require 'webmock/rspec'
+require 'gemirro/utils'
 require 'gemirro/http'
 
 module Gemirro
@@ -7,88 +8,91 @@ module Gemirro
     let(:config) { double('Configuration') }
 
     before do
-      Http.instance_variable_set(:@client, nil)
+      Http.reset!
       allow(Utils).to receive(:configuration).and_return(config)
     end
 
-    describe '.client' do
-      it 'initializes a new HTTPClient' do
-        expect(Http.client).to be_a(HTTPClient)
-      end
+    after { Http.reset! }
 
-      context 'with proxy configuration' do
-        it 'sets proxy configuration' do
-          allow(config).to receive(:proxy).and_return('http://proxy.example.com:8080')
+    describe '.get' do
+      context 'with successful response' do
+        it 'returns the response' do
+          stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'content')
+
+          response = Http.get('http://example.com/')
+          expect(response).to be_a(Net::HTTPSuccess)
+          expect(response.body).to eq('content')
+        end
+
+        it 'reuses the connection for a second request on the same host' do
+          stub_request(:get, 'http://example.com/one').to_return(status: 200, body: 'one')
+          stub_request(:get, 'http://example.com/two').to_return(status: 200, body: 'two')
+
+          Http.get('http://example.com/one')
+          Http.get('http://example.com/two')
 
-          expect(Http.client.proxy.to_s).to eq('http://proxy.example.com:8080')
+          expect(a_request(:get, 'http://example.com/one')).to have_been_made.once
+          expect(a_request(:get, 'http://example.com/two')).to have_been_made.once
         end
       end
 
-      context 'with SSL configuration' do
-        context 'with invalid root CA path' do
-          before do
-            allow(config).to receive(:rootca).and_return('/nonexistent/ca.crt')
-            allow(File).to receive(:file?).with('/nonexistent/ca.crt').and_return(false)
-          end
+      context 'with a redirect' do
+        it 'follows it and returns the final response' do
+          stub_request(:get, 'http://example.com/old')
+            .to_return(status: 302, headers: { 'Location' => 'http://example.com/new' })
+          stub_request(:get, 'http://example.com/new').to_return(status: 200, body: 'moved')
 
-          it 'aborts with error message' do
-            expect { Http.client }.to raise_error(SystemExit)
-          end
+          expect(Http.get('http://example.com/old').body).to eq('moved')
         end
 
-        context 'with verify_mode disabled' do
-          before do
-            allow(config).to receive(:verify_mode).and_return(false)
-          end
+        it 'raises after too many redirects' do
+          stub_request(:get, 'http://example.com/loop')
+            .to_return(status: 302, headers: { 'Location' => 'http://example.com/loop' })
 
-          it 'sets SSL verify mode to VERIFY_NONE' do
-            expect(Http.client.ssl_config.verify_mode).to eq(OpenSSL::SSL::VERIFY_NONE)
-          end
+          expect { Http.get('http://example.com/loop') }
+            .to raise_error(Http::Error, /Too many redirects/)
         end
       end
 
-      context 'with basic auth forced' do
-        before do
-          allow(config).to receive(:basic_auth).and_return(true)
-        end
+      context 'with an error response' do
+        it 'raises Http::Error for a failed request' do
+          stub_request(:get, 'http://example.com/missing').to_return(status: [404, 'Not Found'])
 
-        it 'forces basic authentication' do
-          expect(Http.client.www_auth.basic_auth.force_auth).to be true
+          expect { Http.get('http://example.com/missing') }
+            .to raise_error(Http::Error, '404 Not Found')
         end
       end
-    end
 
-    describe '.get' do
-      let(:client) { instance_double(HTTPClient) }
+      context 'with upstream authentication configured' do
+        it 'sends a basic auth header on every request' do
+          allow(config).to receive(:upstream_user).and_return('user')
+          allow(config).to receive(:upstream_password).and_return('pass')
+          stub_request(:get, 'http://example.com/private')
+            .with(basic_auth: %w[user pass])
+            .to_return(status: 200, body: 'ok')
 
-      before do
-        allow(Http).to receive(:client).and_return(client)
+          expect(Http.get('http://example.com/private').body).to eq('ok')
+        end
       end
 
-      context 'with successful response' do
-        let(:response) { double('Response', status: 200, body: 'content') }
+      context 'with proxy configuration' do
+        it 'routes the request through the configured proxy' do
+          allow(config).to receive(:proxy).and_return('http://proxy.example.com:8080')
+          stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'content')
 
-        it 'returns response for successful request' do
-          allow(client).to receive(:get)
-            .with('http://example.com', follow_redirect: true)
-            .and_return(response)
+          Http.get('http://example.com/')
 
-          expect(Http.get('http://example.com')).to eq(response)
+          expect(WebMock).to have_requested(:get, 'http://example.com/')
         end
       end
 
-      context 'with error response' do
-        let(:response) { double('Response', status: 404, reason: 'Not Found') }
-
-        it 'raises BadResponseError for failed request' do
-          allow(client).to receive(:get)
-            .with('http://example.com', follow_redirect: true)
-            .and_return(response)
+      context 'with an invalid rootca path' do
+        it 'aborts with an error message' do
+          allow(config).to receive(:rootca).and_return('/nonexistent/ca.crt')
 
-          expect { Http.get('http://example.com') }
-            .to raise_error(HTTPClient::BadResponseError, 'Not Found')
+          expect { Http.get('https://example.com/') }.to raise_error(SystemExit)
         end
       end
     end
   end
-end
+end

data/spec/gemirro/server_spec.rb

@@ -1,8 +1,9 @@
+require 'spec_helper'
 require 'rack/test'
 require 'json'
 require 'parallel'
 require 'sinatra/base'
-require 'thin'
+require 'puma'
 require 'base64'
 require 'gemirro/utils'
 require 'gemirro/mirror_directory'

data/spec/gemirro/source_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'gemirro/configuration'
 require 'gemirro/http'
 require 'gemirro/utils'
 require 'gemirro/source'