gemirro 1.6.0 → 3.0.0

data/lib/gemirro/server.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 require 'sinatra/base'
-require 'thin'
+require 'puma'
 require 'uri'
 require 'addressable/uri'
+require 'base64'
 
 module Gemirro
   ##
@@ -32,6 +33,7 @@ module Gemirro
       config.server.port = '2000' if config.server.port.nil?
 
       set :static, true
+      set :server, 'puma'
 
       set :views, Gemirro::Configuration.views_directory
       set :port, config.server.port
@@ -78,39 +80,6 @@ module Gemirro
       erb(:index, {}, gems: Utils.gems_collection)
     end
 
-    ##
-    # Return gem dependencies as marshaled binary
-    #
-    # @return [nil]
-    #
-    get '/api/v1/dependencies' do
-      content_type 'application/octet-stream'
-      if params[:gems].to_s != '' && params[:gems].to_s.split(',').any?
-        Marshal.dump(dependencies_loader(params[:gems].to_s.split(',')))
-      else
-        200
-      end
-    end
-
-    ##
-    # Return gem dependencies as json
-    #
-    # @return [nil]
-    #
-    get '/api/v1/dependencies.json' do
-      content_type 'application/json'
-
-      return '[]' unless params[:gems]
-
-      gem_names = params[:gems].to_s
-                               .split(',')
-                               .map(&:strip)
-                               .reject(&:empty?)
-      return '[]' if gem_names.empty?
-
-      JSON.dump(dependencies_loader(gem_names))
-    end
-
     ##
     # compact_index, Return list of available gem names
     #
@@ -120,10 +89,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 
@@ -136,10 +105,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 
@@ -155,10 +124,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 

data/lib/gemirro/source.rb

@@ -33,23 +33,10 @@ module Gemirro
     #
     def fetch_versions
       Utils.logger.info(
-        "Fetching #{Configuration.versions_file} on #{@name} (#{@host})"
+        "Fetching versions on #{@name} (#{@host})"
       )
 
-      Http.get("#{host}/#{Configuration.versions_file}").body
-    end
-
-    ##
-    # Fetches a list of all the available Gems and their versions.
-    #
-    # @return [String]
-    #
-    def fetch_prerelease_versions
-      Utils.logger.info(
-        "Fetching #{Configuration.prerelease_versions_file} " \
-        "on #{@name} (#{@host})"
-      )
-      Http.get("#{host}/#{Configuration.prerelease_versions_file}").body
+      Http.get("#{host}/versions").body
     end
 
     ##

data/lib/gemirro/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'gemirro/gem_version'
+
 module Gemirro
   ##
   # The Utils class is responsible for executing specific traitments
@@ -33,12 +35,10 @@ module Gemirro
       @gems_collection ||= { files: {}, values: nil }
 
       file_paths =
-        %i[specs prerelease_specs].collect do |specs_file_type|
-          File.join(
-            configuration.destination,
-            "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz.local"
-          )
-        end
+        Dir.glob(File.join(
+                   Gemirro.configuration.destination,
+                   'versions.*.*.list'
+                 ))
 
       has_file_changed =
         @gems_collection[:files] != file_paths.each_with_object({}) do |f, r|
@@ -50,12 +50,22 @@ module Gemirro
 
       gems = []
 
-      # parallel is not for mtime, it's for the Marshal.
-      Parallel.map(file_paths, in_threads: Utils.configuration.update_thread_count) do |file_path|
-        next unless File.exist?(file_path)
+      CompactIndex::VersionsFile.new(file_paths.last).contents.each_line.with_index do |line, index|
+        next if index < 2
+
+        gem_name = line.split[0]
+        versions = line.split[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
 
-        gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read))
-        @gems_collection[:files][file_path] = File.mtime(file_path)
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
+
+          gems << Gemirro::GemVersion.new(gem_name, version, platform)
+        end
       end
 
       @gems_collection[:values] = GemVersionCollection.new(gems)
@@ -187,7 +197,7 @@ module Gemirro
     # @return [Indexer]
     #
     def self.update_indexes
-      indexer = Gemirro::Indexer.new(Utils.configuration.destination)
+      indexer = Gemirro::Indexer.new
       indexer.only_origin = true
       indexer.ui = ::Gem::SilentUI.new
 

data/lib/gemirro/version.rb

@@ -2,5 +2,5 @@
 
 # Gemirro Version
 module Gemirro
-  VERSION = '1.6.0'
+  VERSION = '3.0.0'
 end

data/lib/gemirro/versions_fetcher.rb

@@ -22,29 +22,24 @@ module Gemirro
     # @return [Gemirro::VersionsFile]
     #
     def fetch
-      VersionsFile.load(
-        read_file(Configuration.versions_file),
-        read_file(Configuration.prerelease_versions_file, true)
-      )
+      return unless Gemirro.configuration.versions_file
+
+      VersionsFile.new(read_file(Gemirro.configuration.versions_file))
     end
 
     ##
     # Read file if exists otherwise download its from source
     #
     # @param [String] file name
-    # @param [TrueClass|FalseClass] prerelease Is prerelease or not
     #
-    def read_file(file, prerelease = false)
-      destination = Gemirro.configuration.destination
-      file_dst = File.join(destination, file)
-      unless File.exist?(file_dst)
+    def read_file(file)
+      unless File.exist?(file)
         throw 'No source defined' unless @source
 
-        File.write(file_dst, @source.fetch_versions) unless prerelease
-        File.write(file_dst, @source.fetch_prerelease_versions) if prerelease
+        File.write(file, @source.fetch_versions)
       end
 
-      File.read(file_dst)
+      File.read(file)
     end
   end
 end

data/lib/gemirro/versions_file.rb

@@ -11,36 +11,24 @@ module Gemirro
   #  @return [Hash]
   #
   class VersionsFile
-    attr_reader :versions, :versions_hash
+    attr_reader :versions_string, :versions_hash
 
     ##
     # Reads the versions file from the specified String.
     #
-    # @param [String] spec_content
-    # @param [String] prerelease_content
+    # @param [String] versions_content
     # @return [Gemirro::VersionsFile]
     #
-    def self.load(spec_content, prerelease_content)
-      buffer = StringIO.new(spec_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions = Marshal.load(reader.read)
-
-      buffer = StringIO.new(prerelease_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions.concat(Marshal.load(reader.read))
-
-      instance = new(versions)
-
-      reader.close
-
-      instance
-    end
 
     ##
-    # @param [Array] versions
+    # @param [String] versions
     #
-    def initialize(versions)
-      @versions      = versions
+    def initialize(versions_string)
+      unless versions_string.is_a? String
+        throw "#{versions_string.class} is wrong format, expect String; #{versions_string.inspect}"
+      end
+
+      @versions_string = versions_string
       @versions_hash = create_versions_hash
     end
 
@@ -53,10 +41,24 @@ module Gemirro
     def create_versions_hash
       hash = Hash.new { |h, k| h[k] = [] }
 
-      versions.each do |version|
-        hash[version[0]] << version
-      end
+      versions_string.each_line.with_index do |line, index|
+        next if index < 2
 
+        parts = line.split
+        gem_name = parts[0]
+        parts[-1]
+        versions = parts[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
+
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
+          hash[gem_name] << [gem_name, ::Gem::Version.new(version), platform]
+        end
+      end
       hash
     end
 

data/lib/gemirro.rb

@@ -4,14 +4,15 @@ require 'builder'
 require 'confstruct'
 require 'digest/sha2'
 require 'fileutils'
-require 'httpclient'
 require 'json'
 require 'logger'
+require 'net/http'
+require 'openssl'
+require 'optparse'
 require 'parallel'
 require 'rubygems'
 require 'rubygems/indexer'
 require 'rubygems/user_interaction'
-require 'slop'
 require 'stringio'
 require 'tempfile'
 require 'zlib'
@@ -34,8 +35,3 @@ require 'gemirro/versions_fetcher'
 require 'gemirro/gems_fetcher'
 
 require 'gemirro/cli'
-require 'gemirro/cli/index'
-require 'gemirro/cli/init'
-require 'gemirro/cli/list'
-require 'gemirro/cli/server'
-require 'gemirro/cli/update'

data/spec/gemirro/cli_spec.rb

@@ -1,7 +1,6 @@
 require 'spec_helper'
 require 'gemirro/cli'
 require 'gemirro/mirror_file'
-require 'slop'
 
 # Gemirro tests
 module Gemirro
@@ -10,21 +9,47 @@ module Gemirro
     describe 'CLI' do
       include FakeFS::SpecHelpers
 
-      it 'should return options' do
-        options = CLI.options
-        expect(options).to be_a(::Slop)
-        expect(options.config[:strict]).to be_truthy
-        expect(options.config[:banner])
-          .to eq('Usage: gemirro [COMMAND] [OPTIONS]')
-        expect(options.to_s)
-          .to match(/-v, --version(\s+)Shows the current version/)
-        expect(options.to_s)
-          .to match(/-h, --help(\s+)Display this help message./)
+      it 'should print version information for -v' do
+        expect { CLI.run(['-v']) }.to output(/gemirro v.* on ruby/).to_stdout
+      end
+
+      it 'should print version information for --version' do
+        expect { CLI.run(['--version']) }.to output(/gemirro v.* on ruby/).to_stdout
+      end
+
+      it 'should print usage for -h' do
+        expect { CLI.run(['-h']) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
+
+      it 'should print usage for --help' do
+        expect { CLI.run(['--help']) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
+
+      it 'should print usage when no argument is given' do
+        expect { CLI.run([]) }.to output(/Usage: gemirro \[COMMAND\] \[OPTIONS\]/).to_stdout
+      end
 
-        version = options.fetch_option(:v)
-        expect(version.short).to eq('v')
-        expect(version.long).to eq('version')
-        expect { version.call }.to output(/gemirro v.* on ruby/).to_stdout
+      it 'should list the available commands in the usage banner' do
+        output = capture_stdout { CLI.run(['-h']) }
+
+        CLI::COMMANDS.each do |name, description|
+          expect(output).to match(/#{name}(\s+)#{Regexp.escape(description)}/)
+        end
+      end
+
+      it 'should set $PROGRAM_NAME' do
+        $PROGRAM_NAME = 'rspec'
+        CLI.run(['-v'])
+        expect($PROGRAM_NAME).to eq('gemirro')
+      end
+
+      it 'should print an error and exit for an unknown command' do
+        expect { CLI.run(['nope']) }.to output(/Unknown command: nope/).to_stdout.and raise_error(SystemExit)
+      end
+
+      it 'should print an option error for an unknown flag' do
+        expect { CLI.run(['list', '--nope']) }
+          .to output(/invalid option: --nope/).to_stdout.and raise_error(SystemExit)
       end
 
       it 'should retrieve version information' do
@@ -45,6 +70,15 @@ module Gemirro
         file.write('test')
         expect { CLI.load_configuration('config.rb') }.to raise_error LoadError
       end
+
+      def capture_stdout
+        original = $stdout
+        $stdout = StringIO.new
+        yield
+        $stdout.string
+      ensure
+        $stdout = original
+      end
     end
   end
 end

data/spec/gemirro/configuration_spec.rb

@@ -39,8 +39,8 @@ module Gemirro
       expect(Configuration.marshal_identifier).to match(/Marshal\.(\d+)\.(\d+)/)
     end
 
-    it 'should return versions file' do
-      expect(Configuration.versions_file).to match(/specs\.(\d+)\.(\d+).gz/)
+    it 'should error on version since no source is defined' do
+      expect(Gemirro.configuration.versions_file).to match(nil)
     end
 
     it 'should return marshal file' do
@@ -84,5 +84,12 @@ module Gemirro
       expect(result.host).to eq('https://rubygems.org')
       expect(result.name).to eq('rubygems')
     end
+
+
+    it 'should return versions file with source' do
+      result = @config.define_source('RubyGems', 'https://rubygems.org') do
+      end
+      expect(@config.versions_file).to match(/rubygems_org_versions$/)
+    end
   end
 end

data/spec/gemirro/gems_fetcher_spec.rb

@@ -12,7 +12,7 @@ module Gemirro
 
     before(:each) do
       @source = Source.new('RubyGems', 'https://rubygems.org')
-      @versions_file = VersionsFile.new(['0.0.1', '0.0.2'])
+      @versions_file = VersionsFile.new(%(created_at: 2025-04-24T03:46:59Z\n---\nrack 3.0.0,3.0.1 d545a45462d63b1b4865bbb89a109366))
       @fetcher = GemsFetcher.new(@source, @versions_file)
       Gemirro.configuration.ignored_gems.clear
     end
@@ -47,7 +47,7 @@ module Gemirro
       version = ::Gem::Version.new('0.0.1')
       Utils.configuration.ignore_gem('gemirro', '0.0.1', 'ruby')
       allow(@source).to receive(:fetch_gem)
-        .once.with('gemirro', version).and_raise(ArgumentError)
+        .once.with('gemirro-0.0.1.gem').and_raise(ArgumentError)
       allow(Utils.logger).to receive(:error)
         .once.with(/Failed to retrieve/)
       allow(Utils.logger).to receive(:debug)

data/spec/gemirro/http_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
-require 'httpclient'
+require 'webmock/rspec'
+require 'gemirro/utils'
 require 'gemirro/http'
 
 module Gemirro
@@ -7,88 +8,91 @@ module Gemirro
     let(:config) { double('Configuration') }
 
     before do
-      Http.instance_variable_set(:@client, nil)
+      Http.reset!
       allow(Utils).to receive(:configuration).and_return(config)
     end
 
-    describe '.client' do
-      it 'initializes a new HTTPClient' do
-        expect(Http.client).to be_a(HTTPClient)
-      end
+    after { Http.reset! }
 
-      context 'with proxy configuration' do
-        it 'sets proxy configuration' do
-          allow(config).to receive(:proxy).and_return('http://proxy.example.com:8080')
+    describe '.get' do
+      context 'with successful response' do
+        it 'returns the response' do
+          stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'content')
+
+          response = Http.get('http://example.com/')
+          expect(response).to be_a(Net::HTTPSuccess)
+          expect(response.body).to eq('content')
+        end
+
+        it 'reuses the connection for a second request on the same host' do
+          stub_request(:get, 'http://example.com/one').to_return(status: 200, body: 'one')
+          stub_request(:get, 'http://example.com/two').to_return(status: 200, body: 'two')
+
+          Http.get('http://example.com/one')
+          Http.get('http://example.com/two')
 
-          expect(Http.client.proxy.to_s).to eq('http://proxy.example.com:8080')
+          expect(a_request(:get, 'http://example.com/one')).to have_been_made.once
+          expect(a_request(:get, 'http://example.com/two')).to have_been_made.once
         end
       end
 
-      context 'with SSL configuration' do
-        context 'with invalid root CA path' do
-          before do
-            allow(config).to receive(:rootca).and_return('/nonexistent/ca.crt')
-            allow(File).to receive(:file?).with('/nonexistent/ca.crt').and_return(false)
-          end
+      context 'with a redirect' do
+        it 'follows it and returns the final response' do
+          stub_request(:get, 'http://example.com/old')
+            .to_return(status: 302, headers: { 'Location' => 'http://example.com/new' })
+          stub_request(:get, 'http://example.com/new').to_return(status: 200, body: 'moved')
 
-          it 'aborts with error message' do
-            expect { Http.client }.to raise_error(SystemExit)
-          end
+          expect(Http.get('http://example.com/old').body).to eq('moved')
         end
 
-        context 'with verify_mode disabled' do
-          before do
-            allow(config).to receive(:verify_mode).and_return(false)
-          end
+        it 'raises after too many redirects' do
+          stub_request(:get, 'http://example.com/loop')
+            .to_return(status: 302, headers: { 'Location' => 'http://example.com/loop' })
 
-          it 'sets SSL verify mode to VERIFY_NONE' do
-            expect(Http.client.ssl_config.verify_mode).to eq(OpenSSL::SSL::VERIFY_NONE)
-          end
+          expect { Http.get('http://example.com/loop') }
+            .to raise_error(Http::Error, /Too many redirects/)
         end
       end
 
-      context 'with basic auth forced' do
-        before do
-          allow(config).to receive(:basic_auth).and_return(true)
-        end
+      context 'with an error response' do
+        it 'raises Http::Error for a failed request' do
+          stub_request(:get, 'http://example.com/missing').to_return(status: [404, 'Not Found'])
 
-        it 'forces basic authentication' do
-          expect(Http.client.www_auth.basic_auth.force_auth).to be true
+          expect { Http.get('http://example.com/missing') }
+            .to raise_error(Http::Error, '404 Not Found')
         end
       end
-    end
 
-    describe '.get' do
-      let(:client) { instance_double(HTTPClient) }
+      context 'with upstream authentication configured' do
+        it 'sends a basic auth header on every request' do
+          allow(config).to receive(:upstream_user).and_return('user')
+          allow(config).to receive(:upstream_password).and_return('pass')
+          stub_request(:get, 'http://example.com/private')
+            .with(basic_auth: %w[user pass])
+            .to_return(status: 200, body: 'ok')
 
-      before do
-        allow(Http).to receive(:client).and_return(client)
+          expect(Http.get('http://example.com/private').body).to eq('ok')
+        end
       end
 
-      context 'with successful response' do
-        let(:response) { double('Response', status: 200, body: 'content') }
+      context 'with proxy configuration' do
+        it 'routes the request through the configured proxy' do
+          allow(config).to receive(:proxy).and_return('http://proxy.example.com:8080')
+          stub_request(:get, 'http://example.com/').to_return(status: 200, body: 'content')
 
-        it 'returns response for successful request' do
-          allow(client).to receive(:get)
-            .with('http://example.com', follow_redirect: true)
-            .and_return(response)
+          Http.get('http://example.com/')
 
-          expect(Http.get('http://example.com')).to eq(response)
+          expect(WebMock).to have_requested(:get, 'http://example.com/')
         end
       end
 
-      context 'with error response' do
-        let(:response) { double('Response', status: 404, reason: 'Not Found') }
-
-        it 'raises BadResponseError for failed request' do
-          allow(client).to receive(:get)
-            .with('http://example.com', follow_redirect: true)
-            .and_return(response)
+      context 'with an invalid rootca path' do
+        it 'aborts with an error message' do
+          allow(config).to receive(:rootca).and_return('/nonexistent/ca.crt')
 
-          expect { Http.get('http://example.com') }
-            .to raise_error(HTTPClient::BadResponseError, 'Not Found')
+          expect { Http.get('https://example.com/') }.to raise_error(SystemExit)
         end
       end
     end
   end
-end
+end

data/spec/gemirro/indexer_spec.rb

@@ -24,7 +24,7 @@ module Gemirro
 
       dir = MirrorDirectory.new('/tmp')
       dir.add_directory('test')
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
 
       Struct.new('HttpGet', :code, :body)
       http_get = Struct::HttpGet.new(200, 'bad')
@@ -43,7 +43,7 @@ module Gemirro
       allow(::Gem.configuration).to receive(:really_verbose)
         .once.and_return(true)
 
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
       indexer.quick_marshal_dir = '/tmp/gem_generate_index/quick/Marshal.4.8'
       indexer.dest_directory = '/tmp/test'
       indexer.directory = '/tmp/gem_generate_index'
@@ -97,7 +97,7 @@ module Gemirro
     end
 
     it 'should build indices' do
-      indexer = Indexer.new('/')
+      indexer = Indexer.new
       dir = MirrorDirectory.new('/')
       dir.add_directory('gems')
       dir.add_directory('quick')
@@ -140,7 +140,7 @@ module Gemirro
     end
 
     it 'should update index and exit ruby gems' do
-      indexer = Indexer.new('/')
+      indexer = Indexer.new
       MirrorDirectory.new('/')
       MirrorFile.new('/specs.4.8').write('')
       expect { indexer.update_index }.to raise_error(::Gem::SystemExitException)
@@ -152,7 +152,7 @@ module Gemirro
       dir.add_directory('test/gems')
       dir.add_directory('test/quick')
 
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
       indexer.quick_marshal_dir = '/tmp/gem_generate_index/quick/Marshal.4.8'
       indexer.dest_directory = '/tmp/test'
       indexer.directory = '/tmp/gem_generate_index'