gemirro 1.6.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23b3ba616d8007252b6be8b5a3edd675246bb89839e8ae13fd2b01dcf9dff4d4
-  data.tar.gz: a82c036b510ce0da4f1c6475a7b6baac5afbcc065ce911bec8a6e789b84ad9ee
+  metadata.gz: e54b02e5575feb8b7a4fcbb40f5fd91764fb0f589368e4a9207f6a5074fe2f05
+  data.tar.gz: 14449b2b15d5477216adaecea6e36aff26dae8b30104e3db627bdcdbdf7c6b7d
 SHA512:
-  metadata.gz: 6f31d4f862a961c03b097b9eff5bff4a5d8b9e02816346a9dcf581c393213b4f8754be127115374895dd3613179a550b4e1e63bb26fdaa13e50ebe1646693998
-  data.tar.gz: fc8957ef3a2231aca1c9c386cc27316b252e46267b360948b8cfe6fd0f1300b6a788edf6fbe3553d2a6889536216bf9921f06ec121f3cebb68a50a7111a636ed
+  metadata.gz: ba26ee7c502dd1bbfc02fd55e1263097dedad78670281d8f775bd9c5940be1c46032130cd76be02fbdee3b8a4c2f24504f39b38c7e5901497c7eff28d13a0307
+  data.tar.gz: ce425c173bcfc46ea6298cc0911ddec8a838f321749f9ec782da8323afdaf3268f6071d629be5fa6b129fbe29d2198e930b6202225d23f17ac582fac71fc1533

data/.gitignore

@@ -4,3 +4,4 @@
 /.ruby-version
 /*.gem
 /.bundle
+.DS_Store

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gemirro (1.6.0)
+    gemirro (2.0.0)
       addressable (~> 2.8)
       builder (~> 3.2)
       compact_index (~> 0.15)
@@ -50,7 +50,7 @@ GEM
       racc
     public_suffix (6.0.1)
     racc (1.8.1)
-    rack (2.2.10)
+    rack (2.2.14)
     rack-protection (3.2.0)
       base64 (>= 0.1.0)
       rack (~> 2.2, >= 2.2.4)

data/MANIFEST

@@ -55,9 +55,6 @@ template/config.rb
 template/logs/.gitkeep
 template/public/dist/css/gemirro.css
 template/public/gems/.gitkeep
-template/public/latest_specs.4.8
-template/public/prerelease_specs.4.8
-template/public/specs.4.8
 views/gem.erb
 views/index.erb
 views/layout.erb

data/lib/gemirro/cli/index.rb

@@ -19,10 +19,11 @@ Gemirro::CLI.options.command 'index' do
       abort
     end
 
-    indexer    = Gemirro::Indexer.new(config.destination)
+    indexer    = Gemirro::Indexer.new
     indexer.ui = Gem::SilentUI.new
 
-    if File.exist?(File.join(config.destination, "specs.#{Gem.marshal_version}.gz"))
+    if File.exist?(File.join(config.versions_file))
+      indexer.download_source_versions
       if opts[:u]
         config.logger.info('Generating index updates')
         indexer.update_index
@@ -31,7 +32,7 @@ Gemirro::CLI.options.command 'index' do
         indexer.generate_index
       end
     else
-      config.logger.error("/public/specs.#{Gem.marshal_version}.gz file is missing.")
+      config.logger.error("#{File.basename(config.versions_file)} file is missing.")
       config.logger.error('Run "gemirro update" before running index.')
     end
   end

data/lib/gemirro/cli/init.rb

@@ -26,11 +26,6 @@ Gemirro::CLI.options.command 'init' do
       end
     end
 
-    # make sure index updates blank local specs
-    ['specs.4.8', 'latest_specs.4.8', 'prerelease_specs.4.8'].each do |s|
-      File.utime(Time.at(0), Time.at(0), File.join(directory, 'public', s))
-    end
-
     puts "Initialized empty mirror in #{directory}"
   end
 end

data/lib/gemirro/cli.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require File.expand_path('../version', __FILE__)
+
 module Gemirro
   # CLI mode
   module CLI
@@ -58,7 +60,7 @@ module Gemirro
     # @return [String]
     #
     def self.version_information
-      "gemirro v#{VERSION} on #{RUBY_DESCRIPTION}"
+      "gemirro v#{Gemirro::VERSION} on #{RUBY_DESCRIPTION}"
     end
   end
 end

data/lib/gemirro/configuration.rb

@@ -99,18 +99,10 @@ module Gemirro
     #
     # @return [String]
     #
-    def self.versions_file
-      "specs.#{marshal_version}.gz"
-    end
+    def versions_file
+      return unless @source
 
-    ##
-    # Returns the name of the file that contains an index
-    # of all the prerelease versions.
-    #
-    # @return [String]
-    #
-    def self.prerelease_versions_file
-      "prerelease_specs.#{marshal_version}.gz"
+      File.expand_path("#{URI.parse(@source.host).host.gsub('.', '_')}_versions", destination.to_s)
     end
 
     ##

data/lib/gemirro/gems_fetcher.rb

@@ -55,6 +55,8 @@ module Gemirro
     # @return [Array]
     #
     def versions_for(gem)
+      return [] unless @versions_file
+
       available = @versions_file.versions_for(gem.name)
       return [available.last] if gem.only_latest?
 

data/lib/gemirro/indexer.rb

@@ -35,19 +35,16 @@ module Gemirro
     # @param [Hash] options Indexer options
     # @return [Array]
     ##
-    def initialize(directory, options = {})
+    def initialize(options = {})
       require 'fileutils'
       require 'tmpdir'
       require 'zlib'
       require 'builder/xchar'
       require 'compact_index'
 
-      options.merge!({ build_modern: true, build_compact: true })
+      options.merge!({ build_modern: false })
 
-      @build_modern = options[:build_modern]
-      @build_compact = options[:build_compact]
-
-      @dest_directory = directory
+      @dest_directory = Gemirro.configuration.destination
       @directory =
         File.join(Dir.tmpdir, "gem_generate_index_#{rand(1_000_000_000)}")
 
@@ -69,22 +66,12 @@ module Gemirro
       @latest_index =
         File.join(@quick_dir, 'latest_index')
 
-      @specs_index =
-        File.join(@directory, "specs.#{::Gem.marshal_version}")
       @latest_specs_index =
         File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
-      @prerelease_specs_index =
-        File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
-      @dest_specs_index =
-        File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
       @dest_latest_specs_index =
         File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
-      @dest_prerelease_specs_index =
-        File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")
       @infos_dir =
         File.join(@dest_directory, 'info')
-      @api_v1_dependencies_dir =
-        File.join(@dest_directory, 'api', 'v1', 'dependencies')
 
       @files = []
     end
@@ -92,16 +79,9 @@ module Gemirro
     ##
     # Generate indices on the destination directory
     #
-    def install_indices
-      install_indicies
-    end
-
-    ##
-    # Generate indicies on the destination directory
-    #
     # @return [Array]
     #
-    def install_indicies
+    def install_indices
       Utils.logger
            .debug("Downloading index into production dir #{@dest_directory}")
 
@@ -111,24 +91,19 @@ module Gemirro
       if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
         files.delete @quick_marshal_dir
         FileUtils.mkdir_p(File.dirname(@quick_marshal_dir_base), verbose: verbose)
-        FileUtils.rm_rf(@quick_marshal_dir_base, verbose: verbose)
+        if @quick_marshal_dir_base && File.exist?(@quick_marshal_dir_base)
+          FileUtils.rm_rf(@quick_marshal_dir_base, verbose: verbose)
+        end
         FileUtils.mv(@quick_marshal_dir, @quick_marshal_dir_base, verbose: verbose, force: true)
       end
 
       files.each do |path|
         file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
 
-        if ["#{@specs_index}.gz",
-            "#{@latest_specs_index}.gz",
-            "#{@prerelease_specs_index}.gz"].include?(path)
-          res = build_zlib_file(file, File.join(@directory, file), File.join(@dest_directory, file), true)
-          next unless res
-        else
-          source_content = download_from_source(file)
-          next if source_content.nil?
+        source_content = download_from_source(file)
+        next if source_content.nil?
 
-          MirrorFile.new(File.join(@dest_directory, file)).write(source_content)
-        end
+        MirrorFile.new(File.join(@dest_directory, file)).write(source_content)
 
         FileUtils.rm_rf(path)
       end
@@ -142,7 +117,7 @@ module Gemirro
     #
     def download_from_source(file)
       source_host = Gemirro.configuration.source.host
-      Utils.logger.info("Download from source: #{file}")
+      Utils.logger.info("Download from source #{source_host}/#{file}")
       resp = Http.get("#{source_host}/#{File.basename(file)}")
       return unless resp.code == 200
 
@@ -155,104 +130,18 @@ module Gemirro
     # @return [Array]
     #
     def build_indices
-      build_indicies
-    end
-
-    ##
-    # Build indicies
-    #
-    # @return [Array]
-    #
-    def build_indicies
       specs = *map_gems_to_specs(gem_file_list)
       specs.select! { |s| s.instance_of?(::Gem::Specification) }
       ::Gem::Specification.dirs = []
       ::Gem::Specification.all = specs
 
-      if ::Gem::VERSION >= '2.5.0'
-        build_marshal_gemspecs(specs)
-        build_modern_indices(specs) if @build_modern
-        compress_indices
-      else
-        build_marshal_gemspecs
-        build_modern_indicies if @build_modern
-        compress_indicies
-      end
-
-      build_api_v1_dependencies(specs)
-
-      return unless @build_compact
+      build_marshal_gemspecs(specs)
 
       build_compact_index_names
       build_compact_index_infos(specs)
       build_compact_index_versions(specs)
     end
 
-    ##
-    # Cache Modern Index endpoints /api/v1/dependencies?gems= and /api/v1/dependencies.json?gems=
-    # This single request may include many fragments. server.rb determines which are required per request.
-    #
-    # @return nil
-    #
-    def build_api_v1_dependencies(specs, partial = false)
-      FileUtils.mkdir_p(@api_v1_dependencies_dir)
-
-      if partial
-        specs.collect(&:name).uniq do |name|
-          FileUtils.rm_rf(Dir.glob(File.join(@api_v1_dependencies_dir, "#{name}.*.*.list")))
-        end
-      else
-        FileUtils.rm_rf(Dir.glob(File.join(@api_v1_dependencies_dir, '*.list')))
-      end
-
-      grouped_specs = specs.sort_by(&:name).group_by(&:name)
-      grouped_specs.each_with_index do |(name, gem_versions), index|
-        Utils.logger.info("[#{index + 1}/#{grouped_specs.size}]: Caching /api/v1/dependencies/#{name}")
-
-        gem_versions =
-          gem_versions.sort do |a, b|
-            a.version <=> b.version
-          end
-
-        cg = []
-        Parallel.each_with_index(
-          gem_versions,
-          in_threads: Utils.configuration.update_thread_count
-        ) do |spec, index2|
-          next if spec.nil?
-
-          dependencies = spec.dependencies.select do |d|
-            d.type == :runtime
-          end
-
-          dependencies = dependencies.collect do |d|
-            [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s]
-          end
-
-          cg[index2] =
-            {
-              name: spec.name,
-              number: spec.version.to_s,
-              platform: spec.platform,
-              dependencies: dependencies
-            }
-        end
-
-        Tempfile.create("api_v1_dependencies_#{name}.list") do |f|
-          f.write Marshal.dump(cg)
-          f.rewind
-
-          FileUtils.cp(
-            f.path,
-            File.join(
-              @api_v1_dependencies_dir,
-              "#{name}.#{Digest::MD5.file(f.path).hexdigest}.#{Digest::SHA256.file(f.path).hexdigest}.list"
-            )
-          )
-        end
-      end
-    end
-
     ##
     # Cache compact_index endpoint /names
     # Report all gems with versions available. Does not require opening spec files.
@@ -323,7 +212,7 @@ module Gemirro
         end
 
       Tempfile.create('versions.list') do |f|
-        previous_versions_file = Dir.glob(File.join(@dest_directory, 'versions*.list')).last
+        previous_versions_file = Dir.glob(File.join(@dest_directory, 'versions.*.*.list')).last
 
         if partial && previous_versions_file
           versions_file = CompactIndex::VersionsFile.new(previous_versions_file)
@@ -336,7 +225,7 @@ module Gemirro
         f.write CompactIndex.versions(versions_file, cg)
         f.rewind
 
-        FileUtils.rm_rf(Dir.glob(File.join(@dest_directory, 'versions*.list')))
+        FileUtils.rm_rf(Dir.glob(File.join(@dest_directory, 'versions.*.*.list')))
 
         FileUtils.cp(
           f.path,
@@ -359,7 +248,7 @@ module Gemirro
     # @return nil
     #
     def build_compact_index_infos(specs, partial = false)
-      FileUtils.mkdir_p(@infos_dir)
+      FileUtils.mkdir_p(@infos_dir, verbose: verbose)
 
       if partial
         specs.collect(&:name).uniq do |name|
@@ -468,13 +357,8 @@ module Gemirro
             next
           end
 
-          if ::Gem::VERSION >= '2.5.0'
-            spec.abbreviate
-            spec.sanitize
-          else
-            abbreviate(spec)
-            sanitize(spec)
-          end
+          spec.abbreviate
+          spec.sanitize
 
           spec
         rescue SignalException
@@ -507,14 +391,15 @@ module Gemirro
       indexed_gemfiles = Dir.glob('*.gemspec.rz', base: @quick_marshal_dir_base).collect { |x| x.gsub(/spec.rz$/, '') }
 
       @updated_gems = []
+
       # detect files manually added to public/gems
       @updated_gems += (present_gemfiles - indexed_gemfiles).collect { |x| File.join(@dest_directory, 'gems', x) }
       # detect files manually deleted from public/gems
       @updated_gems += (indexed_gemfiles - present_gemfiles).collect { |x| File.join(@dest_directory, 'gems', x) }
 
-      specs_mtime =
+      versions_mtime =
         begin
-          File.stat(@dest_specs_index).mtime
+          File.stat(Dir.glob(File.join(@dest_directory, 'versions*.list')).last).mtime
         rescue StandardError
           Time.at(0)
         end
@@ -525,7 +410,7 @@ module Gemirro
         gem_file_list.select do |gem|
           gem_mtime = File.stat(gem).mtime
           newest_mtime = gem_mtime if gem_mtime > newest_mtime
-          gem_mtime > specs_mtime
+          gem_mtime > versions_mtime
         end
 
       @updated_gems.uniq!
@@ -538,7 +423,7 @@ module Gemirro
       specs = map_gems_to_specs(@updated_gems)
 
       # specs only includes latest discovered files.
-      # /info/[gemname] and /api/v1/dependencies can not be rebuilt
+      # /info/[gemname] can not be rebuilt
       # incrementally, so retrive specs for all versions of these gems.
       gem_name_updates = specs.collect(&:name).uniq
       u2 =
@@ -549,108 +434,24 @@ module Gemirro
       Utils.logger.info('Reloading for /info/[gemname]')
       version_specs = map_gems_to_specs(u2)
 
-      prerelease, released = specs.partition { |s| s.version.prerelease? }
-
       ::Gem::Specification.dirs = []
       ::Gem::Specification.all = *specs
-      files =
-        if ::Gem::VERSION >= '2.5.0'
-          build_marshal_gemspecs specs
-        else
-          build_marshal_gemspecs
-        end
-
-      ::Gem.time('Updated indexes') do
-        update_specs_index(
-          released,
-          @dest_specs_index,
-          @specs_index
-        )
-        update_specs_index(
-          released,
-          @dest_latest_specs_index,
-          @latest_specs_index
-        )
-        update_specs_index(
-          prerelease,
-          @dest_prerelease_specs_index,
-          @prerelease_specs_index
-        )
-      end
-
-      if ::Gem::VERSION >= '2.5.0'
-        compress_indices
-      else
-        compress_indicies
-      end
+      build_marshal_gemspecs specs
 
-      build_api_v1_dependencies(version_specs, true)
-
-      if @build_compact
-        build_compact_index_names
-        build_compact_index_infos(version_specs, true)
-        build_compact_index_versions(specs, true)
-      end
-
-      Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
-      files << @specs_index
-      files << "#{@specs_index}.gz"
-      files << @latest_specs_index
-      files << "#{@latest_specs_index}.gz"
-      files << @prerelease_specs_index
-      files << "#{@prerelease_specs_index}.gz"
-
-      files.each do |path|
-        file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
-
-        if ["#{@specs_index}.gz", "#{@latest_specs_index}.gz", "#{@prerelease_specs_index}.gz"].include?(path)
-          res = build_zlib_file(file, File.join(@directory, file), File.join(@dest_directory, file))
-          next unless res
-        else
-          FileUtils.mv(
-            File.join(@directory, file),
-            File.join(@dest_directory, file),
-            verbose: verbose,
-            force: true
-          )
-        end
-
-        File.utime(newest_mtime, newest_mtime, File.join(@dest_directory, file))
-      end
-    end
-
-    def build_zlib_file(file, src_name, dst_name, from_source = false)
-      content = Marshal.load(Zlib::GzipReader.open(src_name).read)
-      create_zlib_file("#{dst_name}.local", content)
-
-      return false if @only_origin
-
-      if from_source
-        source_content = download_from_source(file)
-        source_content = Marshal.load(
-          Zlib::GzipReader.new(
-            StringIO.new(source_content)
-          ).read
-        )
-      else
-        source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
-      end
-
-      return false if source_content.nil?
-
-      new_content = source_content.concat(content).uniq
-      create_zlib_file(dst_name, new_content)
+      build_compact_index_infos(version_specs, true)
+      build_compact_index_versions(specs, true)
+      build_compact_index_names
     end
 
-    def create_zlib_file(dst_name, content)
-      Tempfile.create(File.basename(dst_name)) do |f|
-        gzf = Zlib::GzipWriter.new(f)
-        gzf.write(Marshal.dump(content))
-        gzf.close
+    def download_source_versions
+      Tempfile.create(File.basename(Gemirro.configuration.versions_file)) do |f|
+        f.write(download_from_source('versions'))
+        f.close
 
+        FileUtils.rm(Gemirro.configuration.versions_file, verbose: verbose)
         FileUtils.cp(
           f.path,
-          dst_name,
+          Gemirro.configuration.versions_file,
           verbose: verbose
         )
       end

data/lib/gemirro/server.rb

@@ -4,6 +4,7 @@ require 'sinatra/base'
 require 'thin'
 require 'uri'
 require 'addressable/uri'
+require 'base64'
 
 module Gemirro
   ##
@@ -78,39 +79,6 @@ module Gemirro
       erb(:index, {}, gems: Utils.gems_collection)
     end
 
-    ##
-    # Return gem dependencies as marshaled binary
-    #
-    # @return [nil]
-    #
-    get '/api/v1/dependencies' do
-      content_type 'application/octet-stream'
-      if params[:gems].to_s != '' && params[:gems].to_s.split(',').any?
-        Marshal.dump(dependencies_loader(params[:gems].to_s.split(',')))
-      else
-        200
-      end
-    end
-
-    ##
-    # Return gem dependencies as json
-    #
-    # @return [nil]
-    #
-    get '/api/v1/dependencies.json' do
-      content_type 'application/json'
-
-      return '[]' unless params[:gems]
-
-      gem_names = params[:gems].to_s
-                               .split(',')
-                               .map(&:strip)
-                               .reject(&:empty?)
-      return '[]' if gem_names.empty?
-
-      JSON.dump(dependencies_loader(gem_names))
-    end
-
     ##
     # compact_index, Return list of available gem names
     #
@@ -120,10 +88,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 
@@ -136,10 +104,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 
@@ -155,10 +123,10 @@ module Gemirro
       content_type 'text/plain'
 
       content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last
-      _, etag, repr_digest, _ = content_path.split('.', -4)
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      headers 'etag' => etag
-      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
       send_file content_path
     end
 

data/lib/gemirro/source.rb

@@ -33,23 +33,10 @@ module Gemirro
     #
     def fetch_versions
       Utils.logger.info(
-        "Fetching #{Configuration.versions_file} on #{@name} (#{@host})"
+        "Fetching versions on #{@name} (#{@host})"
       )
 
-      Http.get("#{host}/#{Configuration.versions_file}").body
-    end
-
-    ##
-    # Fetches a list of all the available Gems and their versions.
-    #
-    # @return [String]
-    #
-    def fetch_prerelease_versions
-      Utils.logger.info(
-        "Fetching #{Configuration.prerelease_versions_file} " \
-        "on #{@name} (#{@host})"
-      )
-      Http.get("#{host}/#{Configuration.prerelease_versions_file}").body
+      Http.get("#{host}/versions").body
     end
 
     ##

data/lib/gemirro/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'gemirro/gem_version'
+
 module Gemirro
   ##
   # The Utils class is responsible for executing specific traitments
@@ -33,12 +35,10 @@ module Gemirro
       @gems_collection ||= { files: {}, values: nil }
 
       file_paths =
-        %i[specs prerelease_specs].collect do |specs_file_type|
-          File.join(
-            configuration.destination,
-            "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz.local"
-          )
-        end
+        Dir.glob(File.join(
+                   Gemirro.configuration.destination,
+                   'versions.*.*.list'
+                 ))
 
       has_file_changed =
         @gems_collection[:files] != file_paths.each_with_object({}) do |f, r|
@@ -50,12 +50,22 @@ module Gemirro
 
       gems = []
 
-      # parallel is not for mtime, it's for the Marshal.
-      Parallel.map(file_paths, in_threads: Utils.configuration.update_thread_count) do |file_path|
-        next unless File.exist?(file_path)
+      CompactIndex::VersionsFile.new(file_paths.last).contents.each_line.with_index do |line, index|
+        next if index < 2
+
+        gem_name = line.split[0]
+        versions = line.split[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
 
-        gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read))
-        @gems_collection[:files][file_path] = File.mtime(file_path)
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
+
+          gems << Gemirro::GemVersion.new(gem_name, version, platform)
+        end
       end
 
       @gems_collection[:values] = GemVersionCollection.new(gems)
@@ -187,7 +197,7 @@ module Gemirro
     # @return [Indexer]
     #
     def self.update_indexes
-      indexer = Gemirro::Indexer.new(Utils.configuration.destination)
+      indexer = Gemirro::Indexer.new
       indexer.only_origin = true
       indexer.ui = ::Gem::SilentUI.new
 

data/lib/gemirro/version.rb

@@ -2,5 +2,5 @@
 
 # Gemirro Version
 module Gemirro
-  VERSION = '1.6.0'
+  VERSION = '2.0.0'
 end

data/lib/gemirro/versions_fetcher.rb

@@ -22,29 +22,24 @@ module Gemirro
     # @return [Gemirro::VersionsFile]
     #
     def fetch
-      VersionsFile.load(
-        read_file(Configuration.versions_file),
-        read_file(Configuration.prerelease_versions_file, true)
-      )
+      return unless Gemirro.configuration.versions_file
+
+      VersionsFile.new(read_file(Gemirro.configuration.versions_file))
     end
 
     ##
     # Read file if exists otherwise download its from source
     #
     # @param [String] file name
-    # @param [TrueClass|FalseClass] prerelease Is prerelease or not
     #
-    def read_file(file, prerelease = false)
-      destination = Gemirro.configuration.destination
-      file_dst = File.join(destination, file)
-      unless File.exist?(file_dst)
+    def read_file(file)
+      unless File.exist?(file)
         throw 'No source defined' unless @source
 
-        File.write(file_dst, @source.fetch_versions) unless prerelease
-        File.write(file_dst, @source.fetch_prerelease_versions) if prerelease
+        File.write(file, @source.fetch_versions)
       end
 
-      File.read(file_dst)
+      File.read(file)
     end
   end
 end

data/lib/gemirro/versions_file.rb

@@ -11,36 +11,24 @@ module Gemirro
   #  @return [Hash]
   #
   class VersionsFile
-    attr_reader :versions, :versions_hash
+    attr_reader :versions_string, :versions_hash
 
     ##
     # Reads the versions file from the specified String.
     #
-    # @param [String] spec_content
-    # @param [String] prerelease_content
+    # @param [String] versions_content
     # @return [Gemirro::VersionsFile]
     #
-    def self.load(spec_content, prerelease_content)
-      buffer = StringIO.new(spec_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions = Marshal.load(reader.read)
-
-      buffer = StringIO.new(prerelease_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions.concat(Marshal.load(reader.read))
-
-      instance = new(versions)
-
-      reader.close
-
-      instance
-    end
 
     ##
-    # @param [Array] versions
+    # @param [String] versions
     #
-    def initialize(versions)
-      @versions      = versions
+    def initialize(versions_string)
+      unless versions_string.is_a? String
+        throw "#{versions_string.class} is wrong format, expect String; #{versions_string.inspect}"
+      end
+
+      @versions_string = versions_string
       @versions_hash = create_versions_hash
     end
 
@@ -53,10 +41,24 @@ module Gemirro
     def create_versions_hash
       hash = Hash.new { |h, k| h[k] = [] }
 
-      versions.each do |version|
-        hash[version[0]] << version
-      end
+      versions_string.each_line.with_index do |line, index|
+        next if index < 2
 
+        parts = line.split
+        gem_name = parts[0]
+        parts[-1]
+        versions = parts[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
+
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
+          hash[gem_name] << [gem_name, ::Gem::Version.new(version), platform]
+        end
+      end
       hash
     end
 

data/spec/gemirro/configuration_spec.rb

@@ -39,8 +39,8 @@ module Gemirro
       expect(Configuration.marshal_identifier).to match(/Marshal\.(\d+)\.(\d+)/)
     end
 
-    it 'should return versions file' do
-      expect(Configuration.versions_file).to match(/specs\.(\d+)\.(\d+).gz/)
+    it 'should error on version since no source is defined' do
+      expect(Gemirro.configuration.versions_file).to match(nil)
     end
 
     it 'should return marshal file' do
@@ -84,5 +84,12 @@ module Gemirro
       expect(result.host).to eq('https://rubygems.org')
       expect(result.name).to eq('rubygems')
     end
+
+
+    it 'should return versions file with source' do
+      result = @config.define_source('RubyGems', 'https://rubygems.org') do
+      end
+      expect(@config.versions_file).to match(/rubygems_org_versions$/)
+    end
   end
 end

data/spec/gemirro/gems_fetcher_spec.rb

@@ -12,7 +12,7 @@ module Gemirro
 
     before(:each) do
       @source = Source.new('RubyGems', 'https://rubygems.org')
-      @versions_file = VersionsFile.new(['0.0.1', '0.0.2'])
+      @versions_file = VersionsFile.new(%(created_at: 2025-04-24T03:46:59Z\n---\nrack 3.0.0,3.0.1 d545a45462d63b1b4865bbb89a109366))
       @fetcher = GemsFetcher.new(@source, @versions_file)
       Gemirro.configuration.ignored_gems.clear
     end

data/spec/gemirro/indexer_spec.rb

@@ -24,7 +24,7 @@ module Gemirro
 
       dir = MirrorDirectory.new('/tmp')
       dir.add_directory('test')
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
 
       Struct.new('HttpGet', :code, :body)
       http_get = Struct::HttpGet.new(200, 'bad')
@@ -43,7 +43,7 @@ module Gemirro
       allow(::Gem.configuration).to receive(:really_verbose)
         .once.and_return(true)
 
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
       indexer.quick_marshal_dir = '/tmp/gem_generate_index/quick/Marshal.4.8'
       indexer.dest_directory = '/tmp/test'
       indexer.directory = '/tmp/gem_generate_index'
@@ -97,7 +97,7 @@ module Gemirro
     end
 
     it 'should build indices' do
-      indexer = Indexer.new('/')
+      indexer = Indexer.new
       dir = MirrorDirectory.new('/')
       dir.add_directory('gems')
       dir.add_directory('quick')
@@ -140,7 +140,7 @@ module Gemirro
     end
 
     it 'should update index and exit ruby gems' do
-      indexer = Indexer.new('/')
+      indexer = Indexer.new
       MirrorDirectory.new('/')
       MirrorFile.new('/specs.4.8').write('')
       expect { indexer.update_index }.to raise_error(::Gem::SystemExitException)
@@ -152,7 +152,7 @@ module Gemirro
       dir.add_directory('test/gems')
       dir.add_directory('test/quick')
 
-      indexer = Indexer.new('/tmp/test')
+      indexer = Indexer.new
       indexer.quick_marshal_dir = '/tmp/gem_generate_index/quick/Marshal.4.8'
       indexer.dest_directory = '/tmp/test'
       indexer.directory = '/tmp/gem_generate_index'

data/spec/gemirro/server_spec.rb

@@ -1,6 +1,9 @@
 require 'rack/test'
 require 'json'
 require 'parallel'
+require 'sinatra/base'
+require 'thin'
+require 'base64'
 require 'gemirro/utils'
 require 'gemirro/mirror_directory'
 require 'gemirro/mirror_file'
@@ -37,7 +40,6 @@ module Gemirro
       Utils.instance_eval('@gems_source_collection = nil')
       FakeFS::FileSystem.clone(Gemirro::Configuration.views_directory)
       allow_any_instance_of(Indexer).to receive(:compress_indices)
-      allow_any_instance_of(Indexer).to receive(:compress_indicies)
       allow_any_instance_of(Indexer).to receive(:rand).and_return('0')
 
       source = Source.new('Rubygems', 'https://rubygems.org')
@@ -51,6 +53,9 @@ module Gemirro
           .and_return(nil)
           .and_yield(@fake_logger)
 
+
+        MirrorFile.new('/var/www/gemirro/versions.md5.aaa256.list').write('created_at: 2025-01-01T00:00:00Z\n---\nvolay 0.1.0\n')
+
         get '/'
         expect(last_response).to be_ok
       end
@@ -68,14 +73,7 @@ module Gemirro
       end
 
       it 'should display gem specifications' do
-        marshal_dump = Marshal.dump([['volay',
-                                      ::Gem::Version.create('0.1.0'),
-                                      'ruby']])
-
-        MirrorFile.new('/var/www/gemirro/specs.4.8.gz.local').write(Marshal.dump({}))
-
-
-        allow(Zlib::GzipReader).to receive(:open).and_return(double(read: marshal_dump))
+        MirrorFile.new('/var/www/gemirro/versions.md5.aaa256.list').write(%(created_at: 2025-01-01T00:00:00Z\n---\nvolay 0.1.0 checksum))
 
         get '/gem/volay'
 
@@ -84,48 +82,41 @@ module Gemirro
       end
       
       it 'responds to compact_index /names' do
-        MirrorFile.new('/var/www/gemirro/names.md5.sha256.list').write('---\n- volay\n')
+        MirrorFile.new('/var/www/gemirro/names.md5.aaa256.list').write(%(---\nvolay))
 
         get '/names'
         expect(last_response.status).to eq(200)
         expect(last_response).to be_ok
-        expect(last_response.body).to  eq('---\n- volay\n')
-        expect(last_response.headers['etag']).to eq("md5")
-        expect(last_response.headers['repr-digest']).to  eq('sha-256="sha256"')
+
+
+        expect(last_response.body).to  eq(%(---\nvolay))
+        expect(last_response.headers['etag']).to eq('"md5"')
+        expect(last_response.headers['repr-digest']).to  eq(%(sha-256=#{Base64.strict_encode64(['aaa256'].pack('H*'))}))
       end
 
       it 'responds to compact_index /info/[gemname]' do
-        marshal_dump = Marshal.dump([['volay',
-                                      ::Gem::Version.create('0.1.0'),
-                                      'ruby']])
-
-        MirrorFile.new('/var/www/gemirro/specs.4.8.gz.local').write(Marshal.dump({}))
-      
-        allow(Zlib::GzipReader).to receive(:open).and_return(double(read: marshal_dump))
-        
-        
         MirrorDirectory.new('/var/www/gemirro/info')
-        MirrorFile.new('/var/www/gemirro/info/volay.md5.sha256.list').write('---\n 0.1.0 |checksum:sha256\n')
+        MirrorFile.new('/var/www/gemirro/info/volay.md5.aaa256.list').write('---\n 0.1.0 |checksum:sha256\n')
         
 
         get '/info/volay'
         expect(last_response.status).to eq(200)
         expect(last_response).to be_ok
         expect(last_response.body).to eq('---\n 0.1.0 |checksum:sha256\n')
-        expect(last_response.headers['etag']).to eq("md5")
-        expect(last_response.headers['repr-digest']).to  eq('sha-256="sha256"')
+        expect(last_response.headers['etag']).to eq('"md5"')
+        expect(last_response.headers['repr-digest']).to  eq(%(sha-256=#{Base64.strict_encode64(['aaa256'].pack('H*'))}))
       end
       
 
       it 'responds to compact_index /versions' do
-        MirrorFile.new('/var/www/gemirro/versions.md5.sha256.list').write('created_at: 2025-01-01T00:00:00Z\m---\nvolay 0.1.0\n')
+        MirrorFile.new('/var/www/gemirro/versions.md5.aaa256.list').write(%(created_at: 2025-01-01T00:00:00Z\n---\nvolay 0.1.0))
       
         get '/versions'
         expect(last_response.status).to eq(200)
         expect(last_response).to be_ok
-        expect(last_response.body).to eq('created_at: 2025-01-01T00:00:00Z\m---\nvolay 0.1.0\n')
-        expect(last_response.headers['etag']).to eq("md5")
-        expect(last_response.headers['repr-digest']).to  eq('sha-256="sha256"')
+        expect(last_response.body).to eq(%(created_at: 2025-01-01T00:00:00Z\n---\nvolay 0.1.0))
+        expect(last_response.headers['etag']).to eq('"md5"')
+        expect(last_response.headers['repr-digest']).to  eq(%(sha-256=#{Base64.strict_encode64(['aaa256'].pack('H*'))}))
       end
       
 
@@ -214,31 +205,20 @@ module Gemirro
     end
 
     context 'dependencies' do
-      it 'should retrieve nothing' do
+      it 'should retrieve nothing and give 404' do
         get '/api/v1/dependencies'
-        expect(last_response.headers['Content-Type'])
-          .to eq('application/octet-stream')
-        expect(last_response.body).to eq('')
-        expect(last_response).to be_ok
+        expect(last_response.status).to eq(404)
+        expect(last_response).to_not be_ok
       end
 
-      it 'should retrieve empty json' do
-        get '/api/v1/dependencies.json'
-        expect(last_response.headers['Content-Type'])
-          .to eq('application/json')
-        expect(last_response.body).to eq('[]')
-        expect(last_response).to be_ok
-      end
 
-      it 'should retrieve empty json when gem was not found' do
+      it 'should retrieve nothing and give 404' do
         get '/api/v1/dependencies.json?gems=gemirro'
-        expect(last_response.headers['Content-Type'])
-          .to eq('application/json')
-        expect(last_response.body).to eq('[]')
-        expect(last_response).to be_ok
+        expect(last_response.status).to eq(404)
+        expect(last_response).to_not be_ok
       end
 
-      it 'should retrieve json when gem was found' do
+      it 'should retrieve nothing and give 404' do
         MirrorDirectory.new('/var/www/gemirro')
                        .add_directory('quick/Marshal.4.8')
         # rubocop:disable Metrics/LineLength
@@ -295,11 +275,8 @@ module Gemirro
         allow(Utils).to receive(:gems_collection)
           .and_return(collection)
         get '/api/v1/dependencies.json?gems=volay'
-        expect(last_response.headers['Content-Type'])
-          .to eq('application/json')
-
-        expect(last_response.body).to match(/"name":"volay"/)
-        expect(last_response).to be_ok
+        expect(last_response.status).to eq(404)
+        expect(last_response).to_not be_ok
       end
     end
   end

data/spec/gemirro/source_spec.rb

@@ -21,20 +21,11 @@ module Gemirro
       Struct.new('FetchVersions', :body)
       result = Struct::FetchVersions.new(true)
       allow(Http).to receive(:get).once.with(
-        "https://rubygems.org/#{Configuration.versions_file}"
+        "https://rubygems.org/versions"
       ).and_return(result)
       expect(@source.fetch_versions).to be_truthy
     end
 
-    it 'should fetch prereleases versions' do
-      Struct.new('FetchPrereleaseVersions', :body)
-      result = Struct::FetchPrereleaseVersions.new(true)
-      allow(Http).to receive(:get).once.with(
-        "https://rubygems.org/#{Configuration.prerelease_versions_file}"
-      ).and_return(result)
-      expect(@source.fetch_prerelease_versions).to be_truthy
-    end
-
     it 'should fetch gem' do
       Struct.new('FetchGem', :body)
       result = Struct::FetchGem.new(true)

data/spec/gemirro/versions_fetcher_spec.rb

@@ -17,10 +17,9 @@ module Gemirro
 
     it 'should fetch versions' do
       allow(@source).to receive(:fetch_versions).once.and_return([])
-      allow(@source).to receive(:fetch_prerelease_versions).once.and_return([])
-      allow(VersionsFile).to receive(:load).with('nothing', 'nothing')
-      allow(File).to receive(:write).twice
-      allow(File).to receive(:read).twice.and_return('nothing')
+      allow(VersionsFile).to receive(:load).with('nothing')
+      allow(File).to receive(:write).once
+      allow(File).to receive(:read).once.and_return('nothing')
       expect(@fetcher.fetch).to be_nil
     end
   end

data/spec/gemirro/versions_file_spec.rb

@@ -7,44 +7,20 @@ module Gemirro
   describe 'VersionsFile' do
     include FakeFS::SpecHelpers
 
-    it 'should be initialized' do
-      @versions_file = VersionsFile.new([['gemirro', '0.0.1'],
-                                         ['gemirro', '0.0.2']])
-      expect(@versions_file.versions).to eq([['gemirro', '0.0.1'],
-                                             ['gemirro', '0.0.2']])
-      expect(@versions_file.versions_hash).to eq(
-        'gemirro' => [
-          ['gemirro', '0.0.1'],
-          ['gemirro', '0.0.2']
-        ]
-      )
-    end
-
     it 'should load versions file' do
-      spec = StringIO.new('w')
-      w_gz = Zlib::GzipWriter.new(spec)
-      w_gz.write(Marshal.dump([['gemirro', '0.0.1'],
-                               ['gemirro', '0.0.2']]))
-      w_gz.close
-      prerelease = StringIO.new('w')
-      w_gz = Zlib::GzipWriter.new(prerelease)
-      w_gz.write(Marshal.dump([['gemirro', '0.0.1.alpha1'],
-                               ['gemirro', '0.0.2.alpha2']]))
-      w_gz.close
+      spec = %(created_at: 2025-01-01T00:00:00Z\n---\ngemirro 0.0.1.alpha1,0.0.1,0.0.2.alpha2,0.0.2 checksum)
 
-      result = VersionsFile.load(spec.string, prerelease.string)
+      result = VersionsFile.new(spec)
       expect(result).to be_a(VersionsFile)
 
-      expect(result.versions).to eq([['gemirro', '0.0.1'],
-                                     ['gemirro', '0.0.2'],
-                                     ['gemirro', '0.0.1.alpha1'],
-                                     ['gemirro', '0.0.2.alpha2']])
+      expect(result.versions_string).to eq(%(created_at: 2025-01-01T00:00:00Z\n---\ngemirro 0.0.1.alpha1,0.0.1,0.0.2.alpha2,0.0.2 checksum))
+
       expect(result.versions_hash).to eq(
         'gemirro' => [
-          ['gemirro', '0.0.1'],
-          ['gemirro', '0.0.2'],
-          ['gemirro', '0.0.1.alpha1'],
-          ['gemirro', '0.0.2.alpha2']
+          ['gemirro', ::Gem::Version.new('0.0.1.alpha1'), 'ruby'],
+          ['gemirro', ::Gem::Version.new('0.0.1'), 'ruby'],
+          ['gemirro', ::Gem::Version.new('0.0.2.alpha2'), 'ruby'],
+          ['gemirro', ::Gem::Version.new('0.0.2'), 'ruby']
         ]
       )
     end

data/template/config.rb

@@ -60,6 +60,6 @@ Gemirro.configuration.configure do
   # All gem in the block will be downloaded with the update command.
   # Other gems will be downloaded with the server.
   define_source 'rubygems', 'https://rubygems.org' do
-    gem 'rack', '>= 1.0.0'
+    gem 'rack', '>= 3.0.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gemirro
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Pierre Rambaud
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-04 00:00:00.000000000 Z
+date: 2025-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -290,9 +290,6 @@ files:
 - template/logs/.gitkeep
 - template/public/dist/css/gemirro.css
 - template/public/gems/.gitkeep
-- template/public/latest_specs.4.8
-- template/public/prerelease_specs.4.8
-- template/public/specs.4.8
 - views/gem.erb
 - views/index.erb
 - views/layout.erb