gemirro 0.10.5 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de7a99ed85512795f32fa3fd05a82216cc056e2d
-  data.tar.gz: b65ffbd28905ef09f0825f0888d4684ea2534c5d
+  metadata.gz: 369070871caf336cca42522e1ecf00a33b8146de
+  data.tar.gz: 45b5271ac12aab2d9db654ecf5ac75d61790f173
 SHA512:
-  metadata.gz: 65eea62ca9a8ccfa7175371faeacd2909e73c55f665c97479dc0b2d5ba960a1e9e3ea2e0917ac96db96ad939cbec86542cced6c8508e1c0146329e6bd57e3b09
-  data.tar.gz: f4422435079d1ad2c47bff8217ccbd6e7646386f3717ee66dcccdc653b47e2d25038b9b305bb765fa277a6bea410d3ec99b3e408ee7f04e3377996a21b1f7397
+  metadata.gz: 023bb1b397f46bd27afd2d777c2bc0a67f257ee9d32b1031a3bb6f883667d7c44b89ef74d4ce25121ee9a0e7868ff7a71a43985c043288b8a6d756db627d077e
+  data.tar.gz: 4aeb7d45c0d41f3ee2b2cbd0d8d47cba52ec659a4357b30e8fac170a48dedce7e755718b007d3997cf91ad8b1eccb4de7d509b436c7e0e18e6f63345edcf1e15

data/.gitignore

@@ -1,6 +1,7 @@
-.project
-coverage
-Gemfile.lock
-pkg
-.ruby-version
-*.gem
+/.project
+/coverage
+/Gemfile.lock
+/pkg
+/.ruby-version
+/*.gem
+/.bundle

data/.rubocop.yml

@@ -1,12 +1,16 @@
 AllCops:
   Include:
     - '**/Gemfile'
-    - lib/**/*
-    - spec/**/*
+    - lib/**/*.rb
+    - spec/**/*.rb
     - gemirro.gemspec
   Exclude:
     - files/**/*
     - templates/**/*
+ModuleLength:
+  Exclude:
+    - lib/**/*
+    - spec/**/*
 MethodLength:
   Exclude:
     - lib/**/*

data/MANIFEST

@@ -28,6 +28,8 @@ lib/gemirro/source.rb
 lib/gemirro/version.rb
 lib/gemirro/versions_fetcher.rb
 lib/gemirro/versions_file.rb
+spec/fixtures/gems/gemirro-0.0.1.gem
+spec/fixtures/quick/gemirro-0.0.1.gemspec.rz
 spec/gemirro/cli_spec.rb
 spec/gemirro/configuration_spec.rb
 spec/gemirro/gem_spec.rb

data/README.md

@@ -41,6 +41,7 @@ Once all the Gems have been downloaded you'll need to generate an index of all t
 
 ```bash
 $ gemirro index
+$ gemirro index --update # Or only update new files
 ```
 
 Last, launch the server, and all requests will check if gems are detected, and download them if necessary and generate index immediately.

data/gemirro.gemspec

@@ -4,13 +4,13 @@ require File.expand_path('../lib/gemirro/version', __FILE__)
 Gem::Specification.new do |s|
   s.name        = 'gemirro'
   s.version     = Gemirro::VERSION
-  s.date        = '2014-08-19'
+  s.date        = Date.today.to_s
   s.authors     = ['Pierre Rambaud']
   s.email       = 'pierre.rambaud86@gmail.com'
   s.license     = 'GPL-3.0'
-  s.summary     = 'Gem for easily creating your own RubyGems mirror.'
+  s.summary     = 'Gem for easily creating your own gems mirror.'
   s.homepage    = 'https://github.com/PierreRambaud/gemirro'
-  s.description = 'Create your own gem mirror with a simple TCPServer.'
+  s.description = 'Create your own gems mirror.'
   s.executables = ['gemirro']
 
   s.files = File.read(File.expand_path('../MANIFEST', __FILE__)).split("\n")

data/lib/gemirro.rb

@@ -11,6 +11,7 @@ require 'httpclient'
 require 'logger'
 require 'stringio'
 require 'json'
+require 'tempfile'
 
 unless $LOAD_PATH.include?(File.expand_path('../', __FILE__))
   $LOAD_PATH.unshift(File.expand_path('../', __FILE__))

data/lib/gemirro/cli/index.rb

@@ -5,6 +5,7 @@ Gemirro::CLI.options.command 'index' do
   separator "\nOptions:\n"
 
   on :c=, :config=, 'Path to the configuration file'
+  on :u, :update, 'Update only'
 
   run do |opts, _args|
     Gemirro::CLI.load_configuration(opts[:c])
@@ -19,6 +20,7 @@ Gemirro::CLI.options.command 'index' do
     indexer.ui = Gem::SilentUI.new
 
     config.logger.info('Generating indexes')
-    indexer.generate_index
+    indexer.generate_index if opts[:u].nil?
+    indexer.update_index unless opts[:u].nil?
   end
 end

data/lib/gemirro/configuration.rb

@@ -9,7 +9,9 @@ module Gemirro
       server: {
         access_log: '/tmp/gemirro.access.log',
         error_log: '/tmp/gemirro.access.log'
-      }
+      },
+
+      update_on_fetch: true
     }
     @configuration ||= Configuration.new(default_config)
   end
@@ -19,7 +21,7 @@ module Gemirro
   # destination directory, source, ignored Gems, etc.
   #
   class Configuration < Confstruct::Configuration
-    attr_reader :mirror_directory
+    attr_reader :mirror_gems_directory, :mirror_gemspecs_directory
     attr_accessor :source, :ignored_gems, :logger
 
     ##
@@ -77,6 +79,16 @@ module Gemirro
       "specs.#{marshal_version}.gz"
     end
 
+    ##
+    # Returns the name of the file that contains an index
+    # of all the prerelease versions.
+    #
+    # @return [String]
+    #
+    def self.prerelease_versions_file
+      "prerelease_specs.#{marshal_version}.gz"
+    end
+
     ##
     # Returns a String containing the Marshal version.
     #
@@ -91,8 +103,8 @@ module Gemirro
     #
     # @return [Gemirro::MirrorDirectory]
     #
-    def mirror_directory
-      @mirror_directory ||= MirrorDirectory.new(gems_directory)
+    def mirror_gems_directory
+      @mirror_gems_directory ||= MirrorDirectory.new(gems_directory)
     end
 
     ##
@@ -101,7 +113,25 @@ module Gemirro
     # @return [String]
     #
     def gems_directory
-      File.join(destination, 'gems')
+      File.join(destination.to_s, 'gems')
+    end
+
+    ##
+    # Return mirror directory
+    #
+    # @return [Gemirro::MirrorDirectory]
+    #
+    def mirror_gemspecs_directory
+      @mirror_gemspecs_directory ||= MirrorDirectory.new(gemspecs_directory)
+    end
+
+    ##
+    # Returns gems directory
+    #
+    # @return [String]
+    #
+    def gemspecs_directory
+      File.join(destination.to_s, 'quick', self.class.marshal_identifier)
     end
 
     ##

data/lib/gemirro/gem.rb

@@ -11,7 +11,8 @@ module Gemirro
   #  @return [Gem::Requirement]
   #
   class Gem
-    attr_reader :name, :requirement
+    attr_reader :name, :requirement, :platform
+    attr_accessor :gemspec
 
     ##
     # Returns a `Gem::Version` instance based on the specified requirement.
@@ -27,15 +28,16 @@ module Gemirro
     # @param [String] name
     # @param [Gem::Requirement|String] requirement
     #
-    def initialize(name, requirement = nil)
+    def initialize(name, requirement = nil, platform = 'ruby')
       requirement ||= ::Gem::Requirement.default
 
       if requirement.is_a?(String)
         requirement = ::Gem::Requirement.new(requirement)
       end
 
-      @name        = name
+      @name = name
       @requirement = requirement
+      @platform = platform
     end
 
     ##
@@ -56,15 +58,34 @@ module Gemirro
       version && !version.segments.reject { |s| s == 0 }.empty?
     end
 
+    def gemspec?
+      @gemspec == true
+    end
+
     ##
-    # Returns the filename of the Gemfile.
+    # Returns the filename of the gem file.
     #
     # @param [String] gem_version
     # @return [String]
     #
     def filename(gem_version = nil)
       gem_version ||= version.to_s
-      "#{name}-#{gem_version}.gem"
+      n = [name, gem_version]
+      n.push(@platform) if @platform != 'ruby'
+      "#{n.join('-')}.gem"
+    end
+
+    ##
+    # Returns the filename of the gemspec file.
+    #
+    # @param [String] gem_version
+    # @return [String]
+    #
+    def gemspec_filename(gem_version = nil)
+      gem_version ||= version.to_s
+      n = [name, gem_version]
+      n.push(@platform) if @platform != 'ruby'
+      "#{n.join('-')}.gemspec.rz"
     end
   end
 end

data/lib/gemirro/gem_version_collection.rb

@@ -72,6 +72,11 @@ module Gemirro
         @grouped = @gems.group_by(&:name).map do |name, collection|
           [name, GemVersionCollection.new(collection)]
         end
+
+        @grouped.reject! do |name, _collection|
+          name.nil?
+        end
+
         @grouped.sort_by! do |name, _collection|
           name.downcase
         end

data/lib/gemirro/gems_fetcher.rb

@@ -27,19 +27,15 @@ module Gemirro
     def fetch
       @source.gems.each do |gem|
         versions_for(gem).each do |version|
-          filename  = gem.filename(version)
-          satisfied = gem.requirement.satisfied_by?(version)
-          name      = gem.name
-
-          if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
-            logger.debug("Skipping #{filename}")
-            next
+          if gem.gemspec?
+            gemfile = fetch_gemspec(gem, version)
+            configuration.mirror_gemspecs_directory
+              .add_file(gem.gemspec_filename(version), gemfile) if gemfile
+          else
+            gemfile = fetch_gem(gem, version)
+            configuration.mirror_gems_directory
+              .add_file(gem.filename(version), gemfile) if gemfile
           end
-
-          configuration.ignore_gem(gem.name, version)
-          logger.info("Fetching #{filename}")
-          gemfile = fetch_gem(gem, version)
-          configuration.mirror_directory.add_file(filename, gemfile) if gemfile
         end
       end
     end
@@ -69,19 +65,62 @@ module Gemirro
     end
 
     ##
-    # Tries to download the Gemfile for the specified Gem and version.
+    # Tries to download gemspec from a given name and version
+    #
+    # @param [Gemirro::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_gemspec(gem, version)
+      filename  = gem.gemspec_filename(version)
+      satisfied = gem.requirement.satisfied_by?(version)
+
+      if gemspec_exists?(filename) || !satisfied
+        logger.debug("Skipping #{filename}")
+        return
+      end
+
+      logger.info("Fetching #{filename}")
+      fetch_from_source(gem, version, true)
+    end
+
+    ##
+    # Tries to download the gem file from a given nam and version
     #
     # @param [Gemirro::Gem] gem
     # @param [Gem::Version] version
     # @return [String]
     #
     def fetch_gem(gem, version)
-      data  = nil
       filename = gem.filename(version)
+      satisfied = gem.requirement.satisfied_by?(version)
+      name = gem.name
+
+      if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
+        logger.debug("Skipping #{filename}")
+        return
+      end
+
+      configuration.ignore_gem(gem.name, version)
+      logger.info("Fetching #{filename}")
 
+      fetch_from_source(gem, version)
+    end
+
+    ##
+    #
+    #
+    # @param [Gemirro::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_from_source(gem, version, gemspec = false)
+      data  = nil
       begin
-        data = @source.fetch_gem(gem.name, version)
+        data = @source.fetch_gem(gem.name, version) unless gemspec
+        data = @source.fetch_gemspec(gem.name, version) if gemspec
       rescue => e
+        filename = gem.filename(version)
         logger.error("Failed to retrieve #{filename}: #{e.message}")
         logger.debug("Adding #{filename} to the list of ignored Gems")
 
@@ -113,7 +152,17 @@ module Gemirro
     # @return [TrueClass|FalseClass]
     #
     def gem_exists?(filename)
-      configuration.mirror_directory.file_exists?(filename)
+      configuration.mirror_gems_directory.file_exists?(filename)
+    end
+
+    ##
+    # Checks if a given Gemspec has already been downloaded.
+    #
+    # @param [String] filename
+    # @return [TrueClass|FalseClass]
+    #
+    def gemspec_exists?(filename)
+      configuration.mirror_gemspecs_directory.file_exists?(filename)
     end
 
     ##

data/lib/gemirro/indexer.rb

@@ -82,8 +82,7 @@ module Gemirro
     # @return [Array]
     #
     def install_indicies
-      verbose = ::Gem.configuration.really_verbose
-      Gemirro.configuration.logger
+      logger
         .debug("Downloading index into production dir #{@dest_directory}")
 
       files = @files
@@ -100,29 +99,14 @@ module Gemirro
 
       files.each do |path|
         file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
-        dst_name = File.join @dest_directory, file
+        src_name = File.join(@directory, file)
+        dst_name = File.join(@dest_directory, file)
 
         if ["#{@specs_index}.gz",
             "#{@latest_specs_index}.gz",
             "#{@prerelease_specs_index}.gz"].include?(path)
-
-          content = Marshal.load(Zlib::GzipReader.open(path).read)
-          Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
-            io.write(Marshal.dump(content))
-          end
-
-          unless @only_origin
-            source_content = download_from_source(file)
-            next if source_content.nil?
-            source_content = Marshal.load(Zlib::GzipReader
-                                            .new(StringIO
-                                                   .new(source_content)).read)
-            new_content = source_content.concat(content).uniq
-
-            Zlib::GzipWriter.open(dst_name) do |io|
-              io.write(Marshal.dump(new_content))
-            end
-          end
+          res = build_zlib_file(file, src_name, dst_name, true)
+          next unless res
         else
           source_content = download_from_source(file)
           next if source_content.nil?
@@ -133,21 +117,189 @@ module Gemirro
       end
     end
 
+    ##
+    # Download file from source
+    #
+    # @param [String] file File path
+    # @return [String]
+    #
     def download_from_source(file)
       source_host = Gemirro.configuration.source.host
+      logger.info("Download from source: #{file}")
       resp = Http.get("#{source_host}/#{File.basename(file)}")
       return unless resp.code == 200
       resp.body
     end
 
+    ##
+    # Build indicies
+    #
+    # @return [Array]
+    #
     def build_indicies
+      specs = *map_gems_to_specs(gem_file_list)
+      specs.reject! { |s| s.class != ::Gem::Specification }
       ::Gem::Specification.dirs = []
-      ::Gem::Specification.all = *map_gems_to_specs(gem_file_list)
+      ::Gem::Specification.all = specs
 
       build_marshal_gemspecs
       build_modern_indicies if @build_modern
 
       compress_indicies
     end
+
+    ##
+    # Map gems file to specs
+    #
+    # @param [Array] gems Gems list
+    # @return [Array]
+    #
+    def map_gems_to_specs(gems)
+      gems.map.with_index do |gemfile, index|
+        # rubocop:disable Metrics/LineLength
+        logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
+        # rubocop:enable Metrics/LineLength
+
+        if File.size(gemfile) == 0
+          logger.warn("Skipping zero-length gem: #{gemfile}")
+          next
+        end
+
+        begin
+          spec = ::Gem::Package.new(gemfile).spec
+          spec.loaded_from = gemfile
+
+          # HACK: fuck this shit - borks all tests that use pl1
+          if File.basename(gemfile, '.gem') != spec.original_name
+            exp = spec.full_name
+            exp << " (#{spec.original_name})" if
+              spec.original_name != spec.full_name
+            msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
+            logger.warn(msg)
+            next
+          end
+
+          abbreviate spec
+          sanitize spec
+
+          spec
+        rescue SignalException
+          msg = 'Received signal, exiting'
+          logger.error(msg)
+          raise
+        rescue StandardError => e
+          msg = ["Unable to process #{gemfile}",
+                 "#{e.message} (#{e.class})",
+                 "\t#{e.backtrace.join "\n\t"}"].join("\n")
+          logger.debug(msg)
+        end
+      end.compact
+    end
+
+    def update_index
+      make_temp_directories
+
+      specs_mtime = File.stat(@dest_specs_index).mtime
+      newest_mtime = Time.at(0)
+
+      updated_gems = gem_file_list.select do |gem|
+        gem_mtime = File.stat(gem).mtime
+        newest_mtime = gem_mtime if gem_mtime > newest_mtime
+        gem_mtime > specs_mtime
+      end
+
+      if updated_gems.empty?
+        logger.info('No new gems')
+        terminate_interaction(0)
+      end
+
+      specs = map_gems_to_specs(updated_gems)
+      prerelease, released = specs.partition { |s| s.version.prerelease? }
+
+      ::Gem::Specification.dirs = []
+      ::Gem::Specification.all = *specs
+      files = build_marshal_gemspecs
+
+      ::Gem.time('Updated indexes') do
+        update_specs_index(released, @dest_specs_index, @specs_index)
+        update_specs_index(released,
+                           @dest_latest_specs_index,
+                           @latest_specs_index)
+        update_specs_index(prerelease,
+                           @dest_prerelease_specs_index,
+                           @prerelease_specs_index)
+      end
+
+      compress_indicies
+
+      logger.info("Updating production dir #{@dest_directory}") if verbose
+      files << @specs_index
+      files << "#{@specs_index}.gz"
+      files << @latest_specs_index
+      files << "#{@latest_specs_index}.gz"
+      files << @prerelease_specs_index
+      files << "#{@prerelease_specs_index}.gz"
+
+      files.each do |path|
+        file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
+        src_name = File.join(@directory, file)
+        dst_name = File.join(@dest_directory, file)
+
+        if ["#{@specs_index}.gz",
+            "#{@latest_specs_index}.gz",
+            "#{@prerelease_specs_index}.gz"].include?(path)
+          res = build_zlib_file(file, src_name, dst_name)
+          next unless res
+        else
+          FileUtils.mv(src_name,
+                       dst_name,
+                       verbose: verbose,
+                       force: true)
+        end
+
+        File.utime(newest_mtime, newest_mtime, dst_name)
+      end
+    end
+
+    def build_zlib_file(file, src_name, dst_name, from_source = false)
+      content = Marshal.load(Zlib::GzipReader.open(src_name).read)
+      create_zlib_file("#{dst_name}.orig", content)
+
+      return false if @only_origin
+
+      if from_source
+        source_content = download_from_source(file)
+        source_content = Marshal.load(Zlib::GzipReader
+                                        .new(StringIO
+                                               .new(source_content)).read)
+      else
+        source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
+      end
+
+      return false if source_content.nil?
+      new_content = source_content.concat(content).uniq
+      create_zlib_file(dst_name, new_content)
+    end
+
+    def create_zlib_file(dst_name, content)
+      temp_file = Tempfile.new('gemirro')
+
+      Zlib::GzipWriter.open(temp_file.path) do |io|
+        io.write(Marshal.dump(content))
+      end
+
+      FileUtils.mv(temp_file.path,
+                   dst_name,
+                   verbose: verbose,
+                   force: true)
+    end
+
+    def verbose
+      @verbose ||= ::Gem.configuration.really_verbose
+    end
+
+    def logger
+      Gemirro.configuration.logger
+    end
   end
 end

data/lib/gemirro/server.rb

@@ -25,8 +25,8 @@ module Gemirro
     error_logger.sync = true
 
     before do
-      Gemirro.configuration.logger = access_logger
       env['rack.errors'] = error_logger
+      Gemirro.configuration.logger = access_logger
     end
 
     ##
@@ -130,26 +130,31 @@ module Gemirro
     def fetch_gem(resource)
       name = File.basename(resource)
       # rubocop:disable Metrics/LineLength
-      regexp = /^(.*)-(\d+(?:\.\d+){2,4}.*?)(?:-x86-(?:(?:mswin|mingw)(?:32|64)).*?)?\.gem(?:spec\.rz)?$/
+      regexp = /^(.*)-(\d+(?:\.\d+){2,4}.*?)(?:-x86-(?:(?:mswin|mingw)(?:32|64)).*?)?\.(gem(?:spec\.rz)?)$/
       # rubocop:enable Metrics/LineLength
       result = name.match(regexp)
       return unless result
 
-      gem_name, gem_version = result.captures
+      gem_name, gem_version, gem_type = result.captures
       return unless gem_name && gem_version
 
       begin
         gem = Gemirro::Gem.new(gem_name, gem_version)
-        return if gems_fetcher.gem_exists?(gem.filename(gem_version))
+        gem.gemspec = true if gem_type == 'gemspec.rz'
+
+        # rubocop:disable Metrics/LineLength
+        return if gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == 'gem'
+        return if gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == 'gemspec.rz'
+        # rubocop:enable Metrics/LineLength
 
         logger.info("Try to download #{gem_name} with version #{gem_version}")
         gems_fetcher.source.gems.clear
         gems_fetcher.source.gems.push(gem)
         gems_fetcher.fetch
 
-        update_indexes
+        update_indexes if configuration.update_on_fetch
       rescue StandardError => e
-        logger.error(e.message)
+        logger.error(e)
       end
     end
 
@@ -164,7 +169,7 @@ module Gemirro
       indexer.ui = ::Gem::SilentUI.new
 
       configuration.logger.info('Generating indexes')
-      indexer.generate_index
+      indexer.update_index
     rescue SystemExit => e
       configuration.logger.info(e.message)
     end
@@ -314,11 +319,10 @@ module Gemirro
       # @return [::Gem::Specification]
       #
       def spec_for(gemname, version, platform = 'ruby')
-        filename = [gemname, version]
-        filename.push(platform) if platform != 'ruby'
+        gem = Gem.new(gemname, version.to_s, platform)
         gemspec_path = File.join('quick',
                                  Gemirro::Configuration.marshal_identifier,
-                                 "#{filename.join('-')}.gemspec.rz")
+                                 gem.gemspec_filename)
         spec_file = File.join(settings.public_folder,
                               gemspec_path)
 

data/lib/gemirro/source.rb

@@ -34,6 +34,15 @@ module Gemirro
       Http.get(host + '/' + Configuration.versions_file).body
     end
 
+    ##
+    # Fetches a list of all the available Gems and their versions.
+    #
+    # @return [String]
+    #
+    def fetch_prerelease_versions
+      Http.get(host + '/' + Configuration.prerelease_versions_file).body
+    end
+
     ##
     # Fetches the `.gem` file of a given Gem and version.
     #
@@ -45,6 +54,18 @@ module Gemirro
       Http.get(host + "/gems/#{name}-#{version}.gem").body
     end
 
+    ##
+    # Fetches the `.gemspec.rz` file of a given Gem and version.
+    #
+    # @param [String] name
+    # @param [String] version
+    # @return [String]
+    #
+    def fetch_gemspec(name, version)
+      marshal = Gemirro::Configuration.marshal_identifier
+      Http.get(host + "/quick/#{marshal}/#{name}-#{version}.gemspec.rz").body
+    end
+
     ##
     # Adds a new Gem to the source.
     #

data/lib/gemirro/version.rb

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 # Gemirro Version
 module Gemirro
-  VERSION = '0.10.5'
+  VERSION = '0.11.0'
 end

data/lib/gemirro/versions_fetcher.rb

@@ -25,7 +25,7 @@ module Gemirro
         "Updating #{source.name} (#{source.host})"
       )
 
-      VersionsFile.load(source.fetch_versions)
+      VersionsFile.load(source.fetch_versions, source.fetch_prerelease_versions)
     end
   end
 end

data/lib/gemirro/versions_file.rb

@@ -15,13 +15,20 @@ module Gemirro
     ##
     # Reads the versions file from the specified String.
     #
-    # @param [String] content
+    # @param [String] spec_content
+    # @param [String] prerelease_content
     # @return [Gemirro::VersionsFile]
     #
-    def self.load(content)
-      buffer   = StringIO.new(content)
-      reader   = Zlib::GzipReader.new(buffer)
-      instance = new(Marshal.load(reader.read))
+    def self.load(spec_content, prerelease_content)
+      buffer = StringIO.new(spec_content)
+      reader = Zlib::GzipReader.new(buffer)
+      versions = Marshal.load(reader.read)
+
+      buffer = StringIO.new(prerelease_content)
+      reader = Zlib::GzipReader.new(buffer)
+      versions.concat(Marshal.load(reader.read))
+
+      instance = new(versions)
 
       reader.close
 

data/spec/gemirro/configuration_spec.rb

@@ -58,8 +58,8 @@ module Gemirro
 
     it 'return mirror directory' do
       allow(@config).to receive(:gems_directory).once.and_return('/tmp')
-      expect(@config.mirror_directory).to be_a(MirrorDirectory)
-      expect(@config.mirror_directory.path).to eq('/tmp')
+      expect(@config.mirror_gems_directory).to be_a(MirrorDirectory)
+      expect(@config.mirror_gems_directory.path).to eq('/tmp')
     end
 
     it 'should return gems directory' do

data/spec/gemirro/gems_fetcher_spec.rb

@@ -13,8 +13,9 @@ module Gemirro
 
     before(:each) do
       @source = Source.new('RubyGems', 'https://rubygems.org')
-      @versions_file = VersionsFile.new(['0.0.1'])
+      @versions_file = VersionsFile.new(['0.0.1', '0.0.2'])
       @fetcher = GemsFetcher.new(@source, @versions_file)
+      Gemirro.configuration.ignored_gems.clear
     end
 
     it 'should be initialized' do
@@ -34,19 +35,25 @@ module Gemirro
       @fetcher.configuration.destination = './'
       expect(@fetcher.gem_exists?('test')).to be_falsy
       MirrorDirectory.new('./').add_directory('gems')
+      MirrorDirectory.new('./').add_directory('quick/Marshal.4.8')
       MirrorFile.new('gems/test').write('content')
       expect(@fetcher.gem_exists?('test')).to be_truthy
     end
 
     it 'should ignore gem' do
+      allow(@fetcher.logger).to receive(:info)
+        .once.with('Fetching gemirro-0.0.1.gem')
       expect(@fetcher.ignore_gem?('gemirro', '0.0.1')).to be_falsy
       @fetcher.configuration.ignore_gem('gemirro', '0.0.1')
       expect(@fetcher.ignore_gem?('gemirro', '0.0.1')).to be_truthy
     end
 
     it 'should log error when fetch gem failed' do
+      allow(@fetcher.logger).to receive(:info)
+        .once.with('Fetching gemirro-0.0.1.gem')
       gem = Gem.new('gemirro')
       version = ::Gem::Version.new('0.0.1')
+      @fetcher.configuration.ignore_gem('gemirro', '0.0.1')
       allow(@source).to receive(:fetch_gem)
         .once.with('gemirro', version).and_raise(ArgumentError)
       allow(@fetcher.logger).to receive(:error)
@@ -59,6 +66,9 @@ module Gemirro
     end
 
     it 'should fetch gem' do
+      allow(@fetcher.logger).to receive(:info)
+        .once.with('Fetching gemirro-0.0.1.gem')
+      MirrorDirectory.new('./').add_directory('gems')
       gem = Gem.new('gemirro')
       version = ::Gem::Version.new('0.0.1')
       allow(@source).to receive(:fetch_gem)
@@ -67,6 +77,35 @@ module Gemirro
       expect(@fetcher.fetch_gem(gem, version)).to eq('gemirro')
     end
 
+    it 'should fetch gemspec' do
+      allow(@fetcher.logger).to receive(:info)
+        .once.with('Fetching gemirro-0.0.1.gemspec.rz')
+      MirrorDirectory.new('./').add_directory('quick/Marshal.4.8')
+      gem = Gem.new('gemirro')
+      gem.gemspec = true
+      version = ::Gem::Version.new('0.0.1')
+      allow(@source).to receive(:fetch_gemspec)
+        .once.with('gemirro', version).and_return('gemirro')
+
+      expect(@fetcher.fetch_gemspec(gem, version)).to eq('gemirro')
+    end
+
+    it 'should not fetch gemspec if file exists' do
+      allow(@fetcher.logger).to receive(:info)
+        .once.with('Fetching gemirro-0.0.1.gemspec.rz')
+      allow(@fetcher).to receive(:gemspec_exists?)
+        .once.with('gemirro-0.0.1.gemspec.rz')
+        .and_return(true)
+      allow(@fetcher.logger).to receive(:debug)
+        .once.with('Skipping gemirro-0.0.1.gemspec.rz')
+
+      gem = Gem.new('gemirro')
+      gem.gemspec = true
+      version = ::Gem::Version.new('0.0.1')
+
+      expect(@fetcher.fetch_gemspec(gem, version)).to be_nil
+    end
+
     it 'should retrieve versions for specific gem' do
       gem = Gem.new('gemirro', '0.0.2')
       allow(@versions_file).to receive(:versions_for)
@@ -75,6 +114,7 @@ module Gemirro
     end
 
     it 'should fetch all gems and log debug if gem is not satisfied' do
+      MirrorDirectory.new('./').add_directory('gems')
       gem = Gem.new('gemirro', '0.0.1')
       allow(gem.requirement).to receive(:satisfied_by?)
         .once.with(nil).and_return(false)
@@ -87,18 +127,21 @@ module Gemirro
     it 'should fetch all gems' do
       gem = Gem.new('gemirro', '0.0.2')
       @fetcher.source.gems << gem
-      allow(@fetcher).to receive(:versions_for).once.and_return(['0.0.2'])
-      allow(gem.requirement).to receive(:satisfied_by?)
-        .once.with('0.0.2').and_return(true)
+      gemspec = Gem.new('gemirro', '0.0.1')
+      gemspec.gemspec = true
+      @fetcher.source.gems << gemspec
+
+      allow(@fetcher).to receive(:fetch_gemspec)
+        .once.with(gemspec, nil).and_return('gemfile')
       allow(@fetcher).to receive(:fetch_gem)
-        .once.with(gem, '0.0.2').and_return('gemfile')
-      allow(@fetcher.configuration).to receive(:ignore_gem)
-        .once.with('gemirro', '0.0.2')
-      allow(@fetcher.logger).to receive(:info)
-        .once.with('Fetching gemirro-0.0.2.gem')
-      allow(@fetcher.configuration.mirror_directory).to receive(:add_file)
+        .once.with(gem, nil).and_return('gemfile')
+
+      allow(@fetcher.configuration.mirror_gems_directory).to receive(:add_file)
         .once.with('gemirro-0.0.2.gem', 'gemfile')
-      expect(@fetcher.fetch).to eq([gem])
+      allow(@fetcher.configuration.mirror_gemspecs_directory)
+        .to receive(:add_file)
+        .once.with('gemirro-0.0.1.gemspec.rz', 'gemfile')
+      expect(@fetcher.fetch).to eq([gem, gemspec])
     end
   end
 end

data/spec/gemirro/indexer_spec.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 require 'spec_helper'
 require 'rubygems/indexer'
+require 'tempfile'
 require 'gemirro/source'
 require 'gemirro/indexer'
 require 'gemirro/mirror_file'
@@ -11,6 +12,28 @@ module Gemirro
   describe 'Indexer' do
     include FakeFS::SpecHelpers
 
+    before(:each) do
+      allow_any_instance_of(Logger).to receive(:info)
+      allow_any_instance_of(Logger).to receive(:debug)
+      allow_any_instance_of(Logger).to receive(:warn)
+    end
+
+    it 'should download from source' do
+      source = Source.new('Rubygems', 'https://rubygems.org')
+      allow(Gemirro.configuration).to receive(:source).and_return(source)
+
+      dir = MirrorDirectory.new('/tmp')
+      dir.add_directory('test')
+      indexer = Indexer.new('/tmp/test')
+
+      Struct.new('HttpGet', :code, :body)
+      http_get = Struct::HttpGet.new(200, 'bad')
+      allow(Http).to receive(:get).once.and_return(http_get)
+
+      expect(indexer.download_from_source('something'))
+        .to eq('bad')
+    end
+
     it 'should install indicies' do
       dir = MirrorDirectory.new('/tmp')
       dir.add_directory('test')
@@ -44,25 +67,19 @@ module Gemirro
               '/tmp/test/quick/Marshal.4.8',
               verbose: true, force: true)
 
+      allow(FileUtils).to receive(:mv)
+
       source = Source.new('Rubygems', 'https://rubygems.org')
       allow(Gemirro.configuration).to receive(:source).and_return(source)
 
-      Struct.new('HttpGet', :code, :body)
       wio = StringIO.new('w')
       w_gz = Zlib::GzipWriter.new(wio)
       w_gz.write(['content'])
       w_gz.close
-      http_get = Struct::HttpGet.new(200, wio.string)
+      allow(indexer).to receive(:download_from_source).and_return(wio.string)
+
       allow(Marshal).to receive(:load).and_return(['content'])
       allow(Marshal).to receive(:dump).and_return(['content'])
-      allow(Zlib::GzipWriter).to receive(:open)
-        .once.with('/tmp/test/specs.4.8.gz.orig')
-      allow(Zlib::GzipWriter).to receive(:open)
-        .once.with('/tmp/test/specs.4.8.gz')
-      allow(Http).to receive(:get)
-        .with('https://rubygems.org/specs.4.8.gz').and_return(http_get)
-      allow(Http).to receive(:get)
-        .with('https://rubygems.org/something.4.8.gz').and_return(http_get)
 
       Struct.new('GzipReader', :read)
       gzip_reader = Struct::GzipReader.new(wio.string)
@@ -76,15 +93,6 @@ module Gemirro
                            '/tmp/gem_generate_index/something.4.8.gz'])
     end
 
-    it 'should exit if there is no new gems' do
-      dir = MirrorDirectory.new('./')
-      dir.add_directory('gem_generate_index/gems')
-      MirrorFile.new('./specs.4.8').write('')
-
-      indexer = Indexer.new('./')
-      allow(indexer).to receive(:make_temp_directories).and_return(true)
-    end
-
     it 'should build indicies' do
       indexer = Indexer.new('/')
       dir = MirrorDirectory.new('/')
@@ -94,19 +102,82 @@ module Gemirro
       dir.add_directory("#{indexer.directory.gsub(%r{^/}, '')}/gems")
       dir.add_directory("#{indexer.directory.gsub(%r{^/}, '')}/quick")
 
+      fixtures_dir = File.dirname(__FILE__) + '/../fixtures'
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/gems/gemirro-0.0.1.gem",
+               "#{indexer.directory}/gems/gemirro-0.0.1.gem")
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/gems/gemirro-0.0.1.gem",
+               '/gems/gemirro-0.0.1.gem')
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/gems/gemirro-0.0.1.gem",
+               '/gems/gemirral-0.0.1.gem') # Skipping misnamed
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/quick/gemirro-0.0.1.gemspec.rz",
+               "#{indexer.directory}/quick/gemirro-0.0.1.gemspec.rz")
+
+      MirrorFile.new('gems/gemirro-0.0.2.gem').write('') # Empty file
+      MirrorFile.new('gems/gemirro-0.0.3.gem').write('Error') # Empty file
       MirrorFile.new('/specs.4.8').write('')
-      MirrorFile.new("#{indexer.directory}/gems/gemirro-0.1.0.gem").write('')
-      MirrorFile.new('gems/gemirro-0.1.0.gem').write('')
-      MirrorFile.new("#{indexer.directory}/quick/gemirro-0.1.0.gemspec.rz")
-        .write('test')
 
       allow(indexer).to receive(:gem_file_list)
-        .and_return(['gems/gemirro-0.1.0.gem'])
+        .and_return(['gems/gemirro-0.0.1.gem',
+                     'gems/gemirro-0.0.2.gem',
+                     'gems/gemirro-0.0.3.gem',
+                     'gems/gemirral-0.0.1.gem'])
       allow(indexer).to receive(:build_marshal_gemspecs).once.and_return([
-        "#{indexer.directory}/quick/gemirro-0.1.0.gemspec.rz"])
+        "#{indexer.directory}/quick/gemirro-0.0.1.gemspec.rz"])
       allow(indexer).to receive(:compress_indicies).once.and_return(true)
 
       indexer.build_indicies
     end
+
+    it 'should update index and exit ruby gems' do
+      indexer = Indexer.new('/')
+      MirrorDirectory.new('/')
+      MirrorFile.new('/specs.4.8').write('')
+      expect { indexer.update_index }.to raise_error(::Gem::SystemExitException)
+    end
+
+    it 'should update index' do
+      dir = MirrorDirectory.new('/tmp')
+      dir.add_directory('gem_generate_index/quick/Marshal.4.8')
+      dir.add_directory('test/gems')
+      dir.add_directory('test/quick')
+
+      indexer = Indexer.new('/tmp/test')
+      indexer.quick_marshal_dir = '/tmp/gem_generate_index/quick/Marshal.4.8'
+      indexer.dest_directory = '/tmp/test'
+      indexer.directory = '/tmp/gem_generate_index'
+      indexer.instance_variable_set('@specs_index',
+                                    '/tmp/gem_generate_index/specs.4.8')
+
+      MirrorFile.new("#{indexer.directory}/specs.4.8.gz").write('')
+      MirrorFile.new("#{indexer.directory}/specs.4.8").write('')
+      MirrorFile.new("#{indexer.dest_directory}/specs.4.8").write('')
+      File.utime(Time.at(0), Time.at(0), "#{indexer.dest_directory}/specs.4.8")
+
+      fixtures_dir = File.dirname(__FILE__) + '/../fixtures'
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/gems/gemirro-0.0.1.gem",
+               "#{indexer.dest_directory}/gems/gemirro-0.0.1.gem")
+      FakeFS::FileSystem
+        .clone("#{fixtures_dir}/quick/gemirro-0.0.1.gemspec.rz",
+               "#{indexer.directory}/quick/gemirro-0.0.1.gemspec.rz")
+
+      allow(indexer).to receive(:make_temp_directories)
+      allow(indexer).to receive(:update_specs_index)
+      allow(indexer).to receive(:compress_indicies)
+      allow(indexer).to receive(:build_zlib_file)
+      allow(indexer).to receive(:build_marshal_gemspecs).once.and_return([
+        "#{indexer.directory}/quick/gemirro-0.0.1.gemspec.rz"])
+
+      allow(Marshal).to receive(:load).and_return(['content'])
+      allow(Marshal).to receive(:dump).and_return(['content'])
+      allow(FileUtils).to receive(:mv)
+      allow(File).to receive(:utime)
+
+      indexer.update_index
+    end
   end
 end

data/spec/gemirro/server_spec.rb

@@ -21,7 +21,6 @@ RSpec.configure do |c|
 end
 
 # Server tests
-# rubocop:disable Metrics/ModuleLength
 module Gemirro
   describe 'Gemirro::Server' do
     include FakeFS::SpecHelpers
@@ -131,7 +130,7 @@ module Gemirro
         gem_indexer = Struct::GemIndexer.new
         allow(gem_indexer).to receive(:only_origin=).once.and_return(true)
         allow(gem_indexer).to receive(:ui=).once.and_return(true)
-        allow(gem_indexer).to receive(:generate_index).once.and_return(true)
+        allow(gem_indexer).to receive(:update_index).once.and_return(true)
 
         allow(Gemirro.configuration).to receive(:source)
           .twice.and_return(source)
@@ -171,7 +170,7 @@ module Gemirro
         gem_indexer = Struct::GemIndexer.new
         allow(gem_indexer).to receive(:only_origin=).once.and_return(true)
         allow(gem_indexer).to receive(:ui=).once.and_return(true)
-        allow(gem_indexer).to receive(:generate_index)
+        allow(gem_indexer).to receive(:update_index)
           .once.and_raise(SystemExit)
 
         allow(Gemirro.configuration).to receive(:source)

data/spec/gemirro/source_spec.rb

@@ -33,6 +33,14 @@ module Gemirro
       expect(@source.fetch_gem('gemirro', '0.0.1')).to be_truthy
     end
 
+    it 'should fetch gemspec' do
+      Struct.new('FetchGemspec', :body)
+      result = Struct::FetchGemspec.new(true)
+      allow(Http).to receive(:get).once.with(
+        'https://rubygems.org/quick/Marshal.4.8/gemirro-0.0.1.gemspec.rz').and_return(result)
+      expect(@source.fetch_gemspec('gemirro', '0.0.1')).to be_truthy
+    end
+
     it 'should add gems' do
       expect(@source.gems).to eq([])
       @source.gem('gemirro')

data/spec/gemirro/versions_fetcher_spec.rb

@@ -18,7 +18,8 @@ module Gemirro
       allow(Gemirro.configuration.logger).to receive(:info)
         .once.with("Updating #{@source.name} (#{@source.host})")
       allow(@source).to receive(:fetch_versions).once.and_return([])
-      allow(VersionsFile).to receive(:load).with([])
+      allow(@source).to receive(:fetch_prerelease_versions).once.and_return([])
+      allow(VersionsFile).to receive(:load).with([], [])
       expect(@fetcher.fetch).to be_nil
     end
   end

data/spec/gemirro/versions_file_spec.rb

@@ -26,25 +26,36 @@ module Gemirro
     end
 
     it 'should load versions file' do
-      wio = StringIO.new('w')
-      w_gz = Zlib::GzipWriter.new(wio)
+      spec = StringIO.new('w')
+      w_gz = Zlib::GzipWriter.new(spec)
       w_gz.write(Marshal.dump([
         ['gemirro', '0.0.1'],
         ['gemirro', '0.0.2']
       ]))
       w_gz.close
+      prerelease = StringIO.new('w')
+      w_gz = Zlib::GzipWriter.new(prerelease)
+      w_gz.write(Marshal.dump([
+        ['gemirro', '0.0.1.alpha1'],
+        ['gemirro', '0.0.2.alpha2']
+      ]))
+      w_gz.close
 
-      result = VersionsFile.load(wio.string)
+      result = VersionsFile.load(spec.string, prerelease.string)
       expect(result).to be_a(VersionsFile)
 
       expect(result.versions).to eq([
         ['gemirro', '0.0.1'],
-        ['gemirro', '0.0.2']
+        ['gemirro', '0.0.2'],
+        ['gemirro', '0.0.1.alpha1'],
+        ['gemirro', '0.0.2.alpha2']
       ])
       expect(result.versions_hash).to eq(
         'gemirro' => [
           ['gemirro', '0.0.1'],
-          ['gemirro', '0.0.2']
+          ['gemirro', '0.0.2'],
+          ['gemirro', '0.0.1.alpha1'],
+          ['gemirro', '0.0.2.alpha2']
         ]
       )
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gemirro
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.11.0
 platform: ruby
 authors:
 - Pierre Rambaud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-19 00:00:00.000000000 Z
+date: 2015-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -178,7 +178,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.6.7
-description: Create your own gem mirror with a simple TCPServer.
+description: Create your own gems mirror.
 email: pierre.rambaud86@gmail.com
 executables:
 - gemirro
@@ -215,6 +215,8 @@ files:
 - lib/gemirro/version.rb
 - lib/gemirro/versions_fetcher.rb
 - lib/gemirro/versions_file.rb
+- spec/fixtures/gems/gemirro-0.0.1.gem
+- spec/fixtures/quick/gemirro-0.0.1.gemspec.rz
 - spec/gemirro/cli_spec.rb
 - spec/gemirro/configuration_spec.rb
 - spec/gemirro/gem_spec.rb
@@ -272,6 +274,6 @@ rubyforge_project:
 rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
-summary: Gem for easily creating your own RubyGems mirror.
+summary: Gem for easily creating your own gems mirror.
 test_files: []
 has_rdoc: