gem-mirror 0.0.1

data/lib/gem-mirror/cli.rb

@@ -0,0 +1,65 @@
+module GemMirror
+  module CLI
+    ##
+    # Hash containing the default Slop options.
+    #
+    # @return [Hash]
+    #
+    SLOP_OPTIONS = {
+      :strict => true,
+      :banner => 'Usage: gem-mirror [COMMAND] [OPTIONS]'
+    }
+
+    ##
+    # @return [Slop]
+    #
+    def self.options
+      return @options ||= default_options
+    end
+
+    ##
+    # Loads the specified configuration file or displays an error if it doesn't
+    # exist.
+    #
+    # @param [String] config_file
+    # @return [GemMirror::Configuration]
+    #
+    def self.load_configuration(config_file)
+      config_file ||= Configuration.default_configuration_file
+      config_file   = File.expand_path(config_file, Dir.pwd)
+
+      unless File.file?(config_file)
+        abort "The configuration file #{config_file} does not exist"
+      end
+
+      require(config_file)
+    end
+
+    ##
+    # @return [Slop]
+    #
+    def self.default_options
+      return Slop.new(SLOP_OPTIONS.dup) do
+        separator "\nOptions:\n"
+
+        on :h, :help, 'Shows this help message' do
+          puts self
+          exit
+        end
+
+        on :v, :version, 'Shows the current version' do
+          puts CLI.version_information
+        end
+      end
+    end
+
+    ##
+    # Returns a String containing some platform/version related information.
+    #
+    # @return [String]
+    #
+    def self.version_information
+      return "gem-mirror v#{VERSION} on #{RUBY_DESCRIPTION}"
+    end
+  end # CLI
+end # GemMirror

data/lib/gem-mirror/cli/checksum.rb

@@ -0,0 +1,41 @@
+GemMirror::CLI.options.command 'checksum' do
+  banner      'Usage: gem-mirror checksum [OPTIONS]'
+  description 'Generates SHA512 checksums of all gems'
+  separator   "\nOptions:\n"
+
+  on :h, :help, 'Shows this help message' do
+    puts self
+    exit
+  end
+
+  on :c=, :config=, 'Path to the configuration file'
+
+  run do |opts, args|
+    GemMirror::CLI.load_configuration(opts[:c])
+
+    config = GemMirror.configuration
+
+    unless File.directory?(config.checksums)
+      config.logger("The directory #{config.checksums} does not exist")
+      abort
+    end
+
+    unless File.directory?(config.destination)
+      config.logger("The directory #{config.destination} does not exist")
+      abort
+    end
+
+    Dir[File.join(config.gems_directory, '*.gem')].each do |gem|
+      basename = File.basename(gem)
+      name     = basename + '.sha512'
+
+      config.logger.info("Creating checksum for #{basename}")
+
+      hash   = Digest::SHA512.hexdigest(File.read(gem))
+      handle = File.open(File.join(config.checksums, name), 'w')
+
+      handle.write(hash)
+      handle.close
+    end
+  end
+end

data/lib/gem-mirror/cli/index.rb

@@ -0,0 +1,31 @@
+GemMirror::CLI.options.command 'index' do
+  banner      'Usage: gem-mirror index [OPTIONS]'
+  description 'Indexes a list of Gems'
+  separator   "\nOptions:\n"
+
+  on :h, :help, 'Shows this help message' do
+    puts self
+    exit
+  end
+
+  on :c=, :config=, 'Path to the configuration file'
+  on :l, :legacy, 'Build legacy indexes'
+
+  run do |opts, args|
+    GemMirror::CLI.load_configuration(opts[:c])
+
+    config = GemMirror.configuration
+
+    unless File.directory?(config.destination)
+      config.logger.error("The directory #{config.destination} does not exist")
+      abort
+    end
+
+    indexer    = Gem::Indexer.new(config.destination, :build_legacy => opts[:l])
+    indexer.ui = Gem::SilentUI.new
+
+    config.logger.info("Generating indexes")
+
+    indexer.generate_index
+  end
+end

data/lib/gem-mirror/cli/init.rb

@@ -0,0 +1,21 @@
+GemMirror::CLI.options.command 'init' do
+  banner      'Usage: gem-mirror init [DIRECTORY] [OPTIONS]'
+  description 'Sets up a new mirror'
+  separator   "\nOptions:\n"
+
+  on :h, :help, 'Shows this help message' do
+    puts self
+    exit
+  end
+
+  run do |opts, args|
+    directory = File.expand_path(args[0] || Dir.pwd)
+    template  = GemMirror::Configuration.template_directory
+
+    Dir.mkdir(directory) unless File.directory?(directory)
+
+    FileUtils.cp_r(File.join(template, '.'), directory)
+
+    puts "Initialized empty mirror in #{directory}"
+  end
+end

data/lib/gem-mirror/cli/update.rb

@@ -0,0 +1,23 @@
+GemMirror::CLI.options.command 'update' do
+  banner      'Usage: gem-mirror update [OPTIONS]'
+  description 'Updates the list of Gems'
+  separator   "\nOptions:\n"
+
+  on :h, :help, 'Shows this help message' do
+    puts self
+    exit
+  end
+
+  on :c=, :config=, 'Path to the configuration file'
+
+  run do |opts, args|
+    GemMirror::CLI.load_configuration(opts[:c])
+
+    GemMirror.configuration.sources.each do |source|
+      versions = GemMirror::VersionsFetcher.new(source).fetch
+      gems     = GemMirror::GemsFetcher.new(source, versions)
+
+      gems.fetch
+    end
+  end
+end

data/lib/gem-mirror/configuration.rb

@@ -0,0 +1,132 @@
+module GemMirror
+  ##
+  # @return [GemMirror::Configuration]
+  #
+  def self.configuration
+    return @configuration ||= Configuration.new
+  end
+
+  ##
+  # Configuration class used for storing data about a mirror such as the
+  # destination directory, sources, ignored Gems, etc.
+  #
+  class Configuration < Confstruct::Configuration
+    ##
+    # @return [Logger]
+    #
+    def logger
+      return @logger ||= Logger.new(STDOUT)
+    end
+
+    ##
+    # @return [String]
+    #
+    def self.template_directory
+      return File.expand_path('../../../template', __FILE__)
+    end
+
+    ##
+    # @return [String]
+    #
+    def self.default_configuration_file
+      return File.expand_path('config.rb', Dir.pwd)
+    end
+
+    ##
+    # Returns the name of the directory that contains the quick
+    # specification files.
+    #
+    # @return [String]
+    #
+    def self.marshal_identifier
+      return "Marshal.#{marshal_version}"
+    end
+
+    ##
+    # Returns the name of the file that contains an index of all the versions.
+    #
+    # @return [String]
+    #
+    def self.versions_file
+      return "specs.#{marshal_version}.gz"
+    end
+
+    ##
+    # Returns a String containing the Marshal version.
+    #
+    # @return [String]
+    #
+    def self.marshal_version
+      return "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}"
+    end
+
+    ##
+    # @return [GemMirror::MirrorDirectory]
+    #
+    def mirror_directory
+      return @mirror_directory ||= MirrorDirectory.new(gems_directory)
+    end
+
+    ##
+    # @return [String]
+    #
+    def gems_directory
+      return File.join(destination, 'gems')
+    end
+
+    ##
+    # Returns a Hash containing various Gems to ignore and their versions.
+    #
+    # @return [Hash]
+    #
+    def ignored_gems
+      return @ignored_gems ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    ##
+    # Adds a Gem to the list of Gems to ignore.
+    #
+    # @param [String] name
+    # @param [String] version
+    #
+    def ignore_gem(name, version)
+      ignored_gems[name] ||= []
+      ignored_gems[name] << version
+    end
+
+    ##
+    # Checks if a Gem should be ignored.
+    #
+    # @param [String] name
+    # @param [String] version
+    # @return [TrueClass|FalseClass]
+    #
+    def ignore_gem?(name, version)
+      return ignored_gems[name].include?(version)
+    end
+
+    ##
+    # Returns a list of sources to mirror.
+    #
+    # @return [Array]
+    #
+    def sources
+      return @sources ||= []
+    end
+
+    ##
+    # Adds a new source to mirror.
+    #
+    # @param [String] name
+    # @param [String] url
+    # @param [Proc] block
+    # @yieldparam [GemMirror::Source] source
+    #
+    def source(name, url, &block)
+      source = Source.new(name, url)
+      source.instance_eval(&block)
+
+      sources << source
+    end
+  end # Configuration
+end # GemMirror

data/lib/gem-mirror/gem.rb

@@ -0,0 +1,53 @@
+module GemMirror
+  ##
+  # The Gem class contains data about a Gem such as the name, requirement as
+  # well as providing some methods to more easily extract the specific version
+  # number.
+  #
+  # @!attribute [r] name
+  #  @return [String]
+  # @!attribute [r] requirement
+  #  @return [Gem::Requirement]
+  #
+  class Gem
+    attr_reader :name, :requirement
+
+    ##
+    # Returns a `Gem::Version` instance based on the specified requirement.
+    #
+    # @param [Gem::Requirement] requirement
+    # @return [Gem::Version]
+    #
+    def self.version_for(requirement)
+      return ::Gem::Version.new(requirement.requirements.sort.last.last.version)
+    end
+
+    ##
+    # @param [String] name
+    # @param [Gem::Requirement] requirement
+    #
+    def initialize(name, requirement = nil)
+      @name        = name
+      @requirement = requirement || ::Gem::Requirement.default
+    end
+
+    ##
+    # @return [Gem::Version]
+    #
+    def version
+      return @version ||= self.class.version_for(requirement)
+    end
+
+    ##
+    # Returns the filename of the Gemfile.
+    #
+    # @param [String] gem_version
+    # @return [String]
+    #
+    def filename(gem_version = nil)
+      gem_version ||= version.to_s
+
+      return "#{name}-#{gem_version}.gem"
+    end
+  end # Gem
+end # GemMirror

data/lib/gem-mirror/gems_fetcher.rb

@@ -0,0 +1,194 @@
+module GemMirror
+  ##
+  # The GemsFetcher class is responsible for downloading Gems from an external
+  # source as well as downloading all the associated dependencies.
+  #
+  # @!attribute [r] source
+  #  @return [Source]
+  # @!attribute [r] versions_file
+  #  @return [GemMirror::VersionsFile]
+  #
+  class GemsFetcher
+    attr_reader :source, :versions_file
+
+    ##
+    # @param [Source] source
+    # @param [GemMirror::VersionsFile] versions_file
+    #
+    def initialize(source, versions_file)
+      @source        = source
+      @versions_file = versions_file
+    end
+
+    ##
+    # Fetches the Gems and all associated dependencies.
+    #
+    def fetch
+      source.gems.each do |gem|
+        versions_file.versions_for(gem.name).each do |version|
+          filename  = gem.filename(version)
+          satisfied = gem.requirement.satisfied_by?(version)
+          name      = gem.name
+
+          if gem_exists?(filename) or ignore_gem?(name, version) or !satisfied
+            logger.debug("Skipping #{filename}")
+            next
+          end
+
+          # Prevent circular dependencies from messing things up.
+          configuration.ignore_gem(gem.name, version)
+
+          spec = fetch_specification(gem, version)
+
+          next unless spec
+
+          spec = load_specification(spec)
+          deps = dependencies_for(spec)
+
+          unless deps.empty?
+            logger.info("Fetching dependencies for #{filename}")
+
+            fetch_dependencies(deps)
+          end
+
+          logger.info("Fetching #{filename}")
+
+          gemfile = fetch_gem(gem, version)
+
+          if gemfile
+            configuration.mirror_directory.add_file(filename, gemfile)
+          end
+        end
+      end
+    end
+
+    ##
+    # Tries to download the specification for a Gem and version. This method
+    # returns the raw inflated data instead of an instance of
+    # `Gem::Specification`.
+    #
+    # @param [GemMirror::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_specification(gem, version)
+      specification = nil
+      filename      = gem.filename(version)
+
+      begin
+        specification = source.fetch_specification(gem.name, version)
+      rescue => e
+        logger.error("Failed to retrieve #{filename}: #{e.message}")
+        logger.debug("Adding #{filename} to the list of ignored Gems")
+
+        configuration.ignore_gem(gem.name, version)
+      end
+
+      return specification
+    end
+
+    ##
+    # Tries to download the Gemfile for the specified Gem and version.
+    #
+    # @param [GemMirror::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_gem(gem, version)
+      gemfile  = nil
+      filename = gem.filename(version)
+
+      begin
+        gemfile = source.fetch_gem(gem.name, version)
+      rescue => e
+        logger.error("Failed to retrieve #{filename}: #{e.message}")
+        logger.debug("Adding #{filename} to the list of ignored Gems")
+
+        configuration.ignore_gem(gem.name, version)
+      end
+
+      return gemfile
+    end
+
+    ##
+    # Reads the inflated data of a Gemspec and returns the loaded specification
+    # instance.
+    #
+    # @param [String] raw_spec
+    # @return [Gem::Specification]
+    #
+    def load_specification(raw_spec)
+      stream  = Zlib::Inflate.new
+      content = stream.inflate(raw_spec)
+
+      stream.finish
+      stream.close
+
+      return Marshal.load(content)
+    end
+
+    ##
+    # Fetches the Gem files for the specified dependencies.
+    #
+    # @param [Array] deps
+    #
+    def fetch_dependencies(deps)
+      self.class.new(source.updated(deps), versions_file).fetch
+    end
+
+    ##
+    # Returns an Array containing all the dependencies of a given Gem
+    # specification.
+    #
+    # @param [Gem::Specification] spec
+    # @return [Array]
+    #
+    def dependencies_for(spec)
+      dependencies          = []
+      possible_dependencies = configuration.development ? spec.dependencies \
+        : spec.runtime_dependencies
+
+      possible_dependencies.each do |dependency|
+        gem = Gem.new(dependency.name, dependency.requirement)
+
+        unless ignore_gem?(gem.name, gem.version)
+          dependencies << gem
+        end
+      end
+
+      return dependencies
+    end
+
+    ##
+    # @see GemMirror::Configuration#logger
+    # @return [Logger]
+    #
+    def logger
+      return configuration.logger
+    end
+
+    ##
+    # @see GemMirror.configuration
+    #
+    def configuration
+      return GemMirror.configuration
+    end
+
+    ##
+    # Checks if a given Gem has already been downloaded.
+    #
+    # @param [String] filename
+    # @return [TrueClass|FalseClass]
+    #
+    def gem_exists?(filename)
+      return configuration.mirror_directory.file_exists?(filename)
+    end
+
+    ##
+    # @see GemMirror::Configuration#ignore_gem?
+    #
+    def ignore_gem?(*args)
+      return configuration.ignore_gem?(*args)
+    end
+  end # GemsFetcher
+end # GemMirror