gem_mirror 0.1.0.pre

data/lib/gem_mirror/gem.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module GemMirror
+  ##
+  # The Gem class contains data about a Gem such as the name, requirement as
+  # well as providing some methods to more easily extract the specific version
+  # number.
+  #
+  # @!attribute [r] name
+  #  @return [String]
+  # @!attribute [r] requirement
+  #  @return [Gem::Requirement]
+  #
+  class Gem
+    attr_reader :name, :requirement
+
+    ##
+    # Returns a `Gem::Version` instance based on the specified requirement.
+    #
+    # @param [Gem::Requirement] requirement
+    # @return [Gem::Version]
+    #
+    def self.version_for(requirement)
+      ::Gem::Version.new(requirement.requirements.max.last.version)
+    end
+
+    ##
+    # @param [String] name
+    # @param [Gem::Requirement] requirement
+    #
+    def initialize(name, requirement = nil)
+      requirement ||= ::Gem::Requirement.default
+
+      requirement = ::Gem::Requirement.new(requirement) if requirement.is_a?(String)
+
+      @name        = name
+      @requirement = requirement
+    end
+
+    ##
+    # @return [Gem::Version]
+    #
+    def version
+      @version ||= self.class.version_for(requirement)
+    end
+
+    ##
+    # @return [TrueClass|FalseClass]
+    #
+    def version?
+      version && !version.segments.reject(&:zero?).empty?
+    end
+
+    ##
+    # Returns the filename of the Gemfile.
+    #
+    # @param [String] gem_version
+    # @return [String]
+    #
+    def filename(gem_version = nil)
+      gem_version ||= version.to_s
+
+      "#{name}-#{gem_version}.gem"
+    end
+  end
+end

data/lib/gem_mirror/gems_fetcher.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+module GemMirror
+  ##
+  # The GemsFetcher class is responsible for downloading Gems from an external
+  # source as well as downloading all the associated dependencies.
+  #
+  # @!attribute [r] source
+  #  @return [Source]
+  # @!attribute [r] versions_file
+  #  @return [GemMirror::VersionsFile]
+  #
+  class GemsFetcher
+    attr_reader :source, :versions_file
+
+    ##
+    # @param [Source] source
+    # @param [GemMirror::VersionsFile] versions_file
+    #
+    def initialize(source, versions_file)
+      @source        = source
+      @versions_file = versions_file
+    end
+
+    ##
+    # Fetches the Gems and all associated dependencies.
+    #
+    def fetch
+      source.gems.each do |gem|
+        versions_for(gem).each do |version|
+          filename = gem.filename(version)
+          begin
+            satisfied = gem.requirement.satisfied_by?(version)
+          rescue StandardError
+            logger.debug("Error determining is requirement satisfied for #{filename}")
+          end
+          name = gem.name
+
+          if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
+            logger.debug("Skipping #{filename}")
+            next
+          end
+
+          # Prevent circular dependencies from messing things up.
+          configuration.ignore_gem(gem.name, version)
+
+          spec = fetch_specification(gem, version)
+
+          next unless spec
+
+          spec = load_specification(spec)
+          deps = dependencies_for(spec)
+
+          unless deps.empty?
+            logger.info("Fetching dependencies for #{filename}")
+
+            fetch_dependencies(deps)
+          end
+
+          logger.info("Fetching #{filename}")
+
+          gemfile = fetch_gem(gem, version)
+
+          configuration.mirror_directory.add_file(filename, gemfile) if gemfile
+        end
+      end
+    end
+
+    ##
+    # Returns an Array containing the versions that should be fetched for a
+    # Gem.
+    #
+    # @param [GemMirror::Gem] gem
+    # @return [Array]
+    #
+    def versions_for(gem)
+      available       = versions_file.versions_for(gem.name)
+      versions        = gem.version? ? [gem.version] : available
+      available_names = available.map(&:to_s)
+
+      # Get rid of invalid versions. Due to Gem::Version having a custom ==
+      # method, which treats "3.4" the same as "3.4.0" we'll have to compare
+      # the versions as String instances.
+      versions = versions.select do |version|
+        available_names.include?(version.to_s)
+      end
+
+      versions = [available.last] if versions.empty?
+
+      versions
+    end
+
+    ##
+    # Tries to download the specification for a Gem and version. This method
+    # returns the raw inflated data instead of an instance of
+    # `Gem::Specification`.
+    #
+    # @param [GemMirror::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_specification(gem, version)
+      specification = nil
+      filename      = gem.filename(version)
+
+      begin
+        specification = source.fetch_specification(gem.name, version)
+      rescue StandardError => e
+        logger.error("Failed to retrieve #{filename}: #{e.message}")
+        logger.debug("Adding #{filename} to the list of ignored Gems")
+
+        configuration.ignore_gem(gem.name, version)
+      end
+
+      specification
+    end
+
+    ##
+    # Tries to download the Gemfile for the specified Gem and version.
+    #
+    # @param [GemMirror::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_gem(gem, version)
+      gemfile  = nil
+      filename = gem.filename(version)
+
+      begin
+        gemfile = source.fetch_gem(gem.name, version)
+      rescue StandardError => e
+        logger.error("Failed to retrieve #{filename}: #{e.message}")
+        logger.debug("Adding #{filename} to the list of ignored Gems")
+
+        configuration.ignore_gem(gem.name, version)
+      end
+
+      gemfile
+    end
+
+    ##
+    # Reads the inflated data of a Gemspec and returns the loaded specification
+    # instance.
+    #
+    # @param [String] raw_spec
+    # @return [Gem::Specification]
+    #
+    def load_specification(raw_spec)
+      stream  = Zlib::Inflate.new
+      content = stream.inflate(raw_spec)
+
+      stream.finish
+      stream.close
+
+      Marshal.load(content)
+    end
+
+    ##
+    # Fetches the Gem files for the specified dependencies.
+    #
+    # @param [Array] deps
+    #
+    def fetch_dependencies(deps)
+      self.class.new(source.updated(deps), versions_file).fetch
+    end
+
+    ##
+    # Returns an Array containing all the dependencies of a given Gem
+    # specification.
+    #
+    # @param [Gem::Specification] spec
+    # @return [Array]
+    #
+    def dependencies_for(spec)
+      possible_dependencies = if configuration.development
+                                spec.dependencies
+                              else
+                                spec.runtime_dependencies
+                              end
+
+      dependencies = filter_dependencies(possible_dependencies)
+
+      assign_gem_versions(dependencies)
+    end
+
+    ##
+    # Filters a list of dependencies based on whether or not they are ignored.
+    #
+    # @param [Array] possible_dependencies
+    # @return [Array]
+    #
+    def filter_dependencies(possible_dependencies)
+      dependencies = []
+
+      possible_dependencies.each do |dependency|
+        gem = Gem.new(dependency.name, dependency.requirement)
+
+        dependencies << gem unless ignore_gem?(gem.name, gem.version)
+      end
+
+      dependencies
+    end
+
+    ##
+    # Processes a list of Gems and sets their versions to the latest one
+    # available in case no specific version is given.
+    #
+    # @param [Array] gems
+    # @return [Array]
+    #
+    def assign_gem_versions(gems)
+      gems.map do |gem|
+        unless gem.version?
+          latest = versions_file.versions_for(gem.name).last
+          gem    = Gem.new(gem.name, latest.to_s) if latest
+        end
+
+        gem
+      end
+    end
+
+    ##
+    # @see GemMirror::Configuration#logger
+    # @return [Logger]
+    #
+    def logger
+      configuration.logger
+    end
+
+    ##
+    # @see GemMirror.configuration
+    #
+    def configuration
+      GemMirror.configuration
+    end
+
+    ##
+    # Checks if a given Gem has already been downloaded.
+    #
+    # @param [String] filename
+    # @return [TrueClass|FalseClass]
+    #
+    def gem_exists?(filename)
+      configuration.mirror_directory.file_exists?(filename)
+    end
+
+    ##
+    # @see GemMirror::Configuration#ignore_gem?
+    #
+    def ignore_gem?(*args)
+      configuration.ignore_gem?(*args)
+    end
+  end
+end

data/lib/gem_mirror/mirror_directory.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module GemMirror
+  ##
+  # The MirrorDirectory is used for dealing with files and directories that are
+  # mirrored from an external source.
+  #
+  # @!attribute [r] path
+  #  @return [String]
+  #
+  class MirrorDirectory
+    attr_reader :path
+
+    ##
+    # @param [String] path
+    #
+    def initialize(path)
+      @path = path
+    end
+
+    ##
+    # Creates a new directory with the given name.
+    #
+    # @param [String] name
+    # @return [GemMirror::MirrorDirectory]
+    #
+    def add_directory(name)
+      full_path = File.join(path, name)
+
+      Dir.mkdir(full_path) unless File.directory?(full_path)
+
+      self.class.new(full_path)
+    end
+
+    ##
+    # Creates a new file with the given name and content.
+    #
+    # @param [String] name
+    # @param [String] content
+    # @return [Gem::MirrorFile]
+    #
+    def add_file(name, content)
+      full_path = File.join(path, name)
+      file      = MirrorFile.new(full_path)
+
+      file.write(content)
+
+      file
+    end
+
+    ##
+    # Checks if a given file exists in the current directory.
+    #
+    # @param [String] name
+    # @return [TrueClass|FalseClass]
+    #
+    def file_exists?(name)
+      File.file?(File.join(path, name))
+    end
+  end
+end

data/lib/gem_mirror/mirror_file.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module GemMirror
+  ##
+  # Similar to {GemMirror::MirrorDirectory} the MirrorFile class is used to
+  # make it easier to read and write data in a directory that mirrors data from
+  # an external source.
+  #
+  # @!attribute [r] path
+  #  @return [String]
+  #
+  class MirrorFile
+    attr_reader :path
+
+    ##
+    # @param [String] path
+    #
+    def initialize(path)
+      @path = path
+    end
+
+    ##
+    # Writes the specified content to the current file. Existing files are
+    # overwritten.
+    #
+    # @param [String] content
+    #
+    def write(content)
+      handle = File.open(path, "w")
+
+      handle.write(content)
+      handle.close
+    end
+
+    ##
+    # Reads the content of the current file.
+    #
+    # @return [String]
+    #
+    def read
+      handle  = File.open(path, "r")
+      content = handle.read
+
+      handle.close
+
+      content
+    end
+
+    ##
+    # Reads the contents of a Gzip encoded file.
+    #
+    # @return [String]
+    #
+    def read_gzip
+      content = nil
+
+      Zlib::GzipReader.open(path) do |gz|
+        content = gz.read
+        gz.close
+      end
+
+      content
+    end
+  end
+end