gemirro 0.0.1

data/lib/gemirro/gem.rb

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+module Gemirro
+  ##
+  # The Gem class contains data about a Gem such as the name, requirement as
+  # well as providing some methods to more easily extract the specific version
+  # number.
+  #
+  # @!attribute [r] name
+  #  @return [String]
+  # @!attribute [r] requirement
+  #  @return [Gem::Requirement]
+  #
+  class Gem
+    attr_reader :name, :requirement
+
+    ##
+    # Returns a `Gem::Version` instance based on the specified requirement.
+    #
+    # @param [Gem::Requirement] requirement
+    # @return [Gem::Version]
+    #
+    def self.version_for(requirement)
+      ::Gem::Version.new(requirement.requirements.sort.last.last.version)
+    end
+
+    ##
+    # @param [String] name
+    # @param [Gem::Requirement|String] requirement
+    #
+    def initialize(name, requirement = nil)
+      requirement ||= ::Gem::Requirement.default
+
+      if requirement.is_a?(String)
+        requirement = ::Gem::Requirement.new(requirement)
+      end
+
+      @name        = name
+      @requirement = requirement
+    end
+
+    ##
+    # Returns the version
+    #
+    # @return [Gem::Version]
+    #
+    def version
+      @version ||= self.class.version_for(requirement)
+    end
+
+    ##
+    # Define if version exists
+    #
+    # @return [TrueClass|FalseClass]
+    #
+    def version?
+      version && !version.segments.reject { |s| s == 0 }.empty?
+    end
+
+    ##
+    # Returns the filename of the Gemfile.
+    #
+    # @param [String] gem_version
+    # @return [String]
+    #
+    def filename(gem_version = nil)
+      gem_version ||= version.to_s
+      "#{name}-#{gem_version}.gem"
+    end
+  end
+end

data/lib/gemirro/gems_fetcher.rb

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+module Gemirro
+  ##
+  # The GemsFetcher class is responsible for downloading Gems from an external
+  # source.
+  #
+  # @!attribute [r] source
+  #  @return [Source]
+  # @!attribute [r] versions_file
+  #  @return [Gemirro::VersionsFile]
+  #
+  class GemsFetcher
+    attr_reader :source, :versions_file
+
+    ##
+    # @param [Source] source
+    # @param [Gemirro::VersionsFile] versions_file
+    #
+    def initialize(source, versions_file)
+      @source        = source
+      @versions_file = versions_file
+    end
+
+    ##
+    # Fetches the Gems.
+    #
+    def fetch
+      @source.gems.each do |gem|
+        versions_for(gem).each do |version|
+          filename  = gem.filename(version)
+          satisfied = gem.requirement.satisfied_by?(version)
+          name      = gem.name
+
+          if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
+            logger.debug("Skipping #{filename}")
+            next
+          end
+
+          configuration.ignore_gem(gem.name, version)
+          logger.info("Fetching #{filename}")
+          gemfile = fetch_gem(gem, version)
+          configuration.mirror_directory.add_file(filename, gemfile) if gemfile
+        end
+      end
+    end
+
+    ##
+    # Returns an Array containing the versions that should be fetched for a
+    # Gem.
+    #
+    # @param [Gemirro::Gem] gem
+    # @return [Array]
+    #
+    def versions_for(gem)
+      available       = @versions_file.versions_for(gem.name)
+      versions        = gem.version? ? [gem.version] : available
+      available_names = available.map(&:to_s)
+
+      # Get rid of invalid versions. Due to Gem::Version having a custom ==
+      # method, which treats "3.4" the same as "3.4.0" we'll have to compare
+      # the versions as String instances.
+      versions = versions.select do |version|
+        available_names.include?(version.to_s)
+      end
+
+      versions = [available.last] if versions.empty?
+
+      versions
+    end
+
+    ##
+    # Tries to download the Gemfile for the specified Gem and version.
+    #
+    # @param [Gemirro::Gem] gem
+    # @param [Gem::Version] version
+    # @return [String]
+    #
+    def fetch_gem(gem, version)
+      data  = nil
+      filename = gem.filename(version)
+
+      begin
+        data = @source.fetch_gem(gem.name, version)
+      rescue => e
+        logger.error("Failed to retrieve #{filename}: #{e.message}")
+        logger.debug("Adding #{filename} to the list of ignored Gems")
+
+        configuration.ignore_gem(gem.name, version)
+      end
+
+      data
+    end
+
+    ##
+    # @see Gemirro::Configuration#logger
+    # @return [Logger]
+    #
+    def logger
+      configuration.logger
+    end
+
+    ##
+    # @see Gemirro.configuration
+    #
+    def configuration
+      Gemirro.configuration
+    end
+
+    ##
+    # Checks if a given Gem has already been downloaded.
+    #
+    # @param [String] filename
+    # @return [TrueClass|FalseClass]
+    #
+    def gem_exists?(filename)
+      configuration.mirror_directory.file_exists?(filename)
+    end
+
+    ##
+    # @see Gemirro::Configuration#ignore_gem?
+    #
+    def ignore_gem?(*args)
+      configuration.ignore_gem?(*args)
+    end
+  end
+end

data/lib/gemirro/http.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+
+module Gemirro
+  ##
+  # The Http class is responsible for executing GET request
+  # to a specific url and return an response as an HTTP::Message
+  #
+  # @!attribute [r] client
+  #  @return [HTTPClient]
+  #
+  class Http
+    attr_accessor :client
+
+    ##
+    # Requests the given HTTP resource.
+    #
+    # @param [String] url
+    # @return [HTTP::Message]
+    #
+    def self.get(url)
+      response = client.get(url, follow_redirect: true)
+
+      unless HTTP::Status.successful?(response.status)
+        fail HTTPClient::BadResponseError, response.reason
+      end
+
+      response
+    end
+
+    ##
+    # @return [HTTPClient]
+    #
+    def self.client
+      @client ||= HTTPClient.new
+    end
+  end
+end

data/lib/gemirro/indexer.rb

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+module Gemirro
+  ##
+  # The Indexer class is responsible for downloading useful file directly
+  # on the source host, such as specs-*.*.gz, marshal information, etc...
+  #
+  # @!attribute [r] files
+  #  @return [Array]
+  # @!attribute [r] quick_marshal_dir
+  #  @return [String]
+  # @!attribute [r] directory
+  #  @return [String]
+  # @!attribute [r] dest_directory
+  #  @return [String]
+  #
+  class Indexer < ::Gem::Indexer
+    attr_accessor :files, :quick_marshal_dir, :directory, :dest_directory
+
+    ##
+    # Generate indicies on the destination directory
+    #
+    # @return [Array]
+    #
+    def install_indicies
+      verbose = ::Gem.configuration.really_verbose
+      say "Downloading index into production dir #{@dest_directory}" if verbose
+
+      files = @files
+      files.delete @quick_marshal_dir if files.include? @quick_dir
+
+      if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
+        files.delete @quick_marshal_dir
+        dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
+        FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
+        FileUtils.rm_rf(dst_name, verbose: verbose)
+        FileUtils.mv(@quick_marshal_dir, dst_name,
+                     verbose: verbose, force: true)
+      end
+
+      files = files.map do |path|
+        path.sub(/^#{Regexp.escape @directory}\/?/, '')
+      end
+
+      files.each do |file|
+        dst_name = File.join @dest_directory, file
+        next if File.exist?(dst_name) &&
+          (File.mtime(dst_name) >= Time.now - 360)
+
+        resp = Http.get("#{Gemirro.configuration.source.host}/#{file}")
+        next unless resp.code == 200
+        MirrorFile.new(dst_name).write(resp.body)
+      end
+    end
+  end
+end

data/lib/gemirro/mirror_directory.rb

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+module Gemirro
+  ##
+  # The MirrorDirectory is used for dealing with files and directories that are
+  # mirrored from an external source.
+  #
+  # @!attribute [r] path
+  #  @return [String]
+  #
+  class MirrorDirectory
+    attr_reader :path
+
+    ##
+    # @param [String] path
+    #
+    def initialize(path)
+      @path = path
+    end
+
+    ##
+    # Creates directory or directories with the given path.
+    #
+    # @param [String] dir_path
+    # @return [Gemirro::MirrorDirectory]
+    #
+    def add_directory(dir_path)
+      full_path = File.join(@path, dir_path)
+      FileUtils.mkdir_p(full_path) unless File.directory?(full_path)
+
+      self.class.new(full_path)
+    end
+
+    ##
+    # Creates a new file with the given name and content.
+    #
+    # @param [String] name
+    # @param [String] content
+    # @return [Gem::MirrorFile]
+    #
+    def add_file(name, content)
+      full_path = File.join(@path, name)
+      file      = MirrorFile.new(full_path)
+
+      file.write(content)
+
+      file
+    end
+
+    ##
+    # Checks if a given file exists in the current directory.
+    #
+    # @param [String] name
+    # @return [TrueClass|FalseClass]
+    #
+    def file_exists?(name)
+      File.file?(File.join(@path, name))
+    end
+  end
+end

data/lib/gemirro/mirror_file.rb

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+module Gemirro
+  ##
+  # Similar to {Gemirro::MirrorDirectory} the MirrorFile class is used to
+  # make it easier to read and write data in a directory that mirrors data from
+  # an external source.
+  #
+  # @!attribute [r] path
+  #  @return [String]
+  #
+  class MirrorFile
+    attr_reader :path
+
+    ##
+    # @param [String] path
+    #
+    def initialize(path)
+      @path = path
+    end
+
+    ##
+    # Writes the specified content to the current file. Existing files are
+    # overwritten.
+    #
+    # @param [String] content
+    #
+    def write(content)
+      handle = File.open(@path, 'w')
+
+      handle.write(content)
+      handle.close
+    end
+
+    ##
+    # Reads the content of the current file.
+    #
+    # @return [String]
+    #
+    def read
+      handle  = File.open(@path, 'r')
+      content = handle.read
+
+      handle.close
+
+      content
+    end
+  end
+end

data/lib/gemirro/server.rb

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+
+module Gemirro
+  ##
+  # Launch TCPServer to easily download gems.
+  #
+  # @!attribute [r] server
+  #  @return [TCPServer]
+  # @!attribute [r] destination
+  #  @return [String]
+  # @!attribute [r] versions_fetcher
+  #  @return [VersionsFetcher]
+  # @!attribute [r] gems_fetcher
+  #  @return [Gemirro::GemsFetcher]
+  #
+  class Server
+    attr_reader :server, :destination, :versions_fetcher, :gems_fetcher
+
+    ##
+    # Initialize Server
+    #
+    def initialize
+      configuration.server_host = 'localhost' if configuration.server_host.nil?
+      configuration.server_port = '2000' if configuration.server_port.nil?
+      logger.info('Running server on ' \
+                  "#{configuration.server_host}:#{configuration.server_port}")
+      @server = TCPServer.new(
+        configuration.server_host,
+        configuration.server_port
+      )
+
+      @destination = configuration.destination
+    end
+
+    ##
+    # Run the server and accept all connection
+    #
+    # @return [nil]
+    #
+    def run
+      while (session = server.accept)
+        request = session.gets
+        logger.info(request)
+
+        trimmedrequest = request.gsub(/GET\ \//, '').gsub(/\ HTTP.*/, '').chomp
+        resource = "#{@destination}/#{trimmedrequest}"
+
+        # Try to download gem if file doesn't exists
+        fetch_gem(resource) unless File.exist?(resource)
+
+        # If not found again, return a 404
+        unless File.exist?(resource)
+          logger.warn("404 - #{trimmedrequest.gsub(/^public\//, '')}")
+          session.print "HTTP/1.1 404/Object Not Found\r\n\r\n"
+          session.close
+          next
+        end
+
+        if File.directory?(resource)
+          display_directory(session, resource)
+        else
+          mime_type = MIME::Types.type_for(resource)
+          session.print "HTTP/1.1 200/OK\r\nContent-type:#{mime_type}\r\n\r\n"
+          file = open(resource, 'rb')
+          session.puts(file.read)
+        end
+
+        session.close
+      end
+    end
+
+    ##
+    # Try to fetch gem and download its if it's possible, and
+    # build and install indicies.
+    #
+    # @param [String] resource
+    # @return [Indexer]
+    #
+    def fetch_gem(resource)
+      name = File.basename(resource)
+      regexp = /^(.*)-(\d+(?:\.\d+){,4})\.gem(?:spec\.rz)?$/
+      gem_name, gem_version = name.match(regexp).captures
+
+      return unless gem_name && gem_version
+
+      logger.info("Try to download #{gem_name} with version #{gem_version}")
+      begin
+        gems_fetcher.source.gems.clear
+        gems_fetcher.source.gems.push(Gemirro::Gem.new(gem_name, gem_version))
+        gems_fetcher.fetch
+      rescue StandardError => e
+        logger.error(e.message)
+      end
+
+      generate_index
+    end
+
+    ##
+    # Generate index and install indicies.
+    #
+    # @return [Indexer]
+    #
+    def generate_index
+      indexer    = Indexer.new(configuration.destination)
+      indexer.ui = ::Gem::SilentUI.new
+
+      logger.info('Generating indexes')
+      indexer.generate_index
+    end
+
+    ##
+    # Display directory on the current sesion
+    #
+    # @param [TCPSocket] session
+    # @param [String] resource
+    # @return [Array]
+    #
+    def display_directory(session, resource)
+      session.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n"
+      base_dir = Dir.new(resource)
+      base_dir.entries.sort.each do |f|
+        dir_sign = ''
+        resource_path = resource.gsub(/\/$/, '') + '/' + f
+        dir_sign = '/' if File.directory?(resource_path)
+        resource_path = resource_path.gsub(/^public\//, '')
+        resource_path = resource_path.gsub(@destination, '')
+
+        session.print(
+          "<a href=\"#{resource_path}\">#{f}#{dir_sign}</a><br>"
+        ) unless ['.', '..'].include?(File.basename(resource_path))
+      end
+    end
+
+    ##
+    # @see Gemirro::Configuration#logger
+    # @return [Logger]
+    #
+    def logger
+      configuration.logger
+    end
+
+    ##
+    # @see Gemirro.configuration
+    #
+    def configuration
+      Gemirro.configuration
+    end
+
+    ##
+    # @see Gemirro::VersionsFetcher.fetch
+    #
+    def versions_fetcher
+      @versions_fetcher ||= Gemirro::VersionsFetcher.new(configuration.source).fetch
+    end
+
+    ##
+    # @return [Gemirro::GemsFetcher]
+    #
+    def gems_fetcher
+      @gems_fetcher = Gemirro::GemsFetcher.new(
+        configuration.source, versions_fetcher)
+    end
+  end
+end