gemirro 1.5.0 → 1.6.0

data/lib/gemirro/server.rb

@@ -10,12 +10,6 @@ module Gemirro
   # Launch Sinatra server to easily download gems.
   #
   class Server < Sinatra::Base
-    # rubocop:disable Layout/LineLength
-    URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/.freeze
-    # rubocop:enable Layout/LineLength
-    GEMSPEC_TYPE = 'gemspec.rz'
-    GEM_TYPE = 'gem'
-
     access_logger = Logger.new(Utils.configuration.server.access_log).tap do |logger|
       ::Logger.class_eval { alias_method :write, :<< }
       logger.level = ::Logger::INFO
@@ -62,7 +56,7 @@ module Gemirro
     end
 
     ##
-    # Display information about one gem
+    # Display information about one gem, human readable
     #
     # @return [nil]
     #
@@ -76,7 +70,7 @@ module Gemirro
 
     ##
     # Display home page containing the list of gems already
-    # downloaded on the server
+    # downloaded on the server, human readable
     #
     # @return [nil]
     #
@@ -85,13 +79,17 @@ module Gemirro
     end
 
     ##
-    # Return gem dependencies as binary
+    # Return gem dependencies as marshaled binary
     #
     # @return [nil]
     #
     get '/api/v1/dependencies' do
       content_type 'application/octet-stream'
-      query_gems.any? ? Marshal.dump(query_gems_list) : 200
+      if params[:gems].to_s != '' && params[:gems].to_s.split(',').any?
+        Marshal.dump(dependencies_loader(params[:gems].to_s.split(',')))
+      else
+        200
+      end
     end
 
     ##
@@ -101,196 +99,105 @@ module Gemirro
     #
     get '/api/v1/dependencies.json' do
       content_type 'application/json'
-      query_gems.any? ? JSON.dump(query_gems_list) : {}
+
+      return '[]' unless params[:gems]
+
+      gem_names = params[:gems].to_s
+                               .split(',')
+                               .map(&:strip)
+                               .reject(&:empty?)
+      return '[]' if gem_names.empty?
+
+      JSON.dump(dependencies_loader(gem_names))
     end
 
     ##
-    # Try to get all request and download files
-    # if files aren't found.
+    # compact_index, Return list of available gem names
     #
     # @return [nil]
     #
-    get('*') do |path|
-      resource = "#{settings.public_folder}#{path}"
+    get '/names' do
+      content_type 'text/plain'
 
-      # Try to download gem
-      fetch_gem(resource) unless File.exist?(resource)
-      # If not found again, return a 404
-      return not_found unless File.exist?(resource)
+      content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last
+      _, etag, repr_digest, _ = content_path.split('.', -4)
 
-      send_file(resource)
+      headers 'etag' => etag
+      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      send_file content_path
     end
 
     ##
-    # Try to fetch gem and download its if it's possible, and
-    # build and install indicies.
+    # compact_index, Return list of gem, including versions
     #
-    # @param [String] resource
-    # @return [Indexer]
+    # @return [nil]
     #
-    def fetch_gem(resource)
-      return unless Utils.configuration.fetch_gem
-
-      name = File.basename(resource)
-      result = name.match(URI_REGEXP)
-      return unless result
-
-      gem_name, gem_version, gem_platform, gem_type = result.captures
-      return unless gem_name && gem_version
+    get '/versions' do
+      content_type 'text/plain'
 
-      begin
-        gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
-        gem.gemspec = true if gem_type == GEMSPEC_TYPE
+      content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last
+      _, etag, repr_digest, _ = content_path.split('.', -4)
 
-        return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
-        return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
-
-        Utils.logger
-             .info("Try to download #{gem_name} with version #{gem_version}")
-        Utils.gems_fetcher.source.gems.clear
-        Utils.gems_fetcher.source.gems.push(gem)
-        Utils.gems_fetcher.fetch
-
-        update_indexes if Utils.configuration.update_on_fetch
-      rescue StandardError => e
-        Utils.logger.error(e)
-      end
+      headers 'etag' => etag
+      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      send_file content_path
     end
 
-    ##
-    # Update indexes files
+    # compact_index, Return gem dependencies for all versions of a gem
     #
-    # @return [Indexer]
+    # @return [nil]
     #
-    def update_indexes
-      indexer = Gemirro::Indexer.new(Utils.configuration.destination)
-      indexer.only_origin = true
-      indexer.ui = ::Gem::SilentUI.new
+    get('/info/:gemname') do
+      gems = Utils.gems_collection
+      gem = gems.find_by_name(params[:gemname])
+      return not_found if gem.nil?
 
-      Utils.logger.info('Generating indexes')
-      indexer.update_index
-      indexer.updated_gems.each do |gem|
-        Utils.cache.flush_key(File.basename(gem))
-      end
-    rescue SystemExit => e
-      Utils.logger.info(e.message)
-    end
+      content_type 'text/plain'
 
-    ##
-    # Return all gems pass to query
-    #
-    # @return [Array]
-    #
-    def query_gems
-      params[:gems].to_s.split(',')
+      content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last
+      _, etag, repr_digest, _ = content_path.split('.', -4)
+
+      headers 'etag' => etag
+      headers 'repr-digest' => %(sha-256="#{repr_digest}")
+      send_file content_path
     end
 
     ##
-    # Return gems list from query params
+    # Try to get all request and download files
+    # if files aren't found.
     #
-    # @return [Array]
+    # @return [nil]
     #
-    def query_gems_list
-      Utils.gems_collection(false) # load collection
-      gems = Parallel.map(query_gems, in_threads: 4) do |query_gem|
-        gem_dependencies(query_gem)
-      end
+    get('*') do |path|
+      resource = "#{settings.public_folder}#{path}"
 
-      gems.flatten!
-      gems.reject!(&:empty?)
-      gems
+      # Try to download gem
+      Gemirro::Utils.fetch_gem(resource) unless File.exist?(resource)
+      # If not found again, return a 404
+      return not_found unless File.exist?(resource)
+
+      send_file(resource)
     end
 
     ##
-    # List of versions and dependencies of each version
-    # from a gem name.
+    # Compile fragments for /api/v1/dependencies
     #
-    # @return [Array]
+    # @return [nil]
     #
-    def gem_dependencies(gem_name)
-      Utils.cache.cache(gem_name) do
-        gems = Utils.gems_collection(false)
-        gem_collection = gems.find_by_name(gem_name)
-
-        return '' if gem_collection.nil?
-
-        gem_collection = Parallel.map(gem_collection, in_threads: 4) do |gem|
-          [gem, spec_for(gem.name, gem.number, gem.platform)]
-        end
-        gem_collection.compact!
-
-        Parallel.map(gem_collection, in_threads: 4) do |gem, spec|
-          next if spec.nil?
-
-          dependencies = spec.dependencies.select do |d|
-            d.type == :runtime
-          end
-
-          dependencies = Parallel.map(dependencies, in_threads: 4) do |d|
-            [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s]
-          end
-
-          {
-            name: gem.name,
-            number: gem.number,
-            platform: gem.platform,
-            dependencies: dependencies
-          }
-        end
-      end
-    end
-
-    helpers do
-      ##
-      # Return gem specification from gemname and version
-      #
-      # @param [String] gemname
-      # @param [String] version
-      # @return [::Gem::Specification]
-      #
-      def spec_for(gemname, version, platform = 'ruby')
-        gem = Utils.stored_gem(gemname, version.to_s, platform)
-        gemspec_path = File.join('quick',
-                                 Gemirro::Configuration.marshal_identifier,
-                                 gem.gemspec_filename)
-        spec_file = File.join(settings.public_folder,
-                              gemspec_path)
-        fetch_gem(gemspec_path) unless File.exist?(spec_file)
-
-        return unless File.exist?(spec_file)
-
-        File.open(spec_file, 'r') do |uz_file|
-          uz_file.binmode
-          inflater = Zlib::Inflate.new
-          begin
-            inflate_data = inflater.inflate(uz_file.read)
-          ensure
-            inflater.finish
-            inflater.close
-          end
-          Marshal.load(inflate_data)
+    def dependencies_loader(names)
+      names.collect do |name|
+        f = File.join(settings.public_folder, 'api', 'v1', 'dependencies', "#{name}.*.*.list")
+        Marshal.load(File.read(Dir.glob(f).last))
+      rescue StandardError => e
+        env['rack.errors'].write "Cound not open #{f}\n"
+        env['rack.errors'].write "#{e.message}\n"
+        e.backtrace.each do |err|
+          env['rack.errors'].write "#{err}\n"
         end
+        nil
       end
-
-      ##
-      # Escape string
-      #
-      # @param [String] string
-      # @return [String]
-      #
-      def escape(string)
-        Rack::Utils.escape_html(string)
-      end
-
-      ##
-      # Homepage link
-      #
-      # @param [Gem] spec
-      # @return [String]
-      #
-      def homepage(spec)
-        URI.parse(Addressable::URI.escape(spec.homepage))
-      end
+      .flatten
+      .compact
     end
   end
 end

data/lib/gemirro/utils.rb

@@ -13,84 +13,52 @@ module Gemirro
   #  @return [Gemirro::GemsFetcher]
   #
   class Utils
-    attr_reader(:cache,
-                :versions_fetcher,
-                :gems_fetcher,
-                :gems_collection,
-                :stored_gems)
+    attr_reader(
+      :versions_fetcher,
+      :gems_fetcher,
+      :gems_collection,
+      :stored_gems
+    )
 
-    ##
-    # Cache class to store marshal and data into files
-    #
-    # @return [Gemirro::Cache]
-    #
-    def self.cache
-      @cache ||= Gemirro::Cache
-                 .new(File.join(configuration.destination, '.cache'))
-    end
+    URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/
+    GEMSPEC_TYPE = 'gemspec.rz'
+    GEM_TYPE = 'gem'
 
     ##
-    # Generate Gems collection from Marshal dump
+    # Generate Gems collection from Marshal dump - always the .local file
     #
-    # @param [TrueClass|FalseClass] orig Fetch orig files
     # @return [Gemirro::GemVersionCollection]
     #
-    def self.gems_collection(orig = true)
-      @gems_collection = {} if @gems_collection.nil?
-
-      is_orig = orig ? 1 : 0
-      data = @gems_collection[is_orig]
-      data = { files: {}, values: nil } if data.nil?
-
-      file_paths = specs_files_paths(orig)
-      has_file_changed = false
-      Parallel.map(file_paths, in_threads: 4) do |file_path|
-        next if data[:files].key?(file_path) &&
-                data[:files][file_path] == File.mtime(file_path)
-
-        has_file_changed = true
-      end
+    def self.gems_collection
+      @gems_collection ||= { files: {}, values: nil }
+
+      file_paths =
+        %i[specs prerelease_specs].collect do |specs_file_type|
+          File.join(
+            configuration.destination,
+            "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz.local"
+          )
+        end
+
+      has_file_changed =
+        @gems_collection[:files] != file_paths.each_with_object({}) do |f, r|
+          r[f] = File.mtime(f) if File.exist?(f)
+        end
 
       # Return result if no file changed
-      return data[:values] if !has_file_changed && !data[:values].nil?
+      return @gems_collection[:values] if !has_file_changed && !@gems_collection[:values].nil?
 
       gems = []
-      Parallel.map(file_paths, in_threads: 4) do |file_path|
+
+      # parallel is not for mtime, it's for the Marshal.
+      Parallel.map(file_paths, in_threads: Utils.configuration.update_thread_count) do |file_path|
         next unless File.exist?(file_path)
 
         gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read))
-        data[:files][file_path] = File.mtime(file_path)
+        @gems_collection[:files][file_path] = File.mtime(file_path)
       end
 
-      collection = GemVersionCollection.new(gems)
-      data[:values] = collection
-
-      collection
-    end
-
-    ##
-    # Return specs fils paths
-    #
-    # @param [TrueClass|FalseClass] orig Fetch orig files
-    # @return [Array]
-    #
-    def self.specs_files_paths(orig = true)
-      marshal_version = Gemirro::Configuration.marshal_version
-      Parallel.map(specs_file_types, in_threads: 4) do |specs_file_type|
-        File.join(configuration.destination,
-                  [specs_file_type,
-                   marshal_version,
-                   "gz#{orig ? '.orig' : ''}"].join('.'))
-      end
-    end
-
-    ##
-    # Return specs fils types
-    #
-    # @return [Array]
-    #
-    def self.specs_file_types
-      %i[specs prerelease_specs]
+      @gems_collection[:values] = GemVersionCollection.new(gems)
     end
 
     ##
@@ -112,8 +80,7 @@ module Gemirro
     # @see Gemirro::VersionsFetcher.fetch
     #
     def self.versions_fetcher
-      @versions_fetcher ||= Gemirro::VersionsFetcher
-                            .new(configuration.source).fetch
+      @versions_fetcher ||= Gemirro::VersionsFetcher.new(configuration.source).fetch
     end
 
     ##
@@ -133,13 +100,101 @@ module Gemirro
     def self.stored_gem(gem_name, gem_version, platform = 'ruby')
       platform = 'ruby' if platform.nil?
       @stored_gems ||= {}
-      # rubocop:disable Metrics/LineLength
       @stored_gems[gem_name] = {} unless @stored_gems.key?(gem_name)
       @stored_gems[gem_name][gem_version] = {} unless @stored_gems[gem_name].key?(gem_version)
-      @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform) unless @stored_gems[gem_name][gem_version].key?(platform)
-      # rubocop:enable Metrics/LineLength
+      unless @stored_gems[gem_name][gem_version].key?(platform)
+        @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform)
+      end
 
       @stored_gems[gem_name][gem_version][platform]
     end
+
+    ##
+    # Return gem specification from gemname and version
+    #
+    # @param [String] gemname
+    # @param [String] version
+    # @return [::Gem::Specification]
+    #
+    def self.spec_for(gemname, version, platform)
+      gem = Utils.stored_gem(gemname, version.to_s, platform)
+
+      spec_file =
+        File.join(
+          configuration.destination,
+          'quick',
+          Gemirro::Configuration.marshal_identifier,
+          gem.gemspec_filename
+        )
+
+      fetch_gem(spec_file) unless File.exist?(spec_file)
+
+      # this is a separate action
+      return unless File.exist?(spec_file)
+
+      File.open(spec_file, 'r') do |uz_file|
+        uz_file.binmode
+        inflater = Zlib::Inflate.new
+        begin
+          inflate_data = inflater.inflate(uz_file.read)
+        ensure
+          inflater.finish
+          inflater.close
+        end
+
+        Marshal.load(inflate_data)
+      end
+    end
+
+    ##
+    # Try to fetch gem and download its if it's possible, and
+    # build and install indicies.
+    #
+    # @param [String] resource
+    # @return [Indexer]
+    #
+    def self.fetch_gem(resource)
+      return unless Utils.configuration.fetch_gem
+
+      name = File.basename(resource)
+      result = name.match(URI_REGEXP)
+      return unless result
+
+      gem_name, gem_version, gem_platform, gem_type = result.captures
+      return unless gem_name && gem_version
+
+      begin
+        gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
+        gem.gemspec = true if gem_type == GEMSPEC_TYPE
+
+        return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
+        return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
+
+        Utils.logger.info("Try to download #{gem_name} with version #{gem_version}")
+        Utils.gems_fetcher.source.gems.clear
+        Utils.gems_fetcher.source.gems.push(gem)
+        Utils.gems_fetcher.fetch
+
+        update_indexes if Utils.configuration.update_on_fetch
+      rescue StandardError => e
+        Utils.logger.error(e)
+      end
+    end
+
+    ##
+    # Update indexes files
+    #
+    # @return [Indexer]
+    #
+    def self.update_indexes
+      indexer = Gemirro::Indexer.new(Utils.configuration.destination)
+      indexer.only_origin = true
+      indexer.ui = ::Gem::SilentUI.new
+
+      Utils.logger.info('Generating indexes')
+      indexer.update_index
+    rescue SystemExit => e
+      Utils.logger.info(e.message)
+    end
   end
 end

data/lib/gemirro/version.rb

@@ -2,5 +2,5 @@
 
 # Gemirro Version
 module Gemirro
-  VERSION = '1.5.0'
+  VERSION = '1.6.0'
 end

data/lib/gemirro/versions_fetcher.rb

@@ -22,8 +22,10 @@ module Gemirro
     # @return [Gemirro::VersionsFile]
     #
     def fetch
-      VersionsFile.load(read_file(Configuration.versions_file),
-                        read_file(Configuration.prerelease_versions_file, true))
+      VersionsFile.load(
+        read_file(Configuration.versions_file),
+        read_file(Configuration.prerelease_versions_file, true)
+      )
     end
 
     ##
@@ -36,6 +38,8 @@ module Gemirro
       destination = Gemirro.configuration.destination
       file_dst = File.join(destination, file)
       unless File.exist?(file_dst)
+        throw 'No source defined' unless @source
+
         File.write(file_dst, @source.fetch_versions) unless prerelease
         File.write(file_dst, @source.fetch_prerelease_versions) if prerelease
       end

data/lib/gemirro.rb

@@ -20,7 +20,6 @@ $LOAD_PATH.unshift(File.expand_path('../', __FILE__)) unless $LOAD_PATH.include?
 
 require 'gemirro/version'
 require 'gemirro/configuration'
-require 'gemirro/cache'
 require 'gemirro/utils'
 require 'gemirro/gem'
 require 'gemirro/gem_version'