gemirro 1.5.0 → 2.0.0

data/lib/gemirro/server.rb

@@ -4,18 +4,13 @@ require 'sinatra/base'
 require 'thin'
 require 'uri'
 require 'addressable/uri'
+require 'base64'
 
 module Gemirro
   ##
   # Launch Sinatra server to easily download gems.
   #
   class Server < Sinatra::Base
-    # rubocop:disable Layout/LineLength
-    URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/.freeze
-    # rubocop:enable Layout/LineLength
-    GEMSPEC_TYPE = 'gemspec.rz'
-    GEM_TYPE = 'gem'
-
     access_logger = Logger.new(Utils.configuration.server.access_log).tap do |logger|
       ::Logger.class_eval { alias_method :write, :<< }
       logger.level = ::Logger::INFO
@@ -62,7 +57,7 @@ module Gemirro
     end
 
     ##
-    # Display information about one gem
+    # Display information about one gem, human readable
     #
     # @return [nil]
     #
@@ -76,7 +71,7 @@ module Gemirro
 
     ##
     # Display home page containing the list of gems already
-    # downloaded on the server
+    # downloaded on the server, human readable
     #
     # @return [nil]
     #
@@ -85,212 +80,92 @@ module Gemirro
     end
 
     ##
-    # Return gem dependencies as binary
+    # compact_index, Return list of available gem names
     #
     # @return [nil]
     #
-    get '/api/v1/dependencies' do
-      content_type 'application/octet-stream'
-      query_gems.any? ? Marshal.dump(query_gems_list) : 200
-    end
+    get '/names' do
+      content_type 'text/plain'
 
-    ##
-    # Return gem dependencies as json
-    #
-    # @return [nil]
-    #
-    get '/api/v1/dependencies.json' do
-      content_type 'application/json'
-      query_gems.any? ? JSON.dump(query_gems_list) : {}
+      content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
+
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
+      send_file content_path
     end
 
     ##
-    # Try to get all request and download files
-    # if files aren't found.
+    # compact_index, Return list of gem, including versions
     #
     # @return [nil]
     #
-    get('*') do |path|
-      resource = "#{settings.public_folder}#{path}"
+    get '/versions' do
+      content_type 'text/plain'
 
-      # Try to download gem
-      fetch_gem(resource) unless File.exist?(resource)
-      # If not found again, return a 404
-      return not_found unless File.exist?(resource)
+      content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      send_file(resource)
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
+      send_file content_path
     end
 
-    ##
-    # Try to fetch gem and download its if it's possible, and
-    # build and install indicies.
+    # compact_index, Return gem dependencies for all versions of a gem
     #
-    # @param [String] resource
-    # @return [Indexer]
+    # @return [nil]
     #
-    def fetch_gem(resource)
-      return unless Utils.configuration.fetch_gem
-
-      name = File.basename(resource)
-      result = name.match(URI_REGEXP)
-      return unless result
-
-      gem_name, gem_version, gem_platform, gem_type = result.captures
-      return unless gem_name && gem_version
-
-      begin
-        gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
-        gem.gemspec = true if gem_type == GEMSPEC_TYPE
-
-        return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
-        return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
-
-        Utils.logger
-             .info("Try to download #{gem_name} with version #{gem_version}")
-        Utils.gems_fetcher.source.gems.clear
-        Utils.gems_fetcher.source.gems.push(gem)
-        Utils.gems_fetcher.fetch
+    get('/info/:gemname') do
+      gems = Utils.gems_collection
+      gem = gems.find_by_name(params[:gemname])
+      return not_found if gem.nil?
 
-        update_indexes if Utils.configuration.update_on_fetch
-      rescue StandardError => e
-        Utils.logger.error(e)
-      end
-    end
+      content_type 'text/plain'
 
-    ##
-    # Update indexes files
-    #
-    # @return [Indexer]
-    #
-    def update_indexes
-      indexer = Gemirro::Indexer.new(Utils.configuration.destination)
-      indexer.only_origin = true
-      indexer.ui = ::Gem::SilentUI.new
+      content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last
+      _, etag, repr_digest, _ = File.basename(content_path).split('.')
 
-      Utils.logger.info('Generating indexes')
-      indexer.update_index
-      indexer.updated_gems.each do |gem|
-        Utils.cache.flush_key(File.basename(gem))
-      end
-    rescue SystemExit => e
-      Utils.logger.info(e.message)
+      headers 'etag' => %("#{etag}")
+      headers 'repr-digest' => %(sha-256=#{Base64.strict_encode64([repr_digest].pack('H*'))})
+      send_file content_path
     end
 
     ##
-    # Return all gems pass to query
+    # Try to get all request and download files
+    # if files aren't found.
     #
-    # @return [Array]
+    # @return [nil]
     #
-    def query_gems
-      params[:gems].to_s.split(',')
-    end
+    get('*') do |path|
+      resource = "#{settings.public_folder}#{path}"
 
-    ##
-    # Return gems list from query params
-    #
-    # @return [Array]
-    #
-    def query_gems_list
-      Utils.gems_collection(false) # load collection
-      gems = Parallel.map(query_gems, in_threads: 4) do |query_gem|
-        gem_dependencies(query_gem)
-      end
+      # Try to download gem
+      Gemirro::Utils.fetch_gem(resource) unless File.exist?(resource)
+      # If not found again, return a 404
+      return not_found unless File.exist?(resource)
 
-      gems.flatten!
-      gems.reject!(&:empty?)
-      gems
+      send_file(resource)
     end
 
     ##
-    # List of versions and dependencies of each version
-    # from a gem name.
+    # Compile fragments for /api/v1/dependencies
     #
-    # @return [Array]
+    # @return [nil]
     #
-    def gem_dependencies(gem_name)
-      Utils.cache.cache(gem_name) do
-        gems = Utils.gems_collection(false)
-        gem_collection = gems.find_by_name(gem_name)
-
-        return '' if gem_collection.nil?
-
-        gem_collection = Parallel.map(gem_collection, in_threads: 4) do |gem|
-          [gem, spec_for(gem.name, gem.number, gem.platform)]
-        end
-        gem_collection.compact!
-
-        Parallel.map(gem_collection, in_threads: 4) do |gem, spec|
-          next if spec.nil?
-
-          dependencies = spec.dependencies.select do |d|
-            d.type == :runtime
-          end
-
-          dependencies = Parallel.map(dependencies, in_threads: 4) do |d|
-            [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s]
-          end
-
-          {
-            name: gem.name,
-            number: gem.number,
-            platform: gem.platform,
-            dependencies: dependencies
-          }
-        end
-      end
-    end
-
-    helpers do
-      ##
-      # Return gem specification from gemname and version
-      #
-      # @param [String] gemname
-      # @param [String] version
-      # @return [::Gem::Specification]
-      #
-      def spec_for(gemname, version, platform = 'ruby')
-        gem = Utils.stored_gem(gemname, version.to_s, platform)
-        gemspec_path = File.join('quick',
-                                 Gemirro::Configuration.marshal_identifier,
-                                 gem.gemspec_filename)
-        spec_file = File.join(settings.public_folder,
-                              gemspec_path)
-        fetch_gem(gemspec_path) unless File.exist?(spec_file)
-
-        return unless File.exist?(spec_file)
-
-        File.open(spec_file, 'r') do |uz_file|
-          uz_file.binmode
-          inflater = Zlib::Inflate.new
-          begin
-            inflate_data = inflater.inflate(uz_file.read)
-          ensure
-            inflater.finish
-            inflater.close
-          end
-          Marshal.load(inflate_data)
+    def dependencies_loader(names)
+      names.collect do |name|
+        f = File.join(settings.public_folder, 'api', 'v1', 'dependencies', "#{name}.*.*.list")
+        Marshal.load(File.read(Dir.glob(f).last))
+      rescue StandardError => e
+        env['rack.errors'].write "Cound not open #{f}\n"
+        env['rack.errors'].write "#{e.message}\n"
+        e.backtrace.each do |err|
+          env['rack.errors'].write "#{err}\n"
         end
+        nil
       end
-
-      ##
-      # Escape string
-      #
-      # @param [String] string
-      # @return [String]
-      #
-      def escape(string)
-        Rack::Utils.escape_html(string)
-      end
-
-      ##
-      # Homepage link
-      #
-      # @param [Gem] spec
-      # @return [String]
-      #
-      def homepage(spec)
-        URI.parse(Addressable::URI.escape(spec.homepage))
-      end
+      .flatten
+      .compact
     end
   end
 end

data/lib/gemirro/source.rb

@@ -33,23 +33,10 @@ module Gemirro
     #
     def fetch_versions
       Utils.logger.info(
-        "Fetching #{Configuration.versions_file} on #{@name} (#{@host})"
+        "Fetching versions on #{@name} (#{@host})"
       )
 
-      Http.get("#{host}/#{Configuration.versions_file}").body
-    end
-
-    ##
-    # Fetches a list of all the available Gems and their versions.
-    #
-    # @return [String]
-    #
-    def fetch_prerelease_versions
-      Utils.logger.info(
-        "Fetching #{Configuration.prerelease_versions_file} " \
-        "on #{@name} (#{@host})"
-      )
-      Http.get("#{host}/#{Configuration.prerelease_versions_file}").body
+      Http.get("#{host}/versions").body
     end
 
     ##

data/lib/gemirro/utils.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'gemirro/gem_version'
+
 module Gemirro
   ##
   # The Utils class is responsible for executing specific traitments
@@ -13,84 +15,60 @@ module Gemirro
   #  @return [Gemirro::GemsFetcher]
   #
   class Utils
-    attr_reader(:cache,
-                :versions_fetcher,
-                :gems_fetcher,
-                :gems_collection,
-                :stored_gems)
+    attr_reader(
+      :versions_fetcher,
+      :gems_fetcher,
+      :gems_collection,
+      :stored_gems
+    )
 
-    ##
-    # Cache class to store marshal and data into files
-    #
-    # @return [Gemirro::Cache]
-    #
-    def self.cache
-      @cache ||= Gemirro::Cache
-                 .new(File.join(configuration.destination, '.cache'))
-    end
+    URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/
+    GEMSPEC_TYPE = 'gemspec.rz'
+    GEM_TYPE = 'gem'
 
     ##
-    # Generate Gems collection from Marshal dump
+    # Generate Gems collection from Marshal dump - always the .local file
     #
-    # @param [TrueClass|FalseClass] orig Fetch orig files
     # @return [Gemirro::GemVersionCollection]
     #
-    def self.gems_collection(orig = true)
-      @gems_collection = {} if @gems_collection.nil?
+    def self.gems_collection
+      @gems_collection ||= { files: {}, values: nil }
 
-      is_orig = orig ? 1 : 0
-      data = @gems_collection[is_orig]
-      data = { files: {}, values: nil } if data.nil?
+      file_paths =
+        Dir.glob(File.join(
+                   Gemirro.configuration.destination,
+                   'versions.*.*.list'
+                 ))
 
-      file_paths = specs_files_paths(orig)
-      has_file_changed = false
-      Parallel.map(file_paths, in_threads: 4) do |file_path|
-        next if data[:files].key?(file_path) &&
-                data[:files][file_path] == File.mtime(file_path)
-
-        has_file_changed = true
-      end
+      has_file_changed =
+        @gems_collection[:files] != file_paths.each_with_object({}) do |f, r|
+          r[f] = File.mtime(f) if File.exist?(f)
+        end
 
       # Return result if no file changed
-      return data[:values] if !has_file_changed && !data[:values].nil?
+      return @gems_collection[:values] if !has_file_changed && !@gems_collection[:values].nil?
 
       gems = []
-      Parallel.map(file_paths, in_threads: 4) do |file_path|
-        next unless File.exist?(file_path)
 
-        gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read))
-        data[:files][file_path] = File.mtime(file_path)
-      end
+      CompactIndex::VersionsFile.new(file_paths.last).contents.each_line.with_index do |line, index|
+        next if index < 2
 
-      collection = GemVersionCollection.new(gems)
-      data[:values] = collection
+        gem_name = line.split[0]
+        versions = line.split[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
 
-      collection
-    end
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
 
-    ##
-    # Return specs fils paths
-    #
-    # @param [TrueClass|FalseClass] orig Fetch orig files
-    # @return [Array]
-    #
-    def self.specs_files_paths(orig = true)
-      marshal_version = Gemirro::Configuration.marshal_version
-      Parallel.map(specs_file_types, in_threads: 4) do |specs_file_type|
-        File.join(configuration.destination,
-                  [specs_file_type,
-                   marshal_version,
-                   "gz#{orig ? '.orig' : ''}"].join('.'))
+          gems << Gemirro::GemVersion.new(gem_name, version, platform)
+        end
       end
-    end
 
-    ##
-    # Return specs fils types
-    #
-    # @return [Array]
-    #
-    def self.specs_file_types
-      %i[specs prerelease_specs]
+      @gems_collection[:values] = GemVersionCollection.new(gems)
     end
 
     ##
@@ -112,8 +90,7 @@ module Gemirro
     # @see Gemirro::VersionsFetcher.fetch
     #
     def self.versions_fetcher
-      @versions_fetcher ||= Gemirro::VersionsFetcher
-                            .new(configuration.source).fetch
+      @versions_fetcher ||= Gemirro::VersionsFetcher.new(configuration.source).fetch
     end
 
     ##
@@ -133,13 +110,101 @@ module Gemirro
     def self.stored_gem(gem_name, gem_version, platform = 'ruby')
       platform = 'ruby' if platform.nil?
       @stored_gems ||= {}
-      # rubocop:disable Metrics/LineLength
       @stored_gems[gem_name] = {} unless @stored_gems.key?(gem_name)
       @stored_gems[gem_name][gem_version] = {} unless @stored_gems[gem_name].key?(gem_version)
-      @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform) unless @stored_gems[gem_name][gem_version].key?(platform)
-      # rubocop:enable Metrics/LineLength
+      unless @stored_gems[gem_name][gem_version].key?(platform)
+        @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform)
+      end
 
       @stored_gems[gem_name][gem_version][platform]
     end
+
+    ##
+    # Return gem specification from gemname and version
+    #
+    # @param [String] gemname
+    # @param [String] version
+    # @return [::Gem::Specification]
+    #
+    def self.spec_for(gemname, version, platform)
+      gem = Utils.stored_gem(gemname, version.to_s, platform)
+
+      spec_file =
+        File.join(
+          configuration.destination,
+          'quick',
+          Gemirro::Configuration.marshal_identifier,
+          gem.gemspec_filename
+        )
+
+      fetch_gem(spec_file) unless File.exist?(spec_file)
+
+      # this is a separate action
+      return unless File.exist?(spec_file)
+
+      File.open(spec_file, 'r') do |uz_file|
+        uz_file.binmode
+        inflater = Zlib::Inflate.new
+        begin
+          inflate_data = inflater.inflate(uz_file.read)
+        ensure
+          inflater.finish
+          inflater.close
+        end
+
+        Marshal.load(inflate_data)
+      end
+    end
+
+    ##
+    # Try to fetch gem and download its if it's possible, and
+    # build and install indicies.
+    #
+    # @param [String] resource
+    # @return [Indexer]
+    #
+    def self.fetch_gem(resource)
+      return unless Utils.configuration.fetch_gem
+
+      name = File.basename(resource)
+      result = name.match(URI_REGEXP)
+      return unless result
+
+      gem_name, gem_version, gem_platform, gem_type = result.captures
+      return unless gem_name && gem_version
+
+      begin
+        gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
+        gem.gemspec = true if gem_type == GEMSPEC_TYPE
+
+        return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
+        return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
+
+        Utils.logger.info("Try to download #{gem_name} with version #{gem_version}")
+        Utils.gems_fetcher.source.gems.clear
+        Utils.gems_fetcher.source.gems.push(gem)
+        Utils.gems_fetcher.fetch
+
+        update_indexes if Utils.configuration.update_on_fetch
+      rescue StandardError => e
+        Utils.logger.error(e)
+      end
+    end
+
+    ##
+    # Update indexes files
+    #
+    # @return [Indexer]
+    #
+    def self.update_indexes
+      indexer = Gemirro::Indexer.new
+      indexer.only_origin = true
+      indexer.ui = ::Gem::SilentUI.new
+
+      Utils.logger.info('Generating indexes')
+      indexer.update_index
+    rescue SystemExit => e
+      Utils.logger.info(e.message)
+    end
   end
 end

data/lib/gemirro/version.rb

@@ -2,5 +2,5 @@
 
 # Gemirro Version
 module Gemirro
-  VERSION = '1.5.0'
+  VERSION = '2.0.0'
 end

data/lib/gemirro/versions_fetcher.rb

@@ -22,25 +22,24 @@ module Gemirro
     # @return [Gemirro::VersionsFile]
     #
     def fetch
-      VersionsFile.load(read_file(Configuration.versions_file),
-                        read_file(Configuration.prerelease_versions_file, true))
+      return unless Gemirro.configuration.versions_file
+
+      VersionsFile.new(read_file(Gemirro.configuration.versions_file))
     end
 
     ##
     # Read file if exists otherwise download its from source
     #
     # @param [String] file name
-    # @param [TrueClass|FalseClass] prerelease Is prerelease or not
     #
-    def read_file(file, prerelease = false)
-      destination = Gemirro.configuration.destination
-      file_dst = File.join(destination, file)
-      unless File.exist?(file_dst)
-        File.write(file_dst, @source.fetch_versions) unless prerelease
-        File.write(file_dst, @source.fetch_prerelease_versions) if prerelease
+    def read_file(file)
+      unless File.exist?(file)
+        throw 'No source defined' unless @source
+
+        File.write(file, @source.fetch_versions)
       end
 
-      File.read(file_dst)
+      File.read(file)
     end
   end
 end

data/lib/gemirro/versions_file.rb

@@ -11,36 +11,24 @@ module Gemirro
   #  @return [Hash]
   #
   class VersionsFile
-    attr_reader :versions, :versions_hash
+    attr_reader :versions_string, :versions_hash
 
     ##
     # Reads the versions file from the specified String.
     #
-    # @param [String] spec_content
-    # @param [String] prerelease_content
+    # @param [String] versions_content
     # @return [Gemirro::VersionsFile]
     #
-    def self.load(spec_content, prerelease_content)
-      buffer = StringIO.new(spec_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions = Marshal.load(reader.read)
-
-      buffer = StringIO.new(prerelease_content)
-      reader = Zlib::GzipReader.new(buffer)
-      versions.concat(Marshal.load(reader.read))
-
-      instance = new(versions)
-
-      reader.close
-
-      instance
-    end
 
     ##
-    # @param [Array] versions
+    # @param [String] versions
     #
-    def initialize(versions)
-      @versions      = versions
+    def initialize(versions_string)
+      unless versions_string.is_a? String
+        throw "#{versions_string.class} is wrong format, expect String; #{versions_string.inspect}"
+      end
+
+      @versions_string = versions_string
       @versions_hash = create_versions_hash
     end
 
@@ -53,10 +41,24 @@ module Gemirro
     def create_versions_hash
       hash = Hash.new { |h, k| h[k] = [] }
 
-      versions.each do |version|
-        hash[version[0]] << version
-      end
+      versions_string.each_line.with_index do |line, index|
+        next if index < 2
 
+        parts = line.split
+        gem_name = parts[0]
+        parts[-1]
+        versions = parts[1..-2].collect { |x| x.split(',') }.flatten # All except first and last
+
+        versions.each do |ver|
+          version, platform =
+            if ver.include?('-')
+              ver.split('-', 2)
+            else
+              [ver, 'ruby']
+            end
+          hash[gem_name] << [gem_name, ::Gem::Version.new(version), platform]
+        end
+      end
       hash
     end