geminabox 3.0.0 → 4.0.0

data/lib/geminabox/gem_version_collection.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rubygems/util'
+
 module Geminabox
 
   # This class represents a sorted collection of Geminabox::GemVersion objects.
@@ -7,6 +9,16 @@ module Geminabox
   class GemVersionCollection
     include Enumerable
 
+    # Loads the collection recorded in the legacy Marshal specs indexes
+    # (specs.4.8.gz and prerelease_specs.4.8.gz) under data_dir.
+    def self.from_specs_index(data_dir)
+      specs = %w[specs prerelease_specs].flat_map do |type|
+        path = File.join(data_dir, "#{type}.#{Gem.marshal_version}.gz")
+        File.exist?(path) ? Marshal.load(Gem::Util.gunzip(Gem.read_binary(path))) : []
+      end
+      new(specs.uniq)
+    end
+
     # Array of Geminabox::GemVersion objects, or an array of [name, version,
     # platform] triples.
     def initialize(initial_gems=[])

data/lib/geminabox/hostess.rb

@@ -23,7 +23,10 @@ module Geminabox
     end
 
     get '/quick/Marshal.4.8/*.gemspec.rz' do
-      content_type('application/x-deflate')
+      # rubygems.org serves the deflated quick specs as octet-stream, and the
+      # compact-index conformance suite asserts it; clients inflate the body
+      # regardless of this header.
+      content_type('application/octet-stream')
       serve
     end
 

data/lib/geminabox/server.rb

@@ -2,6 +2,7 @@
 
 require 'reentrant_flock'
 require 'rubygems/util'
+require 'rss'
 
 module Geminabox
 
@@ -11,11 +12,7 @@ module Geminabox
     set :views, Geminabox.views
     set :host_authorization, { permitted_hosts: [] }
 
-    if Geminabox.rubygems_proxy
-      use Proxy::Hostess
-    else
-      use Hostess
-    end
+    use Hostess
 
     class << self
       def disallow_replace?
@@ -42,6 +39,7 @@ module Geminabox
         if force_rebuild
           indexer.generate_index
           dependency_cache.flush
+          compact_indexer.reindex
         else
           begin
             require 'geminabox/indexer'
@@ -49,6 +47,7 @@ module Geminabox
             return if updated_gemspecs.empty?
             indexer.update_index
             updated_gemspecs.each { |gem| dependency_cache.flush_key(gem.name) }
+            compact_indexer.reindex
           rescue Errno::ENOENT
             with_rlock { reindex(:force_rebuild) }
           rescue => e
@@ -68,6 +67,10 @@ module Geminabox
         @dependency_cache ||= Geminabox::DiskCache.new(File.join(Geminabox.data, "_cache"))
       end
 
+      def compact_indexer
+        Geminabox::CompactIndexer.new(Geminabox.data)
+      end
+
       def with_rlock(&block)
         file_class.open(Geminabox.lockfile, File::RDWR | File::CREAT) do |f|
           ReentrantFlock.synchronize(f, File::LOCK_EX | File::LOCK_NB, &block)
@@ -90,6 +93,7 @@ module Geminabox
     end
 
     get '/' do
+      content_type :html
       @gems = load_gems
       @index_gems = index_gems(@gems)
       @allow_upload = self.class.allow_upload?
@@ -98,15 +102,40 @@ module Geminabox
     end
 
     get '/atom.xml' do
+      content_type 'application/atom+xml'
       @gems = load_gems
       erb :atom, :layout => false
     end
 
+    helpers Geminabox::CompactIndexApi
+
+    get '/versions' do
+      bootstrap_compact_index(self.class.compact_indexer.versions_path)
+      serve_compact_file(self.class.compact_indexer.versions_path)
+    end
+
+    get '/names' do
+      bootstrap_compact_index(self.class.compact_indexer.names_path)
+      serve_compact_file(self.class.compact_indexer.names_path)
+    end
+
+    get '/info/:name' do
+      # Reject anything that is not a gem-name shape before touching the
+      # filesystem. This also blocks NUL bytes and empty names, which would
+      # otherwise raise ArgumentError from File.file? and 500.
+      halt 404 unless params[:name] =~ /\A[a-zA-Z0-9_.-]+\z/
+      path = self.class.compact_indexer.info_path(params[:name])
+      heal_missing_info(params[:name], path) unless File.file?(path)
+      serve_compact_file(path)
+    end
+
     get '/api/v1/dependencies' do
+      content_type 'application/octet-stream'
       query_gems.any? ? Marshal.dump(gem_list) : 200
     end
 
     get '/api/v1/dependencies.json' do
+      content_type :json
       query_gems.any? ? gem_list.to_json : {}
     end
 
@@ -115,6 +144,7 @@ module Geminabox
         error_response(403, 'Gem uploading is disabled')
       end
 
+      content_type :html
       erb :upload
     end
 
@@ -135,6 +165,7 @@ module Geminabox
       @gem = gems[params[:gemname]]
       @allow_delete = self.class.allow_delete?
       halt 404 unless @gem
+      content_type :html
       erb :gem
     end
 
@@ -159,8 +190,16 @@ module Geminabox
       halt 400 unless request.form_data?
 
       serialize_update do
-        gems = load_gems.select { |gem| params['gem_name'] == gem.name and
-                                  params['version'] == gem.number.version }
+        # A yank targets one platform. The client omits the platform param for
+        # a plain-ruby gem, so an absent/blank value means the ruby platform;
+        # without this a `gem yank foo -v 1.0.0` would also destroy foo's java
+        # and mingw builds sharing that version.
+        requested_platform = params['platform'].to_s.empty? ? 'ruby' : params['platform']
+        gems = load_gems.select do |gem|
+          params['gem_name'] == gem.name &&
+            params['version'] == gem.number.version &&
+            (gem.ruby? ? 'ruby' : gem.platform) == requested_platform
+        end
         halt 404, 'Gem not found' if gems.size == 0
         gems.each do |gem|
           gem_path = File.expand_path(File.join(Geminabox.data, 'gems',
@@ -215,6 +254,22 @@ module Geminabox
       self.class.with_rlock(&block)
     end
 
+    def bootstrap_compact_index(path)
+      return if File.exist?(path)
+      serialize_update { self.class.compact_indexer.reindex }
+    end
+
+    # Rebuild info/NAME on the read path when it is missing but versions.list
+    # still lists the gem, so a partially corrupted index does not 404 a gem
+    # Bundler was told exists. The ledger-membership check reads only
+    # versions.list and takes no lock, so an unknown name stays a cheap 404
+    # with no per-gem stats -- arbitrary /info probes cannot force writes.
+    def heal_missing_info(name, path)
+      indexer = self.class.compact_indexer
+      return unless indexer.ledger_lists?(name)
+      serialize_update { indexer.heal_info(name) unless File.file?(path) }
+    end
+
     def handle_incoming_gem(gem)
       begin
         GemStore.create(gem, params[:overwrite])
@@ -261,36 +316,8 @@ HTML
       self.class.dependency_cache
     end
 
-    def all_gems
-      all_gems_with_duplicates.inject(:|)
-    end
-
-    def all_gems_with_duplicates
-      specs_files_paths.map do |specs_file_path|
-        if File.exist?(specs_file_path)
-          Marshal.load(Gem::Util.gunzip(Gem.read_binary(specs_file_path)))
-        else
-          []
-        end
-      end
-    end
-
-    def specs_file_types
-      [:specs, :prerelease_specs]
-    end
-
-    def specs_files_paths
-      specs_file_types.map do |specs_file_type|
-        File.join(Geminabox.data, spec_file_name(specs_file_type))
-      end
-    end
-
-    def spec_file_name(specs_file_type)
-      [specs_file_type, Gem.marshal_version, 'gz'].join('.')
-    end
-
     def load_gems
-      @loaded_gems ||= Geminabox::GemVersionCollection.new(all_gems)
+      @loaded_gems ||= Geminabox::GemVersionCollection.from_specs_index(Geminabox.data)
     end
 
     def index_gems(gems)
@@ -298,25 +325,13 @@ HTML
     end
 
     def gem_list
-      Geminabox.rubygems_proxy ? combined_gem_list : local_gem_list
+      query_gems.map{|query_gem| gem_dependencies(query_gem) }.flatten(1)
     end
 
     def query_gems
       params[:gems].to_s.split(',')
     end
 
-    def local_gem_list
-      query_gems.map{|query_gem| gem_dependencies(query_gem) }.flatten(1)
-    end
-
-    def remote_gem_list
-      RubygemsDependency.for(*query_gems)
-    end
-
-    def combined_gem_list
-      GemListMerge.merge(local_gem_list, remote_gem_list, strategy: Geminabox.rubygems_proxy_merge_strategy)
-    end
-
     helpers do
       def href(text)
         if text && (text.start_with?('http://') || text.start_with?('https://'))

data/lib/geminabox/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Geminabox
-  VERSION = '3.0.0' unless defined? VERSION
+  VERSION = '4.0.0' unless defined? VERSION
 end

data/lib/geminabox.rb

@@ -17,7 +17,6 @@ module Geminabox
   class Error < StandardError ; end
 
   require_relative 'geminabox/version'
-  require_relative 'geminabox/proxy'
   require_relative 'geminabox/http_adapter'
 
   def self.geminabox_path(file)
@@ -27,10 +26,10 @@ module Geminabox
   autoload :Hostess,                geminabox_path('hostess')
   autoload :GemStore,               geminabox_path('gem_store')
   autoload :GemStoreError,          geminabox_path('gem_store_error')
-  autoload :RubygemsDependency,     geminabox_path('rubygems_dependency')
-  autoload :GemListMerge,           geminabox_path('gem_list_merge')
   autoload :GemVersion,             geminabox_path('gem_version')
   autoload :GemVersionCollection,   geminabox_path('gem_version_collection')
+  autoload :CompactIndexer,         geminabox_path('compact_indexer')
+  autoload :CompactIndexApi,        geminabox_path('compact_index_api')
   autoload :Server,                 geminabox_path('server')
   autoload :DiskCache,              geminabox_path('disk_cache')
   autoload :IncomingGem,            geminabox_path('incoming_gem')
@@ -45,14 +44,9 @@ module Geminabox
       :allow_replace,
       :gem_permissions,
       :allow_delete,
-      :rubygems_proxy,
-      :rubygems_proxy_merge_strategy,
       :http_adapter,
       :lockfile,
       :retry_interval,
-      :allow_remote_failure,
-      :ruby_gems_url,
-      :bundler_ruby_gems_url,
       :allow_upload,
       :on_gem_received
     )
@@ -80,17 +74,38 @@ module Geminabox
     views:                          File.join(File.dirname(__FILE__), *%w[.. views]),
     allow_replace:                  false,
     gem_permissions:                0644,
-    rubygems_proxy:                 (ENV['RUBYGEMS_PROXY'] == 'true'),
-    rubygems_proxy_merge_strategy:  ENV.fetch('RUBYGEMS_PROXY_MERGE_STRATEGY') { :local_gems_take_precedence_over_remote_gems }.to_sym,
     allow_delete:                   true,
     http_adapter:                   HttpClientAdapter.new,
     lockfile:                       File.join(ENV.fetch('TMPDIR', Dir.tmpdir), 'geminabox.lockfile'),
     retry_interval:                 60,
-    allow_remote_failure:           false,
-    ruby_gems_url:                  'https://rubygems.org/',
-    bundler_ruby_gems_url:          'https://bundler.rubygems.org/',
     allow_upload:                   true,
     on_gem_received:                nil
   )
 
+  # Removed in 4.0 with the RubyGems proxy. Warn-and-ignore (rather than
+  # NoMethodError at boot) so stale config.ru lines don't break upgrades.
+  # Drop these shims in 5.0.
+  [
+    :rubygems_proxy,
+    :rubygems_proxy_merge_strategy,
+    :allow_remote_failure,
+    :ruby_gems_url,
+    :bundler_ruby_gems_url
+  ].each do |setting|
+    define_singleton_method("#{setting}=") do |_value|
+      warn "[REMOVED] Geminabox.#{setting} was removed in Geminabox 4.0 " \
+           'along with the RubyGems proxy and has no effect. Remove this ' \
+           'line from your config.ru. Migration notes: ' \
+           'https://github.com/geminabox/geminabox/pull/735'
+    end
+    define_singleton_method(setting) { nil }
+  end
+
+  if ENV['RUBYGEMS_PROXY'] == 'true' || ENV.key?('RUBYGEMS_PROXY_MERGE_STRATEGY')
+    warn '[REMOVED] The RUBYGEMS_PROXY and RUBYGEMS_PROXY_MERGE_STRATEGY ' \
+         'environment variables were removed in Geminabox 4.0; Geminabox ' \
+         'now always serves only local gems. Migration notes: ' \
+         'https://github.com/geminabox/geminabox/pull/735'
+  end
+
 end