gem-repair 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de1546b65c1f9dd21344f01ffdb924dd21b1d1ad18a837645702b875f331c468
-  data.tar.gz: 20493791f64c066be2bd29303d7f846ef82d9837f0d271ad13226054f9642a73
+  metadata.gz: c060c1e8acac437d6edfaab550b3a43a7bb19bde988e3307900610ce71fe0db2
+  data.tar.gz: 2c753428a55544e7486f0acda99ca68868880a9367aeabe0994cd27c893925c3
 SHA512:
-  metadata.gz: ac1791aead1e4f8ad1ee21a824a99c0f7bf8d114ed1e12a030d951f3bdfc1bd44dde72e0b79c5958e5ede695a785f286fb47443480356e3cc030b0575d715ae7
-  data.tar.gz: 974848b02311245c9ae757046b3e2d12d4a6e059c562c0888c05d21a6170bd549ed31a0baa45227ca5835e443784792bc517facf62b0151e5022e6b4c0f0437e
+  metadata.gz: 635ff70eec357b7300bc368df9b9dedd2ec88db326b39c5748e2ac00726c97ce6b5c043d00526e34d3e5b8891380468c946dc30214d3ba484a0c118f6f3e9e08
+  data.tar.gz: 748b0a6d4396cfb10018ff025f1be733dd3444babc2c5eb88337878b45c6a7f66fd70a836e83b6310121f4b2704bee2e5cd1478dcffd6eacbd06e668746ff539

data/README.md

@@ -33,12 +33,49 @@ gem repair --jobs 2  # Use 2 threads
 ```
 
 This command will:
-1. Find all installed gems that are missing their compiled C extensions.
-2. Attempt to reinstall these gems using multiple threads.
+1. Remove the copies of built extensions left in the `lib` directory of every gem with extensions (see below).
+2. Find all installed gems that are missing their compiled C extensions.
+3. Attempt to reinstall these gems using multiple threads.
+
+Options:
+
+| Option | Effect |
+| --- | --- |
+| `-j N`, `--jobs N` | Number of parallel threads (default: 4) |
+| `--prune` | Uninstall gems whose extensions are still missing for the running ruby after repair, even when the reinstall itself failed |
+| `--aggressive-sweep` | Also remove the `test`, `spec`, `features` and `tmp` directories of every gem |
+| `-n`, `--dry-run` | Show what would be removed, repaired or uninstalled without changing anything |
+
+## Removing extension copies from lib
+
+RubyGems copies each built extension into the gem's `lib` directory as well as the per-ruby `extensions/<platform>/<abi>/` directory. RubyGems 3.6 added the `install_extension_in_lib` gemrc setting to skip that copy, but it stays on by default. `lib` comes first on the load path, so every other ruby sharing the `GEM_HOME` loads the copy built for the installing ruby and fails with a `LoadError`. The plugin removes those copies right after each install and on every `gem repair`. Copies under versioned directories such as `lib/3.3/`, which precompiled gems use to ship one extension per ABI, are left alone.
+
+### Migrating from gem-sweep
+
+This replaces the `gem sweep` command and the install hook of the gem-sweep plugin. Uninstall it so the two hooks do not run side by side:
+
+```sh
+gem uninstall gem-sweep
+```
+
+`gem sweep --missing-extensions` becomes `gem repair --prune`, `gem sweep --aggressive` becomes `gem repair --aggressive-sweep`, and `gem sweep -n` becomes `gem repair -n`.
+
+## Building for other rubies on install
+
+Several rubies can share one `GEM_HOME`, but `gem install` only compiles extensions for the ruby that runs it. The plugin hooks `Gem.post_install` and builds the extensions of the gem just installed for every other ruby it can find, so switching rubies does not leave the gem ignored with "extensions are not built".
+
+Rubies are looked up under the install roots of mise and rbenv:
+
+| Variable | Default | Rubies |
+| --- | --- | --- |
+| `MISE_DATA_DIR` | `$XDG_DATA_HOME/mise` | `installs/ruby/*` |
+| `RBENV_ROOT` | `~/.rbenv` | `versions/*` |
+
+`GEM_REPAIR_RUBIES` narrows the targets to a comma-separated list of version names, for example `GEM_REPAIR_RUBIES=3.3-dev,3.4-dev`. Set it to an empty string to turn the fan-out off. JRuby, TruffleRuby, mruby and Rubinius installs are always skipped, as is a ruby whose version the gem does not support. The fan-out does not run inside `bundle install`, though the sweep still does. It only runs for a `GEM_HOME` outside the per-ABI directories the running ruby keeps to itself, `Gem.default_dir` and `Gem.user_dir`. The hook also fires when `gem repair` or `gem pristine` reinstalls a gem.
 
 ## Development
 
-After checking out the repo, run `bundle install` to install dependencies. Then, you can run `rake test` to run the tests (though no tests are set up in this initial version).
+After checking out the repo, run `bundle install` to install dependencies. Then, you can run `rake test` to run the tests.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `gem-repair.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [a configured gem server](https://rubygems.org).
 

data/lib/gem_repair/fanout.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+
+module GemRepair
+  # Builds the extensions of a freshly installed gem for the other rubies that
+  # share its GEM_HOME, so switching rubies does not leave the gem ignored
+  # with "extensions are not built".
+  #
+  # Rubies are looked up under the install roots of mise (MISE_DATA_DIR,
+  # default XDG_DATA_HOME/mise) and rbenv (RBENV_ROOT, default ~/.rbenv).
+  # GEM_REPAIR_RUBIES narrows them to a comma-separated list of version names.
+  # An empty value disables the fan-out.
+  module Fanout
+    SKIP = /\A(jruby|truffleruby|mruby|rbx)/
+
+    # Runs inside each target ruby. Exit 2 means there was nothing to do.
+    # spec_file pins the lookup to the shared copy the parent just installed.
+    # install_extension_in_lib is forced off for the reason in GemRepair::Sweep.
+    BUILD = <<~'RUBY'
+      $VERBOSE = nil
+      exit 2 unless defined?(Gem::Specification) && Gem::Specification.method_defined?(:missing_extensions?)
+      name, version, spec_file = ARGV
+      spec = Gem::Specification.find_all_by_name(name, version).find { |s| s.spec_file == spec_file }
+      exit 2 if spec.nil? || spec.default_gem? || !spec.missing_extensions?
+      exit 2 unless spec.required_ruby_version.satisfied_by?(Gem.ruby_version)
+      def Gem.install_extension_in_lib; false; end
+      begin
+        spec.build_extensions
+      rescue Gem::InstallError => e
+        $stderr.puts e.message
+      end
+      exit spec.missing_extensions? ? 1 : 0
+    RUBY
+
+    module_function
+
+    def run(spec, ui: Gem::DefaultUserInteraction.ui)
+      return if spec.extensions.empty?
+      # Bundler autoloads Installer, so only a real load marks bundle install.
+      return if defined?(Bundler) && !Bundler.autoload?(:Installer)
+      # A per-ABI base_dir is private to this ruby, so the others never read it.
+      base_dir = File.realpath(spec.base_dir) rescue File.expand_path(spec.base_dir)
+      return if [Gem.default_dir, Gem.user_dir].any? { |dir| (File.realpath(dir) rescue File.expand_path(dir)) == base_dir }
+
+      env = { "GEM_HOME" => spec.base_dir, "GEM_PATH" => Gem.path.join(File::PATH_SEPARATOR) }
+      # The parent's Bundler setup does not resolve under another ruby.
+      %w[BUNDLER_SETUP RUBYOPT RUBYLIB RUBYGEMS_GEMDEPS BUNDLE_GEMFILE BUNDLE_BIN_PATH].each { |key| env[key] = nil }
+      # The child sees Gem.path through realpath, the parent may not.
+      spec_file = File.realpath(spec.spec_file) rescue spec.spec_file
+      rubies.each do |name, ruby|
+        system(env, ruby, "-e", BUILD, "--", spec.name, spec.version.to_s, spec_file)
+        case $?.exitstatus
+        when 0 then ui.say "Built #{spec.full_name} extensions for #{name}"
+        when 2 then next
+        else ui.alert_warning "Failed to build #{spec.full_name} extensions for #{name}"
+        end
+      end
+    end
+
+    # [name, path] pairs, one per distinct install, the running ruby excluded.
+    def rubies
+      names = ENV["GEM_REPAIR_RUBIES"]&.split(",")&.map(&:strip)
+      return [] if names&.empty?
+
+      # Gem.ruby quotes a path that contains a space, RbConfig.ruby does not.
+      current = File.realpath(RbConfig.ruby)
+      seen = {}
+      roots.flat_map { |root, pattern| Dir.glob(pattern, base: root).sort.map { |path| File.join(root, path) } }.filter_map do |ruby|
+        name = File.basename(File.dirname(File.dirname(ruby)))
+        next if SKIP.match?(name)
+        next if names && !names.include?(name)
+
+        real = begin
+          File.realpath(ruby)
+        rescue SystemCallError
+          next
+        end
+        next if real == current || seen[real]
+
+        seen[real] = true
+        [name, ruby]
+      end
+    end
+
+    def roots
+      mise = ENV["MISE_DATA_DIR"] || File.join(ENV["XDG_DATA_HOME"] || File.expand_path("~/.local/share"), "mise")
+      rbenv = ENV["RBENV_ROOT"] || File.expand_path("~/.rbenv")
+      # Kept apart from the pattern so metacharacters in the roots stay literal.
+      [[mise, "installs/ruby/*/bin/ruby"], [rbenv, "versions/*/bin/ruby"]]
+    end
+  end
+end

data/lib/gem_repair/sweep.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+require "fileutils"
+
+module GemRepair
+  # Removes the copies of built extensions that RubyGems leaves in the gem's
+  # lib next to the per-ABI copy under extensions/. lib comes first on the
+  # load path, so every other ruby sharing the GEM_HOME would load the copy
+  # built for the installing ruby and fail. RubyGems 3.6 added a gemrc setting
+  # to skip the copy, but it stays on by default.
+  module Sweep
+    DLEXT = ".#{RbConfig::CONFIG["DLEXT"]}"
+
+
+    # What gem sweep --aggressive removed on top of the extension copies.
+    DEVELOPMENT = %w[test spec features]
+
+    module_function
+
+    def targets(spec)
+      return [] if spec.default_gem? || spec.extensions.empty?
+
+      # RubyGems copies one tree into both extensions/ and lib, so a file with
+      # no counterpart under extensions/ was shipped by the gem itself.
+      built = Dir.glob(File.join("*", "*", spec.full_name, "**", "*#{DLEXT}"), base: File.join(spec.base_dir, "extensions"))
+      built = built.map { |file| file.split("/", 4).last }
+      (spec.full_require_paths - [spec.extension_dir]).flat_map do |path|
+        Dir.glob(File.join("**", "*#{DLEXT}"), base: path).select { |file| built.include?(file) }.map { |file| File.join(path, file) }
+      end
+    end
+
+    def development_targets(spec)
+      return [] if spec.default_gem?
+
+      root = spec.full_gem_path
+      dirs = DEVELOPMENT.map { |dir| File.join(root, dir) } + Dir.glob("**/tmp", base: root).map { |tmp| File.join(root, tmp) }
+      dirs.select { |dir| File.directory?(dir) && dirs.none? { |other| dir.start_with?("#{other}/") } }
+    end
+
+    def clean(spec, dry_run: false, aggressive: false, ui: Gem::DefaultUserInteraction.ui)
+      paths = targets(spec)
+      paths += development_targets(spec) if aggressive
+      paths.each do |path|
+        if dry_run
+          ui.say "Would remove #{path}"
+        else
+          File.directory?(path) ? FileUtils.rm_r(path) : File.delete(path)
+          ui.say "Removed #{path}"
+        end
+      rescue SystemCallError => e
+        ui.alert_warning "Could not remove #{path}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/rubygems/commands/repair_command.rb

@@ -1,5 +1,8 @@
+require 'zlib'
 require 'rubygems/command'
 require 'rubygems/installer'
+# see rubygems_plugin.rb
+require_relative '../../gem_repair/sweep' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7")
 
 
 class Gem::Commands::RepairCommand < Gem::Command
@@ -10,6 +13,21 @@ class Gem::Commands::RepairCommand < Gem::Command
                'Number of parallel threads to use (default: 4)') do |value, options|
       options[:jobs] = value
     end
+
+    add_option('--prune',
+               'Uninstall gems whose extensions are still missing after repair') do |value, options|
+      options[:prune] = true
+    end
+
+    add_option('--aggressive-sweep',
+               'Also remove the test, spec, features and tmp directories of every gem') do |value, options|
+      options[:aggressive_sweep] = true
+    end
+
+    add_option('-n', '--dry-run',
+               'Show what would be repaired, removed or uninstalled without changing anything') do |value, options|
+      options[:dry_run] = true
+    end
   end
 
   def arguments # :nodoc:
@@ -22,7 +40,18 @@ The repair command finds all installed gems that are missing their compiled
 extensions and attempts to reinstall them. This can be useful after upgrading
 Ruby or changing system libraries.
 
+It also removes the copies of built extensions that older RubyGems leave in
+the lib directory of every gem with extensions, since they shadow the per-ruby
+copy for the other rubies sharing the GEM_HOME.
+
+Reinstalling a gem runs the install hooks, so the fan-out to the other rubies
+described in the README happens here too.
+
 Use -j to specify the number of parallel threads (default: 4).
+Use --prune to uninstall gems that are still missing extensions after repair.
+Use --aggressive-sweep to also remove the test, spec, features and tmp
+directories of every gem, as gem sweep --aggressive did.
+Use -n to only show what would be done.
     EOF
   end
 
@@ -31,8 +60,15 @@ Use -j to specify the number of parallel threads (default: 4).
   end
 
   def execute
+    dry_run = options[:dry_run]
     say "Searching for gems with missing extensions..."
 
+    if defined?(GemRepair::Sweep)
+      Gem::Specification.each do |spec|
+        GemRepair::Sweep.clean(spec, dry_run: dry_run, aggressive: options[:aggressive_sweep], ui: ui)
+      end
+    end
+
     specs = Gem::Specification.select do |spec|
       spec.platform == RUBY_ENGINE && spec.respond_to?(:missing_extensions?) && spec.missing_extensions?
     end
@@ -53,6 +89,12 @@ Use -j to specify the number of parallel threads (default: 4).
 
     say "Found #{specs.count} gem(s) to repair: #{specs.map(&:full_name).join(', ')}"
 
+    if dry_run
+      specs.each { |spec| say "Would repair #{spec.full_name}" }
+      specs.each { |spec| say "Would uninstall #{spec.full_name} unless repaired" } if options[:prune]
+      return
+    end
+
     # Get number of threads from -j option, default to 4
     num_threads = options[:jobs] || 4
     # Running Gem::Installer in multiple threads deadlocks on old Rubies
@@ -80,6 +122,15 @@ Use -j to specify the number of parallel threads (default: 4).
       specs.each { |spec| repair_gem(spec) }
     end
 
+    # Reinstalling extracts the gem again, and the install hook only sweeps lib.
+    if options[:aggressive_sweep] && defined?(GemRepair::Sweep)
+      specs.each { |spec| GemRepair::Sweep.clean(spec, aggressive: true, ui: ui) }
+    end
+
+    if options[:prune]
+      specs.select(&:missing_extensions?).each { |spec| prune_gem(spec) }
+    end
+
     say "Gem repair process complete."
   end
 
@@ -101,7 +152,12 @@ Use -j to specify the number of parallel threads (default: 4).
 
     # Use spec.cache_file if available and valid, otherwise the installer might re-download
     # Forcing a specific installer might be needed if default behavior isn't right
-    installer = Gem::Installer.at(spec.cache_file, installer_options)
+    installer = if Gem::Installer.respond_to?(:at)
+      Gem::Installer.at(spec.cache_file, installer_options)
+    else
+      # Gem::Installer.at was added in RubyGems 2.5.0 (Ruby 2.3)
+      Gem::Installer.new(spec.cache_file, installer_options)
+    end
     installer.install
     say "Successfully repaired #{spec.full_name} to #{spec.base_dir}"
   rescue Gem::Ext::BuildError, Gem::Package::FormatError, Gem::InstallError, Zlib::BufError, NameError => e
@@ -109,4 +165,11 @@ Use -j to specify the number of parallel threads (default: 4).
   rescue => e
     alert_error "An unexpected error occurred while repairing #{spec.full_name}: #{e.message}\n  Backtrace: #{e.backtrace.join("\n             ")}"
   end
+
+  def prune_gem(spec)
+    require 'rubygems/uninstaller'
+    Gem::Uninstaller.new(spec.name, version: spec.version, install_dir: spec.base_dir, executables: true).uninstall_gem(spec)
+  rescue Gem::Exception => e
+    alert_error "Failed to uninstall #{spec.full_name}: #{e.message}"
+  end
 end

data/lib/rubygems_plugin.rb

@@ -2,3 +2,19 @@ require 'rubygems/command_manager'
 require 'rubygems/commands/repair_command'
 
 Gem::CommandManager.instance.register_command :repair
+
+# Every ruby sharing the GEM_HOME loads this plugin, including ones below the
+# required_ruby_version. They still get gem repair, just not the hook.
+if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7")
+  require_relative 'gem_repair/sweep'
+  require_relative 'gem_repair/fanout'
+
+  # Gem::Installer loads this file again when the gem itself is reinstalled.
+  unless defined?(GemRepair::HOOKED)
+    GemRepair::HOOKED = true
+    Gem.post_install do |installer|
+      GemRepair::Sweep.clean(installer.spec)
+      GemRepair::Fanout.run(installer.spec)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gem-repair
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Hiroshi SHIBATA
@@ -19,6 +19,8 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- lib/gem_repair/fanout.rb
+- lib/gem_repair/sweep.rb
 - lib/rubygems/commands/repair_command.rb
 - lib/rubygems_plugin.rb
 homepage: https://github.com/hsbt/gem-repair