bootsnap 1.16.0 → 1.18.6

data/lib/bootsnap/cli/worker_pool.rb

@@ -1,16 +1,88 @@
 # frozen_string_literal: true
 
+require "etc"
+require "rbconfig"
+require "io/wait" unless IO.method_defined?(:wait_readable)
+
 module Bootsnap
   class CLI
     class WorkerPool
       class << self
         def create(size:, jobs:)
+          size ||= default_size
           if size > 0 && Process.respond_to?(:fork)
             new(size: size, jobs: jobs)
           else
             Inline.new(jobs: jobs)
           end
         end
+
+        def default_size
+          nprocessors = Etc.nprocessors
+          size = [nprocessors, cpu_quota&.to_i || nprocessors].min
+          case size
+          when 0, 1
+            0
+          else
+            if fork_defunct?
+              $stderr.puts "warning: faulty fork(2) detected, probably in cross platform docker builds. " \
+                           "Disabling parallel compilation."
+              0
+            else
+              size
+            end
+          end
+        end
+
+        def cpu_quota
+          if RbConfig::CONFIG["target_os"].include?("linux")
+            if File.exist?("/sys/fs/cgroup/cpu.max")
+              # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
+              cpu_max = File.read("/sys/fs/cgroup/cpu.max")
+              return nil if cpu_max.start_with?("max ") # no limit
+
+              max, period = cpu_max.split.map(&:to_f)
+              max / period
+            elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us")
+              # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt
+              max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i
+              # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions
+              # https://docs.kernel.org/scheduler/sched-bwc.html#management
+              return nil if max <= 0
+
+              period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f
+              max / period
+            end
+          end
+        end
+
+        def fork_defunct?
+          return true unless ::Process.respond_to?(:fork)
+
+          # Ref: https://github.com/Shopify/bootsnap/issues/495
+          # The second forked process will hang on some QEMU environments
+          r, w = IO.pipe
+          pids = 2.times.map do
+            ::Process.fork do
+              exit!(true)
+            end
+          end
+          w.close
+          r.wait_readable(1) # Wait at most 1s
+
+          defunct = false
+
+          pids.each do |pid|
+            _pid, status = ::Process.wait2(pid, ::Process::WNOHANG)
+            if status.nil? # Didn't exit in 1s
+              defunct = true
+              Process.kill(:KILL, pid)
+              ::Process.wait2(pid)
+            end
+          end
+
+          defunct
+        end
       end
 
       class Inline

data/lib/bootsnap/cli.rb

@@ -4,7 +4,6 @@ require "bootsnap"
 require "bootsnap/cli/worker_pool"
 require "optparse"
 require "fileutils"
-require "etc"
 
 module Bootsnap
   class CLI
@@ -29,23 +28,23 @@ module Bootsnap
       self.compile_gemfile = false
       self.exclude = nil
       self.verbose = false
-      self.jobs = Etc.nprocessors
+      self.jobs = nil
       self.iseq = true
       self.yaml = true
       self.json = true
     end
 
     def precompile_command(*sources)
-      require "bootsnap/compile_cache/iseq"
-      require "bootsnap/compile_cache/yaml"
-      require "bootsnap/compile_cache/json"
+      require "bootsnap/compile_cache"
 
       fix_default_encoding do
-        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
-        Bootsnap::CompileCache::YAML.init!
-        Bootsnap::CompileCache::YAML.cache_dir = cache_dir
-        Bootsnap::CompileCache::JSON.init!
-        Bootsnap::CompileCache::JSON.cache_dir = cache_dir
+        Bootsnap::CompileCache.setup(
+          cache_dir: cache_dir,
+          iseq: iseq,
+          yaml: yaml,
+          json: json,
+          revalidation: true,
+        )
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
@@ -60,14 +59,16 @@ module Bootsnap
         precompile_json_files(main_sources)
 
         if compile_gemfile
-          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
-          gem_exclude = Regexp.union([exclude, "/spec/", "/test/"].compact)
-          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
-
           # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
+          # Similarly, gems that include Rails engines generally file Ruby files in `app/`.
+          # However some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, "/spec/", "/test/", "/features/"].compact)
+
           gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
-          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] || p }.uniq
+
+          precompile_ruby_files(gem_paths, exclude: gem_exclude)
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
           precompile_json_files(gem_paths, exclude: gem_exclude)
         end
@@ -220,6 +221,9 @@ module Bootsnap
 
     def parser
       @parser ||= OptionParser.new do |opts|
+        opts.version = Bootsnap::VERSION
+        opts.program_name = "bootsnap"
+
         opts.banner = "Usage: bootsnap COMMAND [ARGS]"
         opts.separator ""
         opts.separator "GLOBAL OPTIONS"

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
-require("zlib")
+require "bootsnap/bootsnap"
+require "zlib"
 
 module Bootsnap
   module CompileCache
@@ -12,6 +12,10 @@ module Bootsnap
         def cache_dir=(cache_dir)
           @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
         end
+
+        def supported?
+          CompileCache.supported? && defined?(RubyVM)
+        end
       end
 
       has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
@@ -80,11 +84,9 @@ module Bootsnap
       module InstructionSequenceMixin
         def load_iseq(path)
           # Having coverage enabled prevents iseq dumping/loading.
-          return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
+          return nil if defined?(Coverage) && Coverage.running?
 
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
-        rescue Errno::EACCES
-          Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => error
           if error.message =~ /unmatched platform/
             puts("unmatched platform for file #{path}")
@@ -103,11 +105,15 @@ module Bootsnap
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
-      compile_option_updated
+      compile_option_updated if supported?
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+
+        return unless supported?
+
         Bootsnap::CompileCache::ISeq.compile_option_updated
+
         class << RubyVM::InstructionSequence
           prepend(InstructionSequenceMixin)
         end

data/lib/bootsnap/compile_cache/json.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -46,8 +46,8 @@ module Bootsnap
         end
 
         def init!
-          require("json")
-          require("msgpack")
+          require "json"
+          require "msgpack"
 
           self.msgpack_factory = MessagePack::Factory.new
           self.supported_options = [:symbolize_names]
@@ -74,16 +74,12 @@ module Bootsnap
             return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
           end
 
-          begin
-            ::Bootsnap::CompileCache::Native.fetch(
-              Bootsnap::CompileCache::JSON.cache_dir,
-              File.realpath(path),
-              ::Bootsnap::CompileCache::JSON,
-              kwargs,
-            )
-          rescue Errno::EACCES
-            ::Bootsnap::CompileCache.permission_error(path)
-          end
+          ::Bootsnap::CompileCache::Native.fetch(
+            Bootsnap::CompileCache::JSON.cache_dir,
+            File.realpath(path),
+            ::Bootsnap::CompileCache::JSON,
+            kwargs,
+          )
         end
 
         ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -55,9 +55,9 @@ module Bootsnap
         end
 
         def init!
-          require("yaml")
-          require("msgpack")
-          require("date")
+          require "yaml"
+          require "msgpack"
+          require "date"
 
           @implementation = ::YAML::VERSION >= "4" ? Psych4 : Psych3
           if @implementation::Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
@@ -229,16 +229,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::SafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::SafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -253,16 +249,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::UnsafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::UnsafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
@@ -309,16 +301,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -333,16 +321,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache.rb

@@ -8,12 +8,11 @@ module Bootsnap
     end
 
     Error = Class.new(StandardError)
-    PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:, json:, readonly: false)
+    def self.setup(cache_dir:, iseq:, yaml:, json:, readonly: false, revalidation: false)
       if iseq
         if supported?
-          require_relative("compile_cache/iseq")
+          require_relative "compile_cache/iseq"
           Bootsnap::CompileCache::ISeq.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] bytecode caching is not supported on this implementation of Ruby")
@@ -22,7 +21,7 @@ module Bootsnap
 
       if yaml
         if supported?
-          require_relative("compile_cache/yaml")
+          require_relative "compile_cache/yaml"
           Bootsnap::CompileCache::YAML.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
@@ -31,7 +30,7 @@ module Bootsnap
 
       if json
         if supported?
-          require_relative("compile_cache/json")
+          require_relative "compile_cache/json"
           Bootsnap::CompileCache::JSON.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
@@ -40,21 +39,14 @@ module Bootsnap
 
       if supported? && defined?(Bootsnap::CompileCache::Native)
         Bootsnap::CompileCache::Native.readonly = readonly
+        Bootsnap::CompileCache::Native.revalidation = revalidation
       end
     end
 
-    def self.permission_error(path)
-      cpath = Bootsnap::CompileCache::ISeq.cache_dir
-      raise(
-        PermissionError,
-        "bootsnap doesn't have permission to write cache entries in '#{cpath}' " \
-        "(or, less likely, doesn't have permission to read '#{path}')",
-      )
-    end
-
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
-      RUBY_ENGINE == "ruby" && RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
+      # only enable on 'ruby' (MRI) and TruffleRuby for POSIX (darwin, linux, *bsd), Windows (RubyInstaller2)
+      %w[ruby truffleruby].include?(RUBY_ENGINE) &&
+        RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
     end
   end
 end

data/lib/bootsnap/explicit_require.rb

@@ -25,6 +25,11 @@ module Bootsnap
     # This is useful before bootsnap is fully-initialized to load gems that it
     # depends on, without forcing full LOAD_PATH traversals.
     def self.with_gems(*gems)
+      # Ensure the gems are activated (their paths are in $LOAD_PATH)
+      gems.each do |gem_name|
+        gem gem_name
+      end
+
       orig = $LOAD_PATH.dup
       $LOAD_PATH.clear
       gems.each do |gem|

data/lib/bootsnap/load_path_cache/cache.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative("../explicit_require")
+require_relative "../explicit_require"
 
 module Bootsnap
   module LoadPathCache
@@ -24,8 +24,16 @@ module Bootsnap
         @mutex.synchronize { @dirs[dir] }
       end
 
+      TRUFFLERUBY_LIB_DIR_PREFIX = if RUBY_ENGINE == "truffleruby"
+        "#{File.join(RbConfig::CONFIG['libdir'], 'truffle')}#{File::SEPARATOR}"
+      end
+
       # { 'enumerator' => nil, 'enumerator.so' => nil, ... }
       BUILTIN_FEATURES = $LOADED_FEATURES.each_with_object({}) do |feat, features|
+        if TRUFFLERUBY_LIB_DIR_PREFIX && feat.start_with?(TRUFFLERUBY_LIB_DIR_PREFIX)
+          feat = feat.byteslice(TRUFFLERUBY_LIB_DIR_PREFIX.bytesize..-1)
+        end
+
         # Builtin features are of the form 'enumerator.so'.
         # All others include paths.
         next unless feat.size < 20 && !feat.include?("/")

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -54,6 +54,12 @@ module Bootsnap
             ret
           end
         end
+
+        def dup
+          [] + self
+        end
+
+        alias_method :clone, :dup
       end
 
       def self.register(arr, observer)

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 module Kernel
-  module_function
+  alias_method :require_without_bootsnap, :require
 
-  alias_method(:require_without_bootsnap, :require)
+  alias_method :require, :require # Avoid method redefinition warnings
 
-  def require(path)
+  def require(path) # rubocop:disable Lint/DuplicateMethods
     return require_without_bootsnap(path) unless Bootsnap::LoadPathCache.enabled?
 
     string_path = Bootsnap.rb_get_path(path)
@@ -24,9 +24,7 @@ module Kernel
     elsif false == resolved
       return false
     elsif resolved.nil?
-      error = LoadError.new(+"cannot load such file -- #{path}")
-      error.instance_variable_set(:@path, path)
-      raise error
+      return require_without_bootsnap(path)
     else
       # Note that require registers to $LOADED_FEATURES while load does not.
       ret = require_without_bootsnap(resolved)
@@ -34,4 +32,6 @@ module Kernel
       return ret
     end
   end
+
+  private :require
 end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -142,7 +142,7 @@ module Bootsnap
       # will _never_ run on MacOS, and therefore think they can get away
       # with calling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.)
       #
-      # See <https://ruby-doc.org/core-2.6.4/Kernel.html#method-i-require>.
+      # See <https://docs.ruby-lang.org/en/master/Kernel.html#method-i-require>.
       def extension_elidable?(feature)
         feature.to_s.end_with?(".rb", ".so", ".o", ".dll", ".dylib")
       end

data/lib/bootsnap/load_path_cache/path.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative("path_scanner")
+require_relative "path_scanner"
 
 module Bootsnap
   module LoadPathCache

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative("../explicit_require")
+require_relative "../explicit_require"
 
 module Bootsnap
   module LoadPathCache
@@ -54,7 +54,7 @@ module Bootsnap
 
             absolute_path = "#{absolute_dir_path}/#{name}"
             if File.directory?(absolute_path)
-              next if ignored_directories.include?(name)
+              next if ignored_directories.include?(name) || ignored_directories.include?(absolute_path)
 
               if yield relative_path, absolute_path, true
                 walk(absolute_path, relative_path, &block)

data/lib/bootsnap/load_path_cache/store.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative("../explicit_require")
+require_relative "../explicit_require"
 
-Bootsnap::ExplicitRequire.with_gems("msgpack") { require("msgpack") }
+Bootsnap::ExplicitRequire.with_gems("msgpack") { require "msgpack" }
 
 module Bootsnap
   module LoadPathCache
@@ -122,6 +122,8 @@ module Bootsnap
         stack.reverse_each do |dir|
           Dir.mkdir(dir)
         rescue SystemCallError
+          # Check for broken symlinks. Calling File.realpath will raise Errno::ENOENT if that is the case
+          File.realpath(dir) if File.symlink?(dir)
           raise unless File.directory?(dir)
         end
       end

data/lib/bootsnap/load_path_cache.rb

@@ -38,11 +38,11 @@ module Bootsnap
 
         @loaded_features_index = LoadedFeaturesIndex.new
 
-        @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
         PathScanner.ignored_directories = ignore_directories if ignore_directories
+        @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
         @enabled = true
-        require_relative("load_path_cache/core_ext/kernel_require")
-        require_relative("load_path_cache/core_ext/loaded_features")
+        require_relative "load_path_cache/core_ext/kernel_require"
+        require_relative "load_path_cache/core_ext/loaded_features"
       end
 
       def unload!
@@ -50,22 +50,31 @@ module Bootsnap
         @loaded_features_index = nil
         @realpath_cache = nil
         @load_path_cache = nil
-        ChangeObserver.unregister($LOAD_PATH)
+        ChangeObserver.unregister($LOAD_PATH) if supported?
       end
 
       def supported?
-        RUBY_ENGINE == "ruby" &&
-          RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/
+        if RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
+          case RUBY_ENGINE
+          when "truffleruby"
+            # https://github.com/oracle/truffleruby/issues/3131
+            RUBY_ENGINE_VERSION >= "23.1.0"
+          when "ruby"
+            true
+          else
+            false
+          end
+        end
       end
     end
   end
 end
 
 if Bootsnap::LoadPathCache.supported?
-  require_relative("load_path_cache/path_scanner")
-  require_relative("load_path_cache/path")
-  require_relative("load_path_cache/cache")
-  require_relative("load_path_cache/store")
-  require_relative("load_path_cache/change_observer")
-  require_relative("load_path_cache/loaded_features_index")
+  require_relative "load_path_cache/path_scanner"
+  require_relative "load_path_cache/path"
+  require_relative "load_path_cache/cache"
+  require_relative "load_path_cache/store"
+  require_relative "load_path_cache/change_observer"
+  require_relative "load_path_cache/loaded_features_index"
 end

data/lib/bootsnap/setup.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-require_relative("../bootsnap")
+require_relative "../bootsnap"
 
 Bootsnap.default_setup

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.16.0"
+  VERSION = "1.18.6"
 end