bootsnap 1.18.3 → 1.18.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fa4ab785277ee01a1c8ee75b43f0efb93db42bffcdacc1c8505a65efa03dede
-  data.tar.gz: 8aaaca48ae257b563580023c8fa36a59463f4c30f5463c14f6b8b94bf5fe27df
+  metadata.gz: 49e48977ddbf373c70db5988627366223140c8718fcf36cc4dd55d82a20716d4
+  data.tar.gz: 23b7644313a2c0d2df0a2b6c3816a1634a2f3f94f8c12388c6b674608b92aa82
 SHA512:
-  metadata.gz: 27b48d27d3330c8565952a2fbb979e71013b1e9585bcb3284656192808c304f2874c32a135b14895eec61a7ef038fa71fa111964a56e7aaedc9ff507ef307686
-  data.tar.gz: c3d83a0b068f2908a6298c7cd8e1a660f1228a7ddbfb9409cb3f6c174319f3974388ce73756c04062b17b97a37e199a07bccbb0b8dc1c6224998c58c51194b27
+  metadata.gz: e50126e1a8a1cfa22cee67fc04efd056a2f792a0aee08cbd73ff0a7e5cec0e8f2902b0ddc08a89a3d2a12c119bf042290acd0ec28642c5c6acc86adaa317ef31
+  data.tar.gz: cc8c691e916b5df79c073d0285154df4c3483c09b6fe085d2bde5ff11e1e710a01b68d86db3bbfd253b3cce95b278720e47f94e8989941c899bb80d4a3e7151e

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Unreleased
 
+# 1.18.5
+
+* Attempt to detect a QEMU bug that can cause `bootsnap precompile` to hang forever when building ARM64 docker images
+  from x86_64 machines. See #495.
+* Improve CLI to detect cgroup CPU limits and avoid spawning too many worker processes.
+
+# 1.18.4
+
+* Allow using bootsnap without bundler. See #488.
+* Fix startup failure if the cache directory points to a broken symlink.
+
 # 1.18.3
 
 * Fix the cache corruption issue in the revalidation feature. See #474.
@@ -32,7 +43,7 @@
 
 # 1.17.0
 
-* Ensure `$LOAD_PATH.dup` is Ractor shareable to fix an conflict with `did_you_mean`.
+* Ensure `$LOAD_PATH.dup` is Ractor shareable to fix a conflict with `did_you_mean`.
 * Allow to ignore directories using absolute paths.
 * Support YAML and JSON CompileCache on TruffleRuby.
 * Support LoadPathCache on TruffleRuby.

data/README.md

@@ -188,7 +188,7 @@ result too, raising a `LoadError` without touching the filesystem at all.
 
 Ruby has complex grammar and parsing it is not a particularly cheap operation. Since 1.9, Ruby has
 translated ruby source to an internal bytecode format, which is then executed by the Ruby VM. Since
-2.3.0, Ruby [exposes an API](https://ruby-doc.org/core-2.3.0/RubyVM/InstructionSequence.html) that
+2.3.0, Ruby [exposes an API](https://docs.ruby-lang.org/en/master/RubyVM/InstructionSequence.html) that
 allows caching that bytecode. This allows us to bypass the relatively-expensive compilation step on
 subsequent loads of the same file.
 
@@ -331,7 +331,7 @@ To do so you can use the `bootsnap precompile` command.
 Example:
 
 ```bash
-$ bundle exec bootsnap precompile --gemfile app/ lib/
+$ bundle exec bootsnap precompile --gemfile app/ lib/ config/
 ```
 
 ## When not to use Bootsnap

data/ext/bootsnap/bootsnap.c

@@ -82,7 +82,7 @@ struct bs_cache_key {
 STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
 
 /* Effectively a schema version. Bumping invalidates all previous caches */
-static const uint32_t current_version = 5;
+static const uint32_t current_version = 6;
 
 /* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
  * new OS ABI, etc. */
@@ -146,15 +146,6 @@ struct s2o_data;
 struct i2o_data;
 struct i2s_data;
 
-/* https://bugs.ruby-lang.org/issues/13667 */
-extern VALUE rb_get_coverages(void);
-static VALUE
-bs_rb_coverage_running(VALUE self)
-{
-  VALUE cov = rb_get_coverages();
-  return RTEST(cov) ? Qtrue : Qfalse;
-}
-
 static VALUE
 bs_rb_get_path(VALUE self, VALUE fname)
 {
@@ -193,7 +184,6 @@ Init_bootsnap(void)
   rb_define_module_function(rb_mBootsnap, "instrumentation_enabled=", bs_instrumentation_enabled_set, 1);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "readonly=", bs_readonly_set, 1);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "revalidation=", bs_revalidation_set, 1);
-  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
@@ -922,9 +912,9 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args
   goto succeed; /* output_data is now the correct return. */
 
 #define CLEANUP \
-  if (status != Qfalse) bs_instrumentation(status, path_v); \
   if (current_fd >= 0)  close(current_fd); \
-  if (cache_fd >= 0)    close(cache_fd);
+  if (cache_fd >= 0)    close(cache_fd); \
+  if (status != Qfalse) bs_instrumentation(status, path_v);
 
 succeed:
   CLEANUP;

data/lib/bootsnap/cli/worker_pool.rb

@@ -1,16 +1,88 @@
 # frozen_string_literal: true
 
+require "etc"
+require "rbconfig"
+require "io/wait" unless IO.method_defined?(:wait_readable)
+
 module Bootsnap
   class CLI
     class WorkerPool
       class << self
         def create(size:, jobs:)
+          size ||= default_size
           if size > 0 && Process.respond_to?(:fork)
             new(size: size, jobs: jobs)
           else
             Inline.new(jobs: jobs)
           end
         end
+
+        def default_size
+          nprocessors = Etc.nprocessors
+          size = [nprocessors, cpu_quota || nprocessors].min
+          case size
+          when 0, 1
+            0
+          else
+            if fork_defunct?
+              $stderr.puts "warning: faulty fork(2) detected, probably in cross platform docker builds. " \
+                           "Disabling parallel compilation."
+              0
+            else
+              size
+            end
+          end
+        end
+
+        def cpu_quota
+          if RbConfig::CONFIG["target_os"].include?("linux")
+            if File.exist?("/sys/fs/cgroup/cpu.max")
+              # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
+              cpu_max = File.read("/sys/fs/cgroup/cpu.max")
+              return nil if cpu_max.start_with?("max ") # no limit
+
+              max, period = cpu_max.split.map(&:to_f)
+              max / period
+            elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us")
+              # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt
+              max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i
+              # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions
+              # https://docs.kernel.org/scheduler/sched-bwc.html#management
+              return nil if max <= 0
+
+              period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f
+              max / period
+            end
+          end
+        end
+
+        def fork_defunct?
+          return true unless ::Process.respond_to?(:fork)
+
+          # Ref: https://github.com/Shopify/bootsnap/issues/495
+          # The second forked process will hang on some QEMU environments
+          r, w = IO.pipe
+          pids = 2.times.map do
+            ::Process.fork do
+              exit!(true)
+            end
+          end
+          w.close
+          r.wait_readable(1) # Wait at most 1s
+
+          defunct = false
+
+          pids.each do |pid|
+            _pid, status = ::Process.wait2(pid, ::Process::WNOHANG)
+            if status.nil? # Didn't exit in 1s
+              defunct = true
+              Process.kill(:KILL, pid)
+              ::Process.wait2(pid)
+            end
+          end
+
+          defunct
+        end
       end
 
       class Inline

data/lib/bootsnap/cli.rb

@@ -4,7 +4,6 @@ require "bootsnap"
 require "bootsnap/cli/worker_pool"
 require "optparse"
 require "fileutils"
-require "etc"
 
 module Bootsnap
   class CLI
@@ -29,23 +28,23 @@ module Bootsnap
       self.compile_gemfile = false
       self.exclude = nil
       self.verbose = false
-      self.jobs = Etc.nprocessors
+      self.jobs = nil
       self.iseq = true
       self.yaml = true
       self.json = true
     end
 
     def precompile_command(*sources)
-      require "bootsnap/compile_cache/iseq"
-      require "bootsnap/compile_cache/yaml"
-      require "bootsnap/compile_cache/json"
+      require "bootsnap/compile_cache"
 
       fix_default_encoding do
-        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
-        Bootsnap::CompileCache::YAML.init!
-        Bootsnap::CompileCache::YAML.cache_dir = cache_dir
-        Bootsnap::CompileCache::JSON.init!
-        Bootsnap::CompileCache::JSON.cache_dir = cache_dir
+        Bootsnap::CompileCache.setup(
+          cache_dir: cache_dir,
+          iseq: iseq,
+          yaml: yaml,
+          json: json,
+          revalidation: true,
+        )
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
@@ -222,6 +221,9 @@ module Bootsnap
 
     def parser
       @parser ||= OptionParser.new do |opts|
+        opts.version = Bootsnap::VERSION
+        opts.program_name = "bootsnap"
+
         opts.banner = "Usage: bootsnap COMMAND [ARGS]"
         opts.separator ""
         opts.separator "GLOBAL OPTIONS"

data/lib/bootsnap/compile_cache/iseq.rb

@@ -84,7 +84,7 @@ module Bootsnap
       module InstructionSequenceMixin
         def load_iseq(path)
           # Having coverage enabled prevents iseq dumping/loading.
-          return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
+          return nil if defined?(Coverage) && Coverage.running?
 
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue RuntimeError => error

data/lib/bootsnap/explicit_require.rb

@@ -25,6 +25,11 @@ module Bootsnap
     # This is useful before bootsnap is fully-initialized to load gems that it
     # depends on, without forcing full LOAD_PATH traversals.
     def self.with_gems(*gems)
+      # Ensure the gems are activated (their paths are in $LOAD_PATH)
+      gems.each do |gem_name|
+        gem gem_name
+      end
+
       orig = $LOAD_PATH.dup
       $LOAD_PATH.clear
       gems.each do |gem|

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -142,7 +142,7 @@ module Bootsnap
       # will _never_ run on MacOS, and therefore think they can get away
       # with calling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.)
       #
-      # See <https://ruby-doc.org/core-2.6.4/Kernel.html#method-i-require>.
+      # See <https://docs.ruby-lang.org/en/master/Kernel.html#method-i-require>.
       def extension_elidable?(feature)
         feature.to_s.end_with?(".rb", ".so", ".o", ".dll", ".dylib")
       end

data/lib/bootsnap/load_path_cache/store.rb

@@ -122,6 +122,8 @@ module Bootsnap
         stack.reverse_each do |dir|
           Dir.mkdir(dir)
         rescue SystemCallError
+          # Check for broken symlinks. Calling File.realpath will raise Errno::ENOENT if that is the case
+          File.realpath(dir) if File.symlink?(dir)
           raise unless File.directory?(dir)
         end
       end

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.18.3"
+  VERSION = "1.18.5"
 end

data/lib/bootsnap.rb

@@ -83,7 +83,7 @@ module Bootsnap
       env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["ENV"]
       development_mode = ["", nil, "development"].include?(env)
 
-      unless ENV["DISABLE_BOOTSNAP"]
+      if enabled?("BOOTSNAP")
         cache_dir = ENV["BOOTSNAP_CACHE_DIR"]
         unless cache_dir
           config_dir_frame = caller.detect do |line|
@@ -112,11 +112,12 @@ module Bootsnap
         setup(
           cache_dir: cache_dir,
           development_mode: development_mode,
-          load_path_cache: !ENV["DISABLE_BOOTSNAP_LOAD_PATH_CACHE"],
-          compile_cache_iseq: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
-          compile_cache_yaml: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
-          compile_cache_json: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
-          readonly: !!ENV["BOOTSNAP_READONLY"],
+          load_path_cache: enabled?("BOOTSNAP_LOAD_PATH_CACHE"),
+          compile_cache_iseq: enabled?("BOOTSNAP_COMPILE_CACHE"),
+          compile_cache_yaml: enabled?("BOOTSNAP_COMPILE_CACHE"),
+          compile_cache_json: enabled?("BOOTSNAP_COMPILE_CACHE"),
+          readonly: bool_env("BOOTSNAP_READONLY"),
+          revalidation: bool_env("BOOTSNAP_REVALIDATE"),
           ignore_directories: ignore_directories,
         )
 
@@ -148,5 +149,16 @@ module Bootsnap
 
     # Allow the C extension to redefine `rb_get_path` without warning.
     alias_method :rb_get_path, :rb_get_path # rubocop:disable Lint/DuplicateMethods
+
+    private
+
+    def enabled?(key)
+      !ENV["DISABLE_#{key}"]
+    end
+
+    def bool_env(key, default: false)
+      value = ENV.fetch(key) { default }
+      !["0", "false", false].include?(value)
+    end
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.18.3
+  version: 1.18.5
 platform: ruby
 authors:
 - Burke Libbey
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-01-31 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -68,7 +67,6 @@ metadata:
   changelog_uri: https://github.com/Shopify/bootsnap/blob/main/CHANGELOG.md
   source_code_uri: https://github.com/Shopify/bootsnap
   allowed_push_host: https://rubygems.org
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -83,8 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.5
-signing_key: 
+rubygems_version: 3.6.8
 specification_version: 4
 summary: Boot large ruby/rails apps faster
 test_files: []