bootsnap 1.10.3 → 1.18.6

data/ext/bootsnap/extconf.rb

@@ -1,26 +1,33 @@
 # frozen_string_literal: true
 
-require("mkmf")
+require "mkmf"
 
-if RUBY_ENGINE == "ruby"
-  $CFLAGS << " -O3 "
-  $CFLAGS << " -std=c99"
+if %w[ruby truffleruby].include?(RUBY_ENGINE)
+  have_func "fdatasync", "unistd.h"
+
+  unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
+    append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME
+  end
+
+  append_cflags ["-O3", "-std=c99"]
 
   # ruby.h has some -Wpedantic fails in some cases
   # (e.g. https://github.com/Shopify/bootsnap/issues/15)
   unless ["0", "", nil].include?(ENV["BOOTSNAP_PEDANTIC"])
-    $CFLAGS << " -Wall"
-    $CFLAGS << " -Werror"
-    $CFLAGS << " -Wextra"
-    $CFLAGS << " -Wpedantic"
+    append_cflags([
+      "-Wall",
+      "-Werror",
+      "-Wextra",
+      "-Wpedantic",
 
-    $CFLAGS << " -Wno-unused-parameter" # VALUE self has to be there but we don't care what it is.
-    $CFLAGS << " -Wno-keyword-macro" # hiding return
-    $CFLAGS << " -Wno-gcc-compat" # ruby.h 2.6.0 on macos 10.14, dunno
-    $CFLAGS << " -Wno-compound-token-split-by-macro"
+      "-Wno-unused-parameter", # VALUE self has to be there but we don't care what it is.
+      "-Wno-keyword-macro", # hiding return
+      "-Wno-gcc-compat", # ruby.h 2.6.0 on macos 10.14, dunno
+      "-Wno-compound-token-split-by-macro",
+    ])
   end
 
   create_makefile("bootsnap/bootsnap")
 else
-  File.write("Makefile", dummy_makefile($srcdir).join(""))
+  File.write("Makefile", dummy_makefile($srcdir).join)
 end

data/lib/bootsnap/bundler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  extend(self)
+  extend self
 
   def bundler?
     return false unless defined?(::Bundler)

data/lib/bootsnap/cli/worker_pool.rb

@@ -1,16 +1,88 @@
 # frozen_string_literal: true
 
+require "etc"
+require "rbconfig"
+require "io/wait" unless IO.method_defined?(:wait_readable)
+
 module Bootsnap
   class CLI
     class WorkerPool
       class << self
         def create(size:, jobs:)
+          size ||= default_size
           if size > 0 && Process.respond_to?(:fork)
             new(size: size, jobs: jobs)
           else
             Inline.new(jobs: jobs)
           end
         end
+
+        def default_size
+          nprocessors = Etc.nprocessors
+          size = [nprocessors, cpu_quota&.to_i || nprocessors].min
+          case size
+          when 0, 1
+            0
+          else
+            if fork_defunct?
+              $stderr.puts "warning: faulty fork(2) detected, probably in cross platform docker builds. " \
+                           "Disabling parallel compilation."
+              0
+            else
+              size
+            end
+          end
+        end
+
+        def cpu_quota
+          if RbConfig::CONFIG["target_os"].include?("linux")
+            if File.exist?("/sys/fs/cgroup/cpu.max")
+              # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
+              cpu_max = File.read("/sys/fs/cgroup/cpu.max")
+              return nil if cpu_max.start_with?("max ") # no limit
+
+              max, period = cpu_max.split.map(&:to_f)
+              max / period
+            elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us")
+              # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt
+              max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i
+              # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions
+              # https://docs.kernel.org/scheduler/sched-bwc.html#management
+              return nil if max <= 0
+
+              period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f
+              max / period
+            end
+          end
+        end
+
+        def fork_defunct?
+          return true unless ::Process.respond_to?(:fork)
+
+          # Ref: https://github.com/Shopify/bootsnap/issues/495
+          # The second forked process will hang on some QEMU environments
+          r, w = IO.pipe
+          pids = 2.times.map do
+            ::Process.fork do
+              exit!(true)
+            end
+          end
+          w.close
+          r.wait_readable(1) # Wait at most 1s
+
+          defunct = false
+
+          pids.each do |pid|
+            _pid, status = ::Process.wait2(pid, ::Process::WNOHANG)
+            if status.nil? # Didn't exit in 1s
+              defunct = true
+              Process.kill(:KILL, pid)
+              ::Process.wait2(pid)
+            end
+          end
+
+          defunct
+        end
       end
 
       class Inline

data/lib/bootsnap/cli.rb

@@ -4,7 +4,6 @@ require "bootsnap"
 require "bootsnap/cli/worker_pool"
 require "optparse"
 require "fileutils"
-require "etc"
 
 module Bootsnap
   class CLI
@@ -29,23 +28,23 @@ module Bootsnap
       self.compile_gemfile = false
       self.exclude = nil
       self.verbose = false
-      self.jobs = Etc.nprocessors
+      self.jobs = nil
       self.iseq = true
       self.yaml = true
       self.json = true
     end
 
     def precompile_command(*sources)
-      require "bootsnap/compile_cache/iseq"
-      require "bootsnap/compile_cache/yaml"
-      require "bootsnap/compile_cache/json"
+      require "bootsnap/compile_cache"
 
       fix_default_encoding do
-        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
-        Bootsnap::CompileCache::YAML.init!
-        Bootsnap::CompileCache::YAML.cache_dir = cache_dir
-        Bootsnap::CompileCache::JSON.init!
-        Bootsnap::CompileCache::JSON.cache_dir = cache_dir
+        Bootsnap::CompileCache.setup(
+          cache_dir: cache_dir,
+          iseq: iseq,
+          yaml: yaml,
+          json: json,
+          revalidation: true,
+        )
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
@@ -60,14 +59,16 @@ module Bootsnap
         precompile_json_files(main_sources)
 
         if compile_gemfile
-          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
-          gem_exclude = Regexp.union([exclude, "/spec/", "/test/"].compact)
-          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
-
           # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
-          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
-          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          # Similarly, gems that include Rails engines generally file Ruby files in `app/`.
+          # However some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, "/spec/", "/test/", "/features/"].compact)
+
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] || p }.uniq
+
+          precompile_ruby_files(gem_paths, exclude: gem_exclude)
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
           precompile_json_files(gem_paths, exclude: gem_exclude)
         end
@@ -138,8 +139,8 @@ module Bootsnap
 
     def precompile_yaml(*yaml_files)
       Array(yaml_files).each do |yaml_file|
-        if CompileCache::YAML.precompile(yaml_file)
-          STDERR.puts(yaml_file) if verbose
+        if CompileCache::YAML.precompile(yaml_file) && verbose
+          $stderr.puts(yaml_file)
         end
       end
     end
@@ -161,8 +162,8 @@ module Bootsnap
 
     def precompile_json(*json_files)
       Array(json_files).each do |json_file|
-        if CompileCache::JSON.precompile(json_file)
-          STDERR.puts(json_file) if verbose
+        if CompileCache::JSON.precompile(json_file) && verbose
+          $stderr.puts(json_file)
         end
       end
     end
@@ -172,7 +173,7 @@ module Bootsnap
 
       load_paths.each do |path|
         if !exclude || !exclude.match?(path)
-          list_files(path, "**/*.rb").each do |ruby_file|
+          list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
             if !exclude || !exclude.match?(ruby_file)
               @work_pool.push(:ruby, ruby_file)
             end
@@ -183,8 +184,8 @@ module Bootsnap
 
     def precompile_ruby(*ruby_files)
       Array(ruby_files).each do |ruby_file|
-        if CompileCache::ISeq.precompile(ruby_file)
-          STDERR.puts(ruby_file) if verbose
+        if CompileCache::ISeq.precompile(ruby_file) && verbose
+          $stderr.puts(ruby_file)
         end
       end
     end
@@ -203,9 +204,9 @@ module Bootsnap
     end
 
     def invalid_usage!(message)
-      STDERR.puts message
-      STDERR.puts
-      STDERR.puts parser
+      $stderr.puts message
+      $stderr.puts
+      $stderr.puts parser
       1
     end
 
@@ -220,6 +221,9 @@ module Bootsnap
 
     def parser
       @parser ||= OptionParser.new do |opts|
+        opts.version = Bootsnap::VERSION
+        opts.program_name = "bootsnap"
+
         opts.banner = "Usage: bootsnap COMMAND [ARGS]"
         opts.separator ""
         opts.separator "GLOBAL OPTIONS"

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
-require("zlib")
+require "bootsnap/bootsnap"
+require "zlib"
 
 module Bootsnap
   module CompileCache
@@ -12,6 +12,10 @@ module Bootsnap
         def cache_dir=(cache_dir)
           @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
         end
+
+        def supported?
+          CompileCache.supported? && defined?(RubyVM)
+        end
       end
 
       has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
@@ -34,14 +38,14 @@ module Bootsnap
           begin
             iseq.to_binary
           rescue TypeError
-            return UNCOMPILABLE # ruby bug #18250
+            UNCOMPILABLE # ruby bug #18250
           end
         end
       else
         def self.input_to_storage(_, path)
           RubyVM::InstructionSequence.compile_file(path).to_binary
         rescue SyntaxError
-          return UNCOMPILABLE # syntax error
+          UNCOMPILABLE # syntax error
         end
       end
 
@@ -49,7 +53,7 @@ module Bootsnap
         RubyVM::InstructionSequence.load_from_binary(binary)
       rescue RuntimeError => error
         if error.message == "broken binary format"
-          STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+          $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
           nil
         else
           raise
@@ -80,11 +84,9 @@ module Bootsnap
       module InstructionSequenceMixin
         def load_iseq(path)
           # Having coverage enabled prevents iseq dumping/loading.
-          return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
+          return nil if defined?(Coverage) && Coverage.running?
 
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
-        rescue Errno::EACCES
-          Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => error
           if error.message =~ /unmatched platform/
             puts("unmatched platform for file #{path}")
@@ -103,11 +105,15 @@ module Bootsnap
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
-      compile_option_updated
+      compile_option_updated if supported?
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+
+        return unless supported?
+
         Bootsnap::CompileCache::ISeq.compile_option_updated
+
         class << RubyVM::InstructionSequence
           prepend(InstructionSequenceMixin)
         end

data/lib/bootsnap/compile_cache/json.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -46,18 +46,23 @@ module Bootsnap
         end
 
         def init!
-          require("json")
-          require("msgpack")
+          require "json"
+          require "msgpack"
 
           self.msgpack_factory = MessagePack::Factory.new
           self.supported_options = [:symbolize_names]
-          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
-            if MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
-              self.supported_options = [:freeze]
-            end
+          if supports_freeze?
+            self.supported_options = [:freeze]
           end
           supported_options.freeze
         end
+
+        private
+
+        def supports_freeze?
+          ::JSON.parse('["foo"]', freeze: true).first.frozen? &&
+            MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
+        end
       end
 
       module Patch
@@ -69,16 +74,12 @@ module Bootsnap
             return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
           end
 
-          begin
-            ::Bootsnap::CompileCache::Native.fetch(
-              Bootsnap::CompileCache::JSON.cache_dir,
-              File.realpath(path),
-              ::Bootsnap::CompileCache::JSON,
-              kwargs,
-            )
-          rescue Errno::EACCES
-            ::Bootsnap::CompileCache.permission_error(path)
-          end
+          ::Bootsnap::CompileCache::Native.fetch(
+            Bootsnap::CompileCache::JSON.cache_dir,
+            File.realpath(path),
+            ::Bootsnap::CompileCache::JSON,
+            kwargs,
+          )
         end
 
         ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -55,15 +55,28 @@ module Bootsnap
         end
 
         def init!
-          require("yaml")
-          require("msgpack")
-          require("date")
+          require "yaml"
+          require "msgpack"
+          require "date"
 
           @implementation = ::YAML::VERSION >= "4" ? Psych4 : Psych3
           if @implementation::Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
             @implementation::Patch.send(:remove_method, :unsafe_load_file)
           end
 
+          unless const_defined?(:NoTagsVisitor)
+            visitor = Class.new(Psych::Visitors::NoAliasRuby) do
+              def visit(target)
+                if target.tag
+                  raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
+                end
+
+                super
+              end
+            end
+            const_set(:NoTagsVisitor, visitor)
+          end
+
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
@@ -102,10 +115,8 @@ module Bootsnap
           if params.include?([:key, :symbolize_names])
             supported_options << :symbolize_names
           end
-          if params.include?([:key, :freeze])
-            if factory.load(factory.dump("yaml"), freeze: true).frozen?
-              supported_options << :freeze
-            end
+          if params.include?([:key, :freeze]) && factory.load(factory.dump("yaml"), freeze: true).frozen?
+            supported_options << :freeze
           end
           supported_options.freeze
         end
@@ -118,19 +129,10 @@ module Bootsnap
           ast = ::YAML.parse(payload)
           return ast unless ast
 
-          strict_visitor.create.visit(ast)
-        end
-
-        def strict_visitor
-          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
-            def visit(target)
-              if target.tag
-                raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
-              end
+          loader = ::Psych::ClassLoader::Restricted.new(["Symbol"], [])
+          scanner = ::Psych::ScalarScanner.new(loader)
 
-              super
-            end
-          end
+          NoTagsVisitor.new(scanner, loader).visit(ast)
         end
       end
 
@@ -227,16 +229,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::SafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::SafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -251,16 +249,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::UnsafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::UnsafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
@@ -307,16 +301,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -331,16 +321,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache.rb

@@ -8,12 +8,11 @@ module Bootsnap
     end
 
     Error = Class.new(StandardError)
-    PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:, json:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:, readonly: false, revalidation: false)
       if iseq
         if supported?
-          require_relative("compile_cache/iseq")
+          require_relative "compile_cache/iseq"
           Bootsnap::CompileCache::ISeq.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] bytecode caching is not supported on this implementation of Ruby")
@@ -22,7 +21,7 @@ module Bootsnap
 
       if yaml
         if supported?
-          require_relative("compile_cache/yaml")
+          require_relative "compile_cache/yaml"
           Bootsnap::CompileCache::YAML.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
@@ -31,28 +30,23 @@ module Bootsnap
 
       if json
         if supported?
-          require_relative("compile_cache/json")
+          require_relative "compile_cache/json"
           Bootsnap::CompileCache::JSON.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
         end
       end
-    end
 
-    def self.permission_error(path)
-      cpath = Bootsnap::CompileCache::ISeq.cache_dir
-      raise(
-        PermissionError,
-        "bootsnap doesn't have permission to write cache entries in '#{cpath}' " \
-        "(or, less likely, doesn't have permission to read '#{path}')",
-      )
+      if supported? && defined?(Bootsnap::CompileCache::Native)
+        Bootsnap::CompileCache::Native.readonly = readonly
+        Bootsnap::CompileCache::Native.revalidation = revalidation
+      end
     end
 
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
-      RUBY_ENGINE == "ruby" &&
-        RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ &&
-        Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
+      # only enable on 'ruby' (MRI) and TruffleRuby for POSIX (darwin, linux, *bsd), Windows (RubyInstaller2)
+      %w[ruby truffleruby].include?(RUBY_ENGINE) &&
+        RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
     end
   end
 end

data/lib/bootsnap/explicit_require.rb

@@ -25,6 +25,11 @@ module Bootsnap
     # This is useful before bootsnap is fully-initialized to load gems that it
     # depends on, without forcing full LOAD_PATH traversals.
     def self.with_gems(*gems)
+      # Ensure the gems are activated (their paths are in $LOAD_PATH)
+      gems.each do |gem_name|
+        gem gem_name
+      end
+
       orig = $LOAD_PATH.dup
       $LOAD_PATH.clear
       gems.each do |gem|