bootsnap 1.11.1 → 1.18.3

data/ext/bootsnap/extconf.rb

@@ -1,26 +1,33 @@
 # frozen_string_literal: true
 
-require("mkmf")
+require "mkmf"
 
-if RUBY_ENGINE == "ruby"
-  $CFLAGS << " -O3 "
-  $CFLAGS << " -std=c99"
+if %w[ruby truffleruby].include?(RUBY_ENGINE)
+  have_func "fdatasync", "unistd.h"
+
+  unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
+    append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME
+  end
+
+  append_cflags ["-O3", "-std=c99"]
 
   # ruby.h has some -Wpedantic fails in some cases
   # (e.g. https://github.com/Shopify/bootsnap/issues/15)
   unless ["0", "", nil].include?(ENV["BOOTSNAP_PEDANTIC"])
-    $CFLAGS << " -Wall"
-    $CFLAGS << " -Werror"
-    $CFLAGS << " -Wextra"
-    $CFLAGS << " -Wpedantic"
+    append_cflags([
+      "-Wall",
+      "-Werror",
+      "-Wextra",
+      "-Wpedantic",
 
-    $CFLAGS << " -Wno-unused-parameter" # VALUE self has to be there but we don't care what it is.
-    $CFLAGS << " -Wno-keyword-macro" # hiding return
-    $CFLAGS << " -Wno-gcc-compat" # ruby.h 2.6.0 on macos 10.14, dunno
-    $CFLAGS << " -Wno-compound-token-split-by-macro"
+      "-Wno-unused-parameter", # VALUE self has to be there but we don't care what it is.
+      "-Wno-keyword-macro", # hiding return
+      "-Wno-gcc-compat", # ruby.h 2.6.0 on macos 10.14, dunno
+      "-Wno-compound-token-split-by-macro",
+    ])
   end
 
   create_makefile("bootsnap/bootsnap")
 else
-  File.write("Makefile", dummy_makefile($srcdir).join(""))
+  File.write("Makefile", dummy_makefile($srcdir).join)
 end

data/lib/bootsnap/bundler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  extend(self)
+  extend self
 
   def bundler?
     return false unless defined?(::Bundler)

data/lib/bootsnap/cli.rb

@@ -60,14 +60,16 @@ module Bootsnap
         precompile_json_files(main_sources)
 
         if compile_gemfile
-          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
-          gem_exclude = Regexp.union([exclude, "/spec/", "/test/"].compact)
-          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
-
           # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
-          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
-          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          # Similarly, gems that include Rails engines generally file Ruby files in `app/`.
+          # However some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, "/spec/", "/test/", "/features/"].compact)
+
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] || p }.uniq
+
+          precompile_ruby_files(gem_paths, exclude: gem_exclude)
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
           precompile_json_files(gem_paths, exclude: gem_exclude)
         end
@@ -138,8 +140,8 @@ module Bootsnap
 
     def precompile_yaml(*yaml_files)
       Array(yaml_files).each do |yaml_file|
-        if CompileCache::YAML.precompile(yaml_file)
-          STDERR.puts(yaml_file) if verbose
+        if CompileCache::YAML.precompile(yaml_file) && verbose
+          $stderr.puts(yaml_file)
         end
       end
     end
@@ -161,8 +163,8 @@ module Bootsnap
 
     def precompile_json(*json_files)
       Array(json_files).each do |json_file|
-        if CompileCache::JSON.precompile(json_file)
-          STDERR.puts(json_file) if verbose
+        if CompileCache::JSON.precompile(json_file) && verbose
+          $stderr.puts(json_file)
         end
       end
     end
@@ -172,7 +174,7 @@ module Bootsnap
 
       load_paths.each do |path|
         if !exclude || !exclude.match?(path)
-          list_files(path, "**/*.rb").each do |ruby_file|
+          list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
             if !exclude || !exclude.match?(ruby_file)
               @work_pool.push(:ruby, ruby_file)
             end
@@ -183,8 +185,8 @@ module Bootsnap
 
     def precompile_ruby(*ruby_files)
       Array(ruby_files).each do |ruby_file|
-        if CompileCache::ISeq.precompile(ruby_file)
-          STDERR.puts(ruby_file) if verbose
+        if CompileCache::ISeq.precompile(ruby_file) && verbose
+          $stderr.puts(ruby_file)
         end
       end
     end
@@ -203,9 +205,9 @@ module Bootsnap
     end
 
     def invalid_usage!(message)
-      STDERR.puts message
-      STDERR.puts
-      STDERR.puts parser
+      $stderr.puts message
+      $stderr.puts
+      $stderr.puts parser
       1
     end
 

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
-require("zlib")
+require "bootsnap/bootsnap"
+require "zlib"
 
 module Bootsnap
   module CompileCache
@@ -12,6 +12,10 @@ module Bootsnap
         def cache_dir=(cache_dir)
           @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
         end
+
+        def supported?
+          CompileCache.supported? && defined?(RubyVM)
+        end
       end
 
       has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
@@ -34,14 +38,14 @@ module Bootsnap
           begin
             iseq.to_binary
           rescue TypeError
-            return UNCOMPILABLE # ruby bug #18250
+            UNCOMPILABLE # ruby bug #18250
           end
         end
       else
         def self.input_to_storage(_, path)
           RubyVM::InstructionSequence.compile_file(path).to_binary
         rescue SyntaxError
-          return UNCOMPILABLE # syntax error
+          UNCOMPILABLE # syntax error
         end
       end
 
@@ -49,7 +53,7 @@ module Bootsnap
         RubyVM::InstructionSequence.load_from_binary(binary)
       rescue RuntimeError => error
         if error.message == "broken binary format"
-          STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+          $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
           nil
         else
           raise
@@ -83,8 +87,6 @@ module Bootsnap
           return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
 
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
-        rescue Errno::EACCES
-          Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => error
           if error.message =~ /unmatched platform/
             puts("unmatched platform for file #{path}")
@@ -103,11 +105,15 @@ module Bootsnap
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
-      compile_option_updated
+      compile_option_updated if supported?
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+
+        return unless supported?
+
         Bootsnap::CompileCache::ISeq.compile_option_updated
+
         class << RubyVM::InstructionSequence
           prepend(InstructionSequenceMixin)
         end

data/lib/bootsnap/compile_cache/json.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -46,18 +46,23 @@ module Bootsnap
         end
 
         def init!
-          require("json")
-          require("msgpack")
+          require "json"
+          require "msgpack"
 
           self.msgpack_factory = MessagePack::Factory.new
           self.supported_options = [:symbolize_names]
-          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
-            if MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
-              self.supported_options = [:freeze]
-            end
+          if supports_freeze?
+            self.supported_options = [:freeze]
           end
           supported_options.freeze
         end
+
+        private
+
+        def supports_freeze?
+          ::JSON.parse('["foo"]', freeze: true).first.frozen? &&
+            MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
+        end
       end
 
       module Patch
@@ -69,16 +74,12 @@ module Bootsnap
             return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
           end
 
-          begin
-            ::Bootsnap::CompileCache::Native.fetch(
-              Bootsnap::CompileCache::JSON.cache_dir,
-              File.realpath(path),
-              ::Bootsnap::CompileCache::JSON,
-              kwargs,
-            )
-          rescue Errno::EACCES
-            ::Bootsnap::CompileCache.permission_error(path)
-          end
+          ::Bootsnap::CompileCache::Native.fetch(
+            Bootsnap::CompileCache::JSON.cache_dir,
+            File.realpath(path),
+            ::Bootsnap::CompileCache::JSON,
+            kwargs,
+          )
         end
 
         ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require("bootsnap/bootsnap")
+require "bootsnap/bootsnap"
 
 module Bootsnap
   module CompileCache
@@ -55,15 +55,28 @@ module Bootsnap
         end
 
         def init!
-          require("yaml")
-          require("msgpack")
-          require("date")
+          require "yaml"
+          require "msgpack"
+          require "date"
 
           @implementation = ::YAML::VERSION >= "4" ? Psych4 : Psych3
           if @implementation::Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
             @implementation::Patch.send(:remove_method, :unsafe_load_file)
           end
 
+          unless const_defined?(:NoTagsVisitor)
+            visitor = Class.new(Psych::Visitors::NoAliasRuby) do
+              def visit(target)
+                if target.tag
+                  raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
+                end
+
+                super
+              end
+            end
+            const_set(:NoTagsVisitor, visitor)
+          end
+
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
@@ -102,10 +115,8 @@ module Bootsnap
           if params.include?([:key, :symbolize_names])
             supported_options << :symbolize_names
           end
-          if params.include?([:key, :freeze])
-            if factory.load(factory.dump("yaml"), freeze: true).frozen?
-              supported_options << :freeze
-            end
+          if params.include?([:key, :freeze]) && factory.load(factory.dump("yaml"), freeze: true).frozen?
+            supported_options << :freeze
           end
           supported_options.freeze
         end
@@ -118,19 +129,10 @@ module Bootsnap
           ast = ::YAML.parse(payload)
           return ast unless ast
 
-          strict_visitor.create.visit(ast)
-        end
-
-        def strict_visitor
-          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
-            def visit(target)
-              if target.tag
-                raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
-              end
+          loader = ::Psych::ClassLoader::Restricted.new(["Symbol"], [])
+          scanner = ::Psych::ScalarScanner.new(loader)
 
-              super
-            end
-          end
+          NoTagsVisitor.new(scanner, loader).visit(ast)
         end
       end
 
@@ -227,16 +229,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::SafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::SafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -251,16 +249,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych4::UnsafeLoad,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych4::UnsafeLoad,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
@@ -307,16 +301,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
@@ -331,16 +321,12 @@ module Bootsnap
               return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
             end
 
-            begin
-              CompileCache::Native.fetch(
-                CompileCache::YAML.cache_dir,
-                File.realpath(path),
-                CompileCache::YAML::Psych3,
-                kwargs,
-              )
-            rescue Errno::EACCES
-              CompileCache.permission_error(path)
-            end
+            CompileCache::Native.fetch(
+              CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              CompileCache::YAML::Psych3,
+              kwargs,
+            )
           end
 
           ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)

data/lib/bootsnap/compile_cache.rb

@@ -8,12 +8,11 @@ module Bootsnap
     end
 
     Error = Class.new(StandardError)
-    PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:, json:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:, readonly: false, revalidation: false)
       if iseq
         if supported?
-          require_relative("compile_cache/iseq")
+          require_relative "compile_cache/iseq"
           Bootsnap::CompileCache::ISeq.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] bytecode caching is not supported on this implementation of Ruby")
@@ -22,7 +21,7 @@ module Bootsnap
 
       if yaml
         if supported?
-          require_relative("compile_cache/yaml")
+          require_relative "compile_cache/yaml"
           Bootsnap::CompileCache::YAML.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
@@ -31,26 +30,23 @@ module Bootsnap
 
       if json
         if supported?
-          require_relative("compile_cache/json")
+          require_relative "compile_cache/json"
           Bootsnap::CompileCache::JSON.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
         end
       end
-    end
 
-    def self.permission_error(path)
-      cpath = Bootsnap::CompileCache::ISeq.cache_dir
-      raise(
-        PermissionError,
-        "bootsnap doesn't have permission to write cache entries in '#{cpath}' " \
-        "(or, less likely, doesn't have permission to read '#{path}')",
-      )
+      if supported? && defined?(Bootsnap::CompileCache::Native)
+        Bootsnap::CompileCache::Native.readonly = readonly
+        Bootsnap::CompileCache::Native.revalidation = revalidation
+      end
     end
 
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
-      RUBY_ENGINE == "ruby" && RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
+      # only enable on 'ruby' (MRI) and TruffleRuby for POSIX (darwin, linux, *bsd), Windows (RubyInstaller2)
+      %w[ruby truffleruby].include?(RUBY_ENGINE) &&
+        RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
     end
   end
 end

data/lib/bootsnap/load_path_cache/cache.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative("../explicit_require")
+require_relative "../explicit_require"
 
 module Bootsnap
   module LoadPathCache
@@ -24,8 +24,16 @@ module Bootsnap
         @mutex.synchronize { @dirs[dir] }
       end
 
+      TRUFFLERUBY_LIB_DIR_PREFIX = if RUBY_ENGINE == "truffleruby"
+        "#{File.join(RbConfig::CONFIG['libdir'], 'truffle')}#{File::SEPARATOR}"
+      end
+
       # { 'enumerator' => nil, 'enumerator.so' => nil, ... }
       BUILTIN_FEATURES = $LOADED_FEATURES.each_with_object({}) do |feat, features|
+        if TRUFFLERUBY_LIB_DIR_PREFIX && feat.start_with?(TRUFFLERUBY_LIB_DIR_PREFIX)
+          feat = feat.byteslice(TRUFFLERUBY_LIB_DIR_PREFIX.bytesize..-1)
+        end
+
         # Builtin features are of the form 'enumerator.so'.
         # All others include paths.
         next unless feat.size < 20 && !feat.include?("/")
@@ -45,20 +53,19 @@ module Bootsnap
 
       # Try to resolve this feature to an absolute path without traversing the
       # loadpath.
-      def find(feature, try_extensions: true)
+      def find(feature)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
         feature = feature.to_s.freeze
 
         return feature if Bootsnap.absolute_path?(feature)
 
         if feature.start_with?("./", "../")
-          return try_extensions ? expand_path(feature) : File.expand_path(feature).freeze
+          return expand_path(feature)
         end
 
         @mutex.synchronize do
-          x = search_index(feature, try_extensions: try_extensions)
+          x = search_index(feature)
           return x if x
-          return unless try_extensions
 
           # Ruby has some built-in features that require lies about.
           # For example, 'enumerator' is built in. If you require it, ruby
@@ -115,7 +122,7 @@ module Bootsnap
       def reinitialize(path_obj = @path_obj)
         @mutex.synchronize do
           @path_obj = path_obj
-          ChangeObserver.register(self, @path_obj)
+          ChangeObserver.register(@path_obj, self)
           @index = {}
           @dirs = {}
           @generated_at = now
@@ -183,15 +190,11 @@ module Bootsnap
       end
 
       if DLEXT2
-        def search_index(feature, try_extensions: true)
-          if try_extensions
-            try_index(feature + DOT_RB) ||
-              try_index(feature + DLEXT) ||
-              try_index(feature + DLEXT2) ||
-              try_index(feature)
-          else
+        def search_index(feature)
+          try_index(feature + DOT_RB) ||
+            try_index(feature + DLEXT) ||
+            try_index(feature + DLEXT2) ||
             try_index(feature)
-          end
         end
 
         def maybe_append_extension(feature)
@@ -201,12 +204,8 @@ module Bootsnap
             feature
         end
       else
-        def search_index(feature, try_extensions: true)
-          if try_extensions
-            try_index(feature + DOT_RB) || try_index(feature + DLEXT) || try_index(feature)
-          else
-            try_index(feature)
-          end
+        def search_index(feature)
+          try_index(feature + DOT_RB) || try_index(feature + DLEXT) || try_index(feature)
         end
 
         def maybe_append_extension(feature)
@@ -216,7 +215,7 @@ module Bootsnap
 
       s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
       if s.respond_to?(:-@)
-        if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= "2.7"
+        if ((-s).equal?(s) && (-s.dup).equal?(s)) || RUBY_VERSION >= "2.7"
           def try_index(feature)
             if (path = @index[feature])
               -File.join(path, feature).freeze

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -54,13 +54,30 @@ module Bootsnap
             ret
           end
         end
+
+        def dup
+          [] + self
+        end
+
+        alias_method :clone, :dup
       end
 
-      def self.register(observer, arr)
+      def self.register(arr, observer)
         return if arr.frozen? # can't register observer, but no need to.
 
         arr.instance_variable_set(:@lpc_observer, observer)
-        arr.extend(ArrayMixin)
+        ArrayMixin.instance_methods.each do |method_name|
+          arr.singleton_class.send(:define_method, method_name, ArrayMixin.instance_method(method_name))
+        end
+      end
+
+      def self.unregister(arr)
+        return unless arr.instance_variable_defined?(:@lpc_observer) && arr.instance_variable_get(:@lpc_observer)
+
+        ArrayMixin.instance_methods.each do |method_name|
+          arr.singleton_class.send(:remove_method, method_name)
+        end
+        arr.instance_variable_set(:@lpc_observer, nil)
       end
     end
   end