bootsnap 1.19.0 → 1.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71cd833fdf912a81069349b4de87a96cf15997710494eebb4535245aeeb02305
-  data.tar.gz: e502571bd2d4863d5f548ce50dcd6502cc18a51dcbf00a101ad1b65f47510534
+  metadata.gz: 4ca69038da7071534a7279f36d694ec5bcf5687272d091b314372b1f3195831b
+  data.tar.gz: c5dc747c2c66a71881d99af980085c06bfc378134d25712c51221b7fb6a6c03a
 SHA512:
-  metadata.gz: d5af0fb8cabbb09d026c8a73043c24d0669004a10edbf0ee3f5baaf8597ac99b7022c0bcaef1c25948dcd055cdd4dbac2ed614d9b2a5d6e58f1e94b6cc32f127
-  data.tar.gz: db4c64829ebc8177ed9a83bda8cf801e570f6d48db3e27ddd4522dbd48043a0d619615ba583be4a9ff1a24049785e5dfeed362f36ef1b9115662e9c3b612973d
+  metadata.gz: 9da2aa5050b26d377f46d1de5e8424168e0a60b00160d43ebdaf9f93c4a9006931bf6463a24afa6dd76559ccedc4ce3798d48733d5000a7a5eac08d7c7a03510
+  data.tar.gz: 17a8da7a372189c22f400f5c059947f98c6aa7836390965c92e6e173970a61e2e6aa57d77d3dd11a9ecea0afd729512a115fd38a21c7a30850d1e9bd662b7d26

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Unreleased
 
+# 1.21.1
+
+* Prevent a Ruby crash while scanning load path if `opendir` fails without setting `errno`.
+  According to the C spec this should not happen, but according to user reports, it did.
+
+# 1.21.0
+
+* Fix the `require` decorator to handle `Bootsnap.unload_cache!` being called.
+* Minor optimization: Eagerly clear cache buffers to appease the GC.
+
+# 1.20.1
+
+* Handle broken symlinks in load path scanning code.
+  Should fix `Errno::ENOENT fstatat` issues some users have encountered after upgrading to 1.20.0.
+
+# 1.20.0
+
+* Optimized load path scanning with a C extension. Should be about 2x faster on supported platforms.
+
 # 1.19.0
 
 * Remove JSON parsing cache. Recent versions of the `json` gem are as fast as `msgpack` if not faster.

data/ext/bootsnap/bootsnap.c

@@ -11,7 +11,6 @@
  * here.
  */
 
-#include "bootsnap.h"
 #include "ruby.h"
 #include <stdint.h>
 #include <stdbool.h>
@@ -20,6 +19,11 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/stat.h>
+#include <dirent.h>
+
+#ifndef RBIMPL_ATTR_NORETURN
+#define RBIMPL_ATTR_NORETURN()
+#endif
 
 #ifdef __APPLE__
   // The symbol is present, however not in the headers
@@ -96,7 +100,6 @@ static mode_t current_umask;
 
 /* Bootsnap::CompileCache::{Native, Uncompilable} */
 static VALUE rb_mBootsnap;
-static VALUE rb_mBootsnap_CompileCache;
 static VALUE rb_mBootsnap_CompileCache_Native;
 static VALUE rb_cBootsnap_CompileCache_UNCOMPILABLE;
 static ID instrumentation_method;
@@ -152,6 +155,118 @@ bs_rb_get_path(VALUE self, VALUE fname)
     return rb_get_path(fname);
 }
 
+#ifdef HAVE_FSTATAT
+
+RBIMPL_ATTR_NORETURN()
+static void
+bs_syserr_fail_path(const char *func_name, int n, VALUE path)
+{
+    rb_syserr_fail_str(n, rb_sprintf("%s @ %s", func_name, RSTRING_PTR(path)));
+}
+
+RBIMPL_ATTR_NORETURN()
+static void
+bs_syserr_fail_dir_entry(const char *func_name, int n, VALUE dir, const char *d_name)
+{
+    rb_syserr_fail_str(n, rb_sprintf("%s @ %s/%s", func_name, RSTRING_PTR(dir), d_name));
+}
+
+static VALUE
+bs_rb_scan_dir(VALUE self, VALUE abspath)
+{
+    Check_Type(abspath, T_STRING);
+
+    DIR *dirp = opendir(RSTRING_PTR(abspath));
+
+    VALUE dirs = rb_ary_new();
+    VALUE requirables = rb_ary_new();
+    VALUE result = rb_ary_new_from_args(2, requirables, dirs);
+
+    if (dirp == NULL) {
+        if (errno == ENOTDIR || errno == ENOENT) {
+            return result;
+        }
+
+        // BUG: Some users reported a crash here because Ruby's syserr trigger
+        // a crash if called with `errno == 0`.
+        // The opendir spec is quite clear that if it returns NULL, then `errno` must
+        // be set, and yet here we are.
+        // So turning no errno into EINVAL, and from there I hope to get to the bottom of things.
+        if (errno == 0) {
+            errno = EINVAL;
+        }
+
+        bs_syserr_fail_path("opendir", errno, abspath);
+        return Qundef;
+    }
+
+    struct dirent *entry;
+    struct stat st;
+    int dfd = -1;
+
+    errno = 0;
+    while ((entry = readdir(dirp))) {
+        if (entry->d_name[0] == '.') continue;
+
+        if (RB_UNLIKELY(entry->d_type == DT_UNKNOWN || entry->d_type == DT_LNK)) {
+            // Note: the original implementation of LoadPathCache did follow symlink.
+            // So this is replicated here, but I'm not sure it's a good idea.
+            if (dfd < 0) {
+                dfd = dirfd(dirp);
+                if (dfd < 0) {
+                    int err = errno;
+                    closedir(dirp);
+                    bs_syserr_fail_path("dirfd", err, abspath);
+                    return Qundef;
+                }
+            }
+
+            if (fstatat(dfd, entry->d_name, &st, 0)) {
+                if (errno == ENOENT) {
+                    // Broken symlinK
+                    errno = 0;
+                    continue;
+                }
+                int err = errno;
+                closedir(dirp);
+                bs_syserr_fail_dir_entry("fstatat", err, abspath, entry->d_name);
+                return Qundef;
+            }
+
+            if (S_ISREG(st.st_mode)) {
+                entry->d_type = DT_REG;
+            } else if (S_ISDIR(st.st_mode)) {
+                entry->d_type = DT_DIR;
+            }
+        }
+
+        if (entry->d_type == DT_DIR) {
+            rb_ary_push(dirs, rb_utf8_str_new_cstr(entry->d_name));
+            continue;
+        } else if (entry->d_type == DT_REG) {
+            size_t len = strlen(entry->d_name);
+            bool is_requirable = (
+                // Comparing 4B allows compiler to optimize this into a single 32b integer comparison.
+                (len > 3 && memcmp(entry->d_name + (len - 3), ".rb", 4) == 0)
+                || (len > DLEXT_MAXLEN && memcmp(entry->d_name + (len - DLEXT_MAXLEN), DLEXT, DLEXT_MAXLEN + 1) == 0)
+#ifdef DLEXT2
+                || (len > DLEXT2_MAXLEN && memcmp(entry->d_name + (len - DLEXT2_MAXLEN), DLEXT2, DLEXT2_MAXLEN + 1) == 0)
+#endif
+            );
+            if (is_requirable) {
+                rb_ary_push(requirables, rb_utf8_str_new(entry->d_name, len));
+            }
+        }
+    }
+
+    if (closedir(dirp)) {
+        bs_syserr_fail_path("closedir", errno, abspath);
+        return Qundef;
+    }
+    return result;
+}
+#endif
+
 /*
  * Ruby C extensions are initialized by calling Init_<extname>.
  *
@@ -166,7 +281,14 @@ Init_bootsnap(void)
 
   rb_define_singleton_method(rb_mBootsnap, "rb_get_path", bs_rb_get_path, 1);
 
-  rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
+#ifdef HAVE_FSTATAT
+  VALUE rb_mBootsnap_LoadPathCache = rb_define_module_under(rb_mBootsnap, "LoadPathCache");
+  VALUE rb_mBootsnap_LoadPathCache_Native = rb_define_module_under(rb_mBootsnap_LoadPathCache, "Native");
+
+  rb_define_singleton_method(rb_mBootsnap_LoadPathCache_Native, "scan_dir", bs_rb_scan_dir, 1);
+#endif
+
+  VALUE rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
   rb_mBootsnap_CompileCache_Native = rb_define_module_under(rb_mBootsnap_CompileCache, "Native");
   rb_cBootsnap_CompileCache_UNCOMPILABLE = rb_const_get(rb_mBootsnap_CompileCache, rb_intern("UNCOMPILABLE"));
   rb_global_variable(&rb_cBootsnap_CompileCache_UNCOMPILABLE);

data/ext/bootsnap/extconf.rb

@@ -4,6 +4,7 @@ require "mkmf"
 
 if %w[ruby truffleruby].include?(RUBY_ENGINE)
   have_func "fdatasync", "unistd.h"
+  have_func "fstatat", "sys/stat.h"
 
   unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
     append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME

data/lib/bootsnap/compile_cache/iseq.rb

@@ -50,7 +50,9 @@ module Bootsnap
       end
 
       def self.storage_to_output(binary, _args)
-        RubyVM::InstructionSequence.load_from_binary(binary)
+        iseq = RubyVM::InstructionSequence.load_from_binary(binary)
+        binary.clear
+        iseq
       rescue RuntimeError => error
         if error.message == "broken binary format"
           $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")

data/lib/bootsnap/compile_cache/yaml.rb

@@ -170,7 +170,9 @@ module Bootsnap
             unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
             unpacker.feed(data)
             _safe_loaded = unpacker.unpack
-            unpacker.unpack
+            result = unpacker.unpack
+            data.clear
+            result
           end
 
           def input_to_output(data, kwargs)

data/lib/bootsnap/load_path_cache/cache.rb

@@ -11,19 +11,19 @@ module Bootsnap
         @development_mode = development_mode
         @store = store
         @mutex = Mutex.new
-        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
+        @path_obj = path_obj.map! do |f|
+          if File.exist?(f)
+            File.realpath(f).freeze
+          elsif f.frozen?
+            f
+          else
+            f.dup.freeze
+          end
+        end
         @has_relative_paths = nil
         reinitialize
       end
 
-      # What is the path item that contains the dir as child?
-      # e.g. given "/a/b/c/d" exists, and the path is ["/a/b"], load_dir("c/d")
-      # is "/a/b".
-      def load_dir(dir)
-        reinitialize if stale?
-        @mutex.synchronize { @dirs[dir] }
-      end
-
       TRUFFLERUBY_LIB_DIR_PREFIX = if RUBY_ENGINE == "truffleruby"
         "#{File.join(RbConfig::CONFIG['libdir'], 'truffle')}#{File::SEPARATOR}"
       end
@@ -124,7 +124,6 @@ module Bootsnap
           @path_obj = path_obj
           ChangeObserver.register(@path_obj, self)
           @index = {}
-          @dirs = {}
           @generated_at = now
           push_paths_locked(*@path_obj)
         end
@@ -152,9 +151,8 @@ module Bootsnap
             p = p.to_realpath
 
             expanded_path = p.expanded_path
-            entries, dirs = p.entries_and_dirs(@store)
+            entries = p.entries(@store)
             # push -> low precedence -> set only if unset
-            dirs.each    { |dir| @dirs[dir] ||= path }
             entries.each { |rel| @index[rel] ||= expanded_path }
           end
         end
@@ -169,9 +167,8 @@ module Bootsnap
             p = p.to_realpath
 
             expanded_path = p.expanded_path
-            entries, dirs = p.entries_and_dirs(@store)
+            entries = p.entries(@store)
             # unshift -> high precedence -> unconditional set
-            dirs.each    { |dir| @dirs[dir]  = path }
             entries.each { |rel| @index[rel] = expanded_path }
           end
         end

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -15,8 +15,11 @@ module Kernel
     if Bootsnap::LoadPathCache::FALLBACK_SCAN.equal?(resolved)
       if (cursor = Bootsnap::LoadPathCache.loaded_features_index.cursor(string_path))
         ret = require_without_bootsnap(path)
-        resolved = Bootsnap::LoadPathCache.loaded_features_index.identify(string_path, cursor)
-        Bootsnap::LoadPathCache.loaded_features_index.register(string_path, resolved)
+
+        # The file we required may have unloaded the cache
+        resolved = Bootsnap::LoadPathCache.loaded_features_index&.identify(string_path, cursor)
+        Bootsnap::LoadPathCache.loaded_features_index&.register(string_path, resolved)
+
         return ret
       else
         return require_without_bootsnap(path)
@@ -28,7 +31,9 @@ module Kernel
     else
       # Note that require registers to $LOADED_FEATURES while load does not.
       ret = require_without_bootsnap(resolved)
-      Bootsnap::LoadPathCache.loaded_features_index.register(string_path, resolved)
+
+      # The file we required may have unloaded the cache
+      Bootsnap::LoadPathCache.loaded_features_index&.register(string_path, resolved)
       return ret
     end
   end

data/lib/bootsnap/load_path_cache/path.rb

@@ -54,29 +54,28 @@ module Bootsnap
         !path.start_with?(SLASH)
       end
 
-      # Return a list of all the requirable files and all of the subdirectories
-      # of this +Path+.
-      def entries_and_dirs(store)
+      # Return a list of all the requirable files of this +Path+.
+      def entries(store)
         if stable?
           # the cached_mtime field is unused for 'stable' paths, but is
           # set to zero anyway, just in case we change the stability heuristics.
-          _, entries, dirs = store.get(expanded_path)
-          return [entries, dirs] if entries # cache hit
+          _, entries, = store.get(expanded_path)
+          return entries if entries # cache hit
 
-          entries, dirs = scan!
-          store.set(expanded_path, [0, entries, dirs])
-          return [entries, dirs]
+          entries = PathScanner.call(expanded_path)
+          store.set(expanded_path, [0, entries])
+          return entries
         end
 
-        cached_mtime, entries, dirs = store.get(expanded_path)
+        cached_mtime, entries = store.get(expanded_path)
 
-        current_mtime = latest_mtime(expanded_path, dirs || [])
-        return [[], []]        if current_mtime == -1 # path does not exist
-        return [entries, dirs] if cached_mtime == current_mtime
+        current_mtime = latest_mtime(expanded_path, entries || [])
+        return [] if current_mtime == -1 # path does not exist
+        return entries if cached_mtime == current_mtime
 
-        entries, dirs = scan!
-        store.set(expanded_path, [current_mtime, entries, dirs])
-        [entries, dirs]
+        entries = PathScanner.call(expanded_path)
+        store.set(expanded_path, [current_mtime, entries])
+        entries
       end
 
       def expanded_path
@@ -89,23 +88,31 @@ module Bootsnap
 
       private
 
-      def scan! # (expensive) returns [entries, dirs]
-        PathScanner.call(expanded_path)
-      end
-
       # last time a directory was modified in this subtree. +dirs+ should be a
       # list of relative paths to directories under +path+. e.g. for /a/b and
       # /a/b/c, pass ('/a/b', ['c'])
-      def latest_mtime(path, dirs)
-        max = -1
-        ["", *dirs].each do |dir|
-          curr = begin
-            File.mtime("#{path}/#{dir}").to_i
-                 rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
-                   -1
+      def latest_mtime(root_path, entries)
+        max = begin
+          File.mtime(root_path).to_i
+        rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
+          -1
+        end
+
+        visited = {"." => true}
+
+        entries.each do |relpath|
+          dirname = File.dirname(relpath)
+          visited[dirname] ||= begin
+            begin
+              current = File.mtime(File.join(root_path, dirname)).to_i
+              max = current if current > max
+            rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
+              # ignore
+            end
+            true
           end
-          max = curr if curr > max
         end
+
         max
       end
 

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -6,8 +6,6 @@ module Bootsnap
   module LoadPathCache
     module PathScanner
       REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS
-      NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
-      ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/.freeze
 
       BUNDLE_PATH = if Bootsnap.bundler?
         (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
@@ -20,9 +18,9 @@ module Bootsnap
       class << self
         attr_accessor :ignored_directories
 
-        def call(path)
+        def ruby_call(path)
           path = File.expand_path(path.to_s).freeze
-          return [[], []] unless File.directory?(path)
+          return [] unless File.directory?(path)
 
           # If the bundle path is a descendent of this path, we do additional
           # checks to prevent recursing into the bundle path as we recurse
@@ -33,17 +31,15 @@ module Bootsnap
           # and the bundle path is '.bundle'.
           contains_bundle_path = BUNDLE_PATH.start_with?(path)
 
-          dirs = []
           requirables = []
           walk(path, nil) do |relative_path, absolute_path, is_directory|
             if is_directory
-              dirs << os_path(relative_path)
               !contains_bundle_path || !absolute_path.start_with?(BUNDLE_PATH)
             elsif relative_path.end_with?(*REQUIRABLE_EXTENSIONS)
-              requirables << os_path(relative_path)
+              requirables << relative_path.freeze
             end
           end
-          [requirables, dirs]
+          requirables
         end
 
         def walk(absolute_dir_path, relative_dir_path, &block)
@@ -65,15 +61,58 @@ module Bootsnap
           end
         end
 
-        if RUBY_VERSION >= "3.1"
-          def os_path(path)
-            path.freeze
+        if RUBY_ENGINE == "ruby" && RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
+          require "bootsnap/bootsnap"
+        end
+
+        if defined?(Native.scan_dir)
+          def native_call(root_path)
+            # NOTE: if https://bugs.ruby-lang.org/issues/21800 is accepted we should be able
+            # to have similar performance with pure Ruby
+
+            # If the bundle path is a descendent of this path, we do additional
+            # checks to prevent recursing into the bundle path as we recurse
+            # through this path. We don't want to scan the bundle path because
+            # anything useful in it will be present on other load path items.
+            #
+            # This can happen if, for example, the user adds '.' to the load path,
+            # and the bundle path is '.bundle'.
+            contains_bundle_path = BUNDLE_PATH.start_with?(root_path)
+
+            all_requirables, queue = Native.scan_dir(root_path)
+            all_requirables.each(&:freeze)
+
+            queue.reject! do |dir|
+              ignored_directories.include?(dir) ||
+                (contains_bundle_path && dir.start_with?(BUNDLE_PATH))
+            end
+
+            while (path = queue.pop)
+              requirables, dirs = Native.scan_dir(File.join(root_path, path))
+              dirs.reject! { |dir| ignored_directories.include?(dir) }
+              dirs.map! { |f| File.join(path, f).freeze }
+              requirables.map! { |f| File.join(path, f).freeze }
+
+              if contains_bundle_path
+                dirs.reject! { |dir| dir.start_with?(BUNDLE_PATH) }
+              end
+
+              all_requirables.concat(requirables)
+              queue.concat(dirs)
+            end
+
+            all_requirables
+          rescue SystemCallError => error
+            if ENV["BOOTSNAP_DEBUG"]
+              raise
+            else
+              Bootsnap.logger&.call("Unexpected error: #{error.class}: #{error.message}")
+              ruby_call(root_path)
+            end
           end
+          alias_method :call, :native_call
         else
-          def os_path(path)
-            path.force_encoding(Encoding::US_ASCII) if path.ascii_only?
-            path.freeze
-          end
+          alias_method :call, :ruby_call
         end
       end
     end

data/lib/bootsnap/load_path_cache/store.rb

@@ -8,7 +8,7 @@ module Bootsnap
   module LoadPathCache
     class Store
       VERSION_KEY = "__bootsnap_ruby_version__"
-      CURRENT_VERSION = "#{RUBY_REVISION}-#{RUBY_PLATFORM}".freeze # rubocop:disable Style/RedundantFreeze
+      CURRENT_VERSION = "#{VERSION}-#{RUBY_REVISION}-#{RUBY_PLATFORM}".freeze # rubocop:disable Style/RedundantFreeze
 
       NestedTransactionError = Class.new(StandardError)
       SetOutsideTransactionNotAllowed = Class.new(StandardError)

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.19.0"
+  VERSION = "1.21.1"
 end

data/lib/bootsnap.rb

@@ -2,8 +2,6 @@
 
 require_relative "bootsnap/version"
 require_relative "bootsnap/bundler"
-require_relative "bootsnap/load_path_cache"
-require_relative "bootsnap/compile_cache"
 
 module Bootsnap
   InvalidConfiguration = Class.new(StandardError)
@@ -164,3 +162,6 @@ module Bootsnap
     end
   end
 end
+
+require_relative "bootsnap/compile_cache"
+require_relative "bootsnap/load_path_cache"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.19.0
+  version: 1.21.1
 platform: ruby
 authors:
 - Burke Libbey
@@ -37,7 +37,6 @@ files:
 - README.md
 - exe/bootsnap
 - ext/bootsnap/bootsnap.c
-- ext/bootsnap/bootsnap.h
 - ext/bootsnap/extconf.rb
 - lib/bootsnap.rb
 - lib/bootsnap/bundler.rb
@@ -80,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Boot large ruby/rails apps faster
 test_files: []

data/ext/bootsnap/bootsnap.h

@@ -1,6 +0,0 @@
-#ifndef BOOTSNAP_H
-#define BOOTSNAP_H 1
-
-/* doesn't expose anything */
-
-#endif /* BOOTSNAP_H */