bootsnap 1.19.0 → 1.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71cd833fdf912a81069349b4de87a96cf15997710494eebb4535245aeeb02305
-  data.tar.gz: e502571bd2d4863d5f548ce50dcd6502cc18a51dcbf00a101ad1b65f47510534
+  metadata.gz: 8e0a92c2964b6f614db5d8bf1636026c49796a74ae82b1cf78d2e3ec39337fa2
+  data.tar.gz: 8f2e985c4af37b7056b0ddbee9fdd98b31839eb50ce2bf028d3089d92c6a4a88
 SHA512:
-  metadata.gz: d5af0fb8cabbb09d026c8a73043c24d0669004a10edbf0ee3f5baaf8597ac99b7022c0bcaef1c25948dcd055cdd4dbac2ed614d9b2a5d6e58f1e94b6cc32f127
-  data.tar.gz: db4c64829ebc8177ed9a83bda8cf801e570f6d48db3e27ddd4522dbd48043a0d619615ba583be4a9ff1a24049785e5dfeed362f36ef1b9115662e9c3b612973d
+  metadata.gz: e539a95b7f098a337b97a91cd9c29849b2734fac66716cb46b29bcb8e67c7fddd6b0ca07f3a0a6d9ecf6dca2d28b99f0ee5ae9743193f1b7f3ef0994117391d8
+  data.tar.gz: 56b9d42182d85530439defae95000e4aceec364e6ba4a1e69815127a3a5c782b330cef032c5d6f0ceb9b19ca2da30c6bc192eac33c878664e878772134137545

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Unreleased
 
+# 1.20.0
+
+* Optimized load path scanning with a C extension. Should be about 2x faster on supported platforms.
+
 # 1.19.0
 
 * Remove JSON parsing cache. Recent versions of the `json` gem are as fast as `msgpack` if not faster.

data/ext/bootsnap/bootsnap.c

@@ -11,7 +11,6 @@
  * here.
  */
 
-#include "bootsnap.h"
 #include "ruby.h"
 #include <stdint.h>
 #include <stdbool.h>
@@ -20,6 +19,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/stat.h>
+#include <dirent.h>
 
 #ifdef __APPLE__
   // The symbol is present, however not in the headers
@@ -96,7 +96,6 @@ static mode_t current_umask;
 
 /* Bootsnap::CompileCache::{Native, Uncompilable} */
 static VALUE rb_mBootsnap;
-static VALUE rb_mBootsnap_CompileCache;
 static VALUE rb_mBootsnap_CompileCache_Native;
 static VALUE rb_cBootsnap_CompileCache_UNCOMPILABLE;
 static ID instrumentation_method;
@@ -152,6 +151,84 @@ bs_rb_get_path(VALUE self, VALUE fname)
     return rb_get_path(fname);
 }
 
+#ifdef HAVE_FSTATAT
+static VALUE
+bs_rb_scan_dir(VALUE self, VALUE abspath)
+{
+    Check_Type(abspath, T_STRING);
+
+    DIR *dirp = opendir(RSTRING_PTR(abspath));
+
+    VALUE dirs = rb_ary_new();
+    VALUE requirables = rb_ary_new();
+    VALUE result = rb_ary_new_from_args(2, requirables, dirs);
+
+    if (dirp == NULL) {
+        if (errno == ENOTDIR || errno == ENOENT) {
+            return result;
+        }
+        rb_sys_fail("opendir");
+        return Qundef;
+    }
+
+    struct dirent *entry;
+    struct stat st;
+    int dfd = -1;
+
+    errno = 0;
+    while ((entry = readdir(dirp))) {
+        if (entry->d_name[0] == '.') continue;
+
+        if (RB_UNLIKELY(entry->d_type == DT_UNKNOWN || entry->d_type == DT_LNK)) {
+            // Note: the original implementation of LoadPathCache did follow symlink.
+            // So this is replicated here, but I'm not sure it's a good idea.
+            if (dfd < 0) {
+                dfd = dirfd(dirp);
+                if (dfd < 0) {
+                    rb_sys_fail("dirfd");
+                    return Qundef;
+                }
+            }
+
+            if (fstatat(dfd, entry->d_name, &st, 0)) {
+                rb_sys_fail("fstatat");
+                return Qundef;
+            }
+
+            if (S_ISREG(st.st_mode)) {
+                entry->d_type = DT_REG;
+            } else if (S_ISDIR(st.st_mode)) {
+                entry->d_type = DT_DIR;
+            }
+        }
+
+        if (entry->d_type == DT_DIR) {
+            rb_ary_push(dirs, rb_utf8_str_new_cstr(entry->d_name));
+            continue;
+        } else if (entry->d_type == DT_REG) {
+            size_t len = strlen(entry->d_name);
+            bool is_requirable = (
+                // Comparing 4B allows compiler to optimize this into a single 32b integer comparison.
+                (len > 3 && memcmp(entry->d_name + (len - 3), ".rb", 4) == 0)
+                || (len > DLEXT_MAXLEN && memcmp(entry->d_name + (len - DLEXT_MAXLEN), DLEXT, DLEXT_MAXLEN + 1) == 0)
+#ifdef DLEXT2
+                || (len > DLEXT2_MAXLEN && memcmp(entry->d_name + (len - DLEXT2_MAXLEN), DLEXT2, DLEXT2_MAXLEN + 1) == 0)
+#endif
+            );
+            if (is_requirable) {
+                rb_ary_push(requirables, rb_utf8_str_new(entry->d_name, len));
+            }
+        }
+    }
+
+    if (closedir(dirp)) {
+        rb_sys_fail("closedir");
+        return Qundef;
+    }
+    return result;
+}
+#endif
+
 /*
  * Ruby C extensions are initialized by calling Init_<extname>.
  *
@@ -166,7 +243,14 @@ Init_bootsnap(void)
 
   rb_define_singleton_method(rb_mBootsnap, "rb_get_path", bs_rb_get_path, 1);
 
-  rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
+#ifdef HAVE_FSTATAT
+  VALUE rb_mBootsnap_LoadPathCache = rb_define_module_under(rb_mBootsnap, "LoadPathCache");
+  VALUE rb_mBootsnap_LoadPathCache_Native = rb_define_module_under(rb_mBootsnap_LoadPathCache, "Native");
+
+  rb_define_singleton_method(rb_mBootsnap_LoadPathCache_Native, "scan_dir", bs_rb_scan_dir, 1);
+#endif
+
+  VALUE rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
   rb_mBootsnap_CompileCache_Native = rb_define_module_under(rb_mBootsnap_CompileCache, "Native");
   rb_cBootsnap_CompileCache_UNCOMPILABLE = rb_const_get(rb_mBootsnap_CompileCache, rb_intern("UNCOMPILABLE"));
   rb_global_variable(&rb_cBootsnap_CompileCache_UNCOMPILABLE);

data/ext/bootsnap/extconf.rb

@@ -4,6 +4,7 @@ require "mkmf"
 
 if %w[ruby truffleruby].include?(RUBY_ENGINE)
   have_func "fdatasync", "unistd.h"
+  have_func "fstatat", "sys/stat.h"
 
   unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
     append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME

data/lib/bootsnap/load_path_cache/cache.rb

@@ -11,19 +11,19 @@ module Bootsnap
         @development_mode = development_mode
         @store = store
         @mutex = Mutex.new
-        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
+        @path_obj = path_obj.map! do |f|
+          if File.exist?(f)
+            File.realpath(f).freeze
+          elsif f.frozen?
+            f
+          else
+            f.dup.freeze
+          end
+        end
         @has_relative_paths = nil
         reinitialize
       end
 
-      # What is the path item that contains the dir as child?
-      # e.g. given "/a/b/c/d" exists, and the path is ["/a/b"], load_dir("c/d")
-      # is "/a/b".
-      def load_dir(dir)
-        reinitialize if stale?
-        @mutex.synchronize { @dirs[dir] }
-      end
-
       TRUFFLERUBY_LIB_DIR_PREFIX = if RUBY_ENGINE == "truffleruby"
         "#{File.join(RbConfig::CONFIG['libdir'], 'truffle')}#{File::SEPARATOR}"
       end
@@ -124,7 +124,6 @@ module Bootsnap
           @path_obj = path_obj
           ChangeObserver.register(@path_obj, self)
           @index = {}
-          @dirs = {}
           @generated_at = now
           push_paths_locked(*@path_obj)
         end
@@ -152,9 +151,8 @@ module Bootsnap
             p = p.to_realpath
 
             expanded_path = p.expanded_path
-            entries, dirs = p.entries_and_dirs(@store)
+            entries = p.entries(@store)
             # push -> low precedence -> set only if unset
-            dirs.each    { |dir| @dirs[dir] ||= path }
             entries.each { |rel| @index[rel] ||= expanded_path }
           end
         end
@@ -169,9 +167,8 @@ module Bootsnap
             p = p.to_realpath
 
             expanded_path = p.expanded_path
-            entries, dirs = p.entries_and_dirs(@store)
+            entries = p.entries(@store)
             # unshift -> high precedence -> unconditional set
-            dirs.each    { |dir| @dirs[dir]  = path }
             entries.each { |rel| @index[rel] = expanded_path }
           end
         end

data/lib/bootsnap/load_path_cache/path.rb

@@ -54,29 +54,28 @@ module Bootsnap
         !path.start_with?(SLASH)
       end
 
-      # Return a list of all the requirable files and all of the subdirectories
-      # of this +Path+.
-      def entries_and_dirs(store)
+      # Return a list of all the requirable files of this +Path+.
+      def entries(store)
         if stable?
           # the cached_mtime field is unused for 'stable' paths, but is
           # set to zero anyway, just in case we change the stability heuristics.
-          _, entries, dirs = store.get(expanded_path)
-          return [entries, dirs] if entries # cache hit
+          _, entries, = store.get(expanded_path)
+          return entries if entries # cache hit
 
-          entries, dirs = scan!
-          store.set(expanded_path, [0, entries, dirs])
-          return [entries, dirs]
+          entries = PathScanner.call(expanded_path)
+          store.set(expanded_path, [0, entries])
+          return entries
         end
 
-        cached_mtime, entries, dirs = store.get(expanded_path)
+        cached_mtime, entries = store.get(expanded_path)
 
-        current_mtime = latest_mtime(expanded_path, dirs || [])
-        return [[], []]        if current_mtime == -1 # path does not exist
-        return [entries, dirs] if cached_mtime == current_mtime
+        current_mtime = latest_mtime(expanded_path, entries || [])
+        return [] if current_mtime == -1 # path does not exist
+        return entries if cached_mtime == current_mtime
 
-        entries, dirs = scan!
-        store.set(expanded_path, [current_mtime, entries, dirs])
-        [entries, dirs]
+        entries = PathScanner.call(expanded_path)
+        store.set(expanded_path, [current_mtime, entries])
+        entries
       end
 
       def expanded_path
@@ -89,23 +88,31 @@ module Bootsnap
 
       private
 
-      def scan! # (expensive) returns [entries, dirs]
-        PathScanner.call(expanded_path)
-      end
-
       # last time a directory was modified in this subtree. +dirs+ should be a
       # list of relative paths to directories under +path+. e.g. for /a/b and
       # /a/b/c, pass ('/a/b', ['c'])
-      def latest_mtime(path, dirs)
-        max = -1
-        ["", *dirs].each do |dir|
-          curr = begin
-            File.mtime("#{path}/#{dir}").to_i
-                 rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
-                   -1
+      def latest_mtime(root_path, entries)
+        max = begin
+          File.mtime(root_path).to_i
+        rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
+          -1
+        end
+
+        visited = {"." => true}
+
+        entries.each do |relpath|
+          dirname = File.dirname(relpath)
+          visited[dirname] ||= begin
+            begin
+              current = File.mtime(File.join(root_path, dirname)).to_i
+              max = current if current > max
+            rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
+              # ignore
+            end
+            true
           end
-          max = curr if curr > max
         end
+
         max
       end
 

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -6,8 +6,6 @@ module Bootsnap
   module LoadPathCache
     module PathScanner
       REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS
-      NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
-      ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/.freeze
 
       BUNDLE_PATH = if Bootsnap.bundler?
         (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
@@ -20,9 +18,9 @@ module Bootsnap
       class << self
         attr_accessor :ignored_directories
 
-        def call(path)
+        def ruby_call(path)
           path = File.expand_path(path.to_s).freeze
-          return [[], []] unless File.directory?(path)
+          return [] unless File.directory?(path)
 
           # If the bundle path is a descendent of this path, we do additional
           # checks to prevent recursing into the bundle path as we recurse
@@ -33,17 +31,15 @@ module Bootsnap
           # and the bundle path is '.bundle'.
           contains_bundle_path = BUNDLE_PATH.start_with?(path)
 
-          dirs = []
           requirables = []
           walk(path, nil) do |relative_path, absolute_path, is_directory|
             if is_directory
-              dirs << os_path(relative_path)
               !contains_bundle_path || !absolute_path.start_with?(BUNDLE_PATH)
             elsif relative_path.end_with?(*REQUIRABLE_EXTENSIONS)
-              requirables << os_path(relative_path)
+              requirables << relative_path.freeze
             end
           end
-          [requirables, dirs]
+          requirables
         end
 
         def walk(absolute_dir_path, relative_dir_path, &block)
@@ -65,15 +61,51 @@ module Bootsnap
           end
         end
 
-        if RUBY_VERSION >= "3.1"
-          def os_path(path)
-            path.freeze
+        if RUBY_ENGINE == "ruby" && RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
+          require "bootsnap/bootsnap"
+        end
+
+        if defined?(Native.scan_dir)
+          def native_call(root_path)
+            # NOTE: if https://bugs.ruby-lang.org/issues/21800 is accepted we should be able
+            # to have similar performance with pure Ruby
+
+            # If the bundle path is a descendent of this path, we do additional
+            # checks to prevent recursing into the bundle path as we recurse
+            # through this path. We don't want to scan the bundle path because
+            # anything useful in it will be present on other load path items.
+            #
+            # This can happen if, for example, the user adds '.' to the load path,
+            # and the bundle path is '.bundle'.
+            contains_bundle_path = BUNDLE_PATH.start_with?(root_path)
+
+            all_requirables, queue = Native.scan_dir(root_path)
+            all_requirables.each(&:freeze)
+
+            queue.reject! do |dir|
+              ignored_directories.include?(dir) ||
+                (contains_bundle_path && dir.start_with?(BUNDLE_PATH))
+            end
+
+            while (path = queue.pop)
+              requirables, dirs = Native.scan_dir(File.join(root_path, path))
+              dirs.reject! { |dir| ignored_directories.include?(dir) }
+              dirs.map! { |f| File.join(path, f).freeze }
+              requirables.map! { |f| File.join(path, f).freeze }
+
+              if contains_bundle_path
+                dirs.reject! { |dir| dir.start_with?(BUNDLE_PATH) }
+              end
+
+              all_requirables.concat(requirables)
+              queue.concat(dirs)
+            end
+
+            all_requirables
           end
+          alias_method :call, :native_call
         else
-          def os_path(path)
-            path.force_encoding(Encoding::US_ASCII) if path.ascii_only?
-            path.freeze
-          end
+          alias_method :call, :ruby_call
         end
       end
     end

data/lib/bootsnap/load_path_cache/store.rb

@@ -8,7 +8,7 @@ module Bootsnap
   module LoadPathCache
     class Store
       VERSION_KEY = "__bootsnap_ruby_version__"
-      CURRENT_VERSION = "#{RUBY_REVISION}-#{RUBY_PLATFORM}".freeze # rubocop:disable Style/RedundantFreeze
+      CURRENT_VERSION = "#{VERSION}-#{RUBY_REVISION}-#{RUBY_PLATFORM}".freeze # rubocop:disable Style/RedundantFreeze
 
       NestedTransactionError = Class.new(StandardError)
       SetOutsideTransactionNotAllowed = Class.new(StandardError)

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.19.0"
+  VERSION = "1.20.0"
 end

data/lib/bootsnap.rb

@@ -2,8 +2,8 @@
 
 require_relative "bootsnap/version"
 require_relative "bootsnap/bundler"
-require_relative "bootsnap/load_path_cache"
 require_relative "bootsnap/compile_cache"
+require_relative "bootsnap/load_path_cache"
 
 module Bootsnap
   InvalidConfiguration = Class.new(StandardError)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.19.0
+  version: 1.20.0
 platform: ruby
 authors:
 - Burke Libbey
@@ -37,7 +37,6 @@ files:
 - README.md
 - exe/bootsnap
 - ext/bootsnap/bootsnap.c
-- ext/bootsnap/bootsnap.h
 - ext/bootsnap/extconf.rb
 - lib/bootsnap.rb
 - lib/bootsnap/bundler.rb
@@ -80,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Boot large ruby/rails apps faster
 test_files: []

data/ext/bootsnap/bootsnap.h

@@ -1,6 +0,0 @@
-#ifndef BOOTSNAP_H
-#define BOOTSNAP_H 1
-
-/* doesn't expose anything */
-
-#endif /* BOOTSNAP_H */