bootsnap 1.12.0 → 1.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 761e97c3b4772e9c0af1bd8391d70dddebd94addf88e9295e72a790a85deb5e0
-  data.tar.gz: 17f80fcf67adc6a4e7f9fb475d3e7d5c008c89b491e23804da1f2603e859888c
+  metadata.gz: 6f2b4691e78336f28b1f78e9751235bc558b0f6ad86dee541acd478fc23ef510
+  data.tar.gz: 51eb44a77465f49db0d15425536e17d94f8d3fe77433ab48260b4c81613d9c72
 SHA512:
-  metadata.gz: dca99dc5e57a644e627e8705958c8c4916a9c444c2a64788fd4214aa174d6c3eed7fc15ef7bdb6e06922fb8d1eef3beeff08e316aadbc3015a36e82b1a232309
-  data.tar.gz: 1c9ffc9c5c04694f4838693edda976f58925649f7b14411d61bec64eca10bb04548c5bb4e2be6ff897c8d5a0e929aa2cd885d017ccd1770530971fedba831ec3
+  metadata.gz: 64e98271863cf3dec676eea43ecaecf4d97a543ba5c0d840160d9862b4f212906910e3654263db49457320b5783150c719a8ef479266be96df5243a04addc43b
+  data.tar.gz: 46aa01dff95699d6148ed539c8a56e1d749801ee89e953a37f32b4a3974f1da99fe594c4fb1b04c99a42f030cb4b12bd6dd892fac5b00af4de1f7a560cb420ba

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Unreleased
 
+# 1.14.0
+
+* Require Ruby 2.6.
+* Add a way to skip directories during load path scanning.
+  If you have large non-ruby directories in the middle of your load path, it can severly slow down scanning.
+  Typically this is a problem with `node_modules`. See #277.
+* Fix `Bootsnap.unload_cache!`, it simply wouldn't work at all becaue of a merge mistake. See #421.
+
+# 1.13.0
+
+* Stop decorating `Kernel.load`. This used to be very useful in development because the Rails "classic" autoloader
+  was using `Kernel.load` in dev and `Kernel.require` in production. But Zeitwerk is now the default, and it doesn't
+  use `Kernel.load` at all.
+
+  People still using the classic autoloader might want to stick to `bootsnap 1.12`.
+
+* Add `Bootsnap.unload_cache!`. Applications can call it at the end of their boot sequence when they know
+  no more code will be loaded to reclaim a bit of memory.
+
 # 1.12.0
 
 * `bootsnap precompile` CLI will now also precompile `Rakefile` and `.rake` files.

data/README.md

@@ -52,10 +52,11 @@ require 'bootsnap'
 env = ENV['RAILS_ENV'] || "development"
 Bootsnap.setup(
   cache_dir:            'tmp/cache',          # Path to your cache
+  ignored_directories   ['node_modules'],     # Directory names to skip.
   development_mode:     env == 'development', # Current working environment, e.g. RACK_ENV, RAILS_ENV, etc
   load_path_cache:      true,                 # Optimize the LOAD_PATH with a cache
   compile_cache_iseq:   true,                 # Compile Ruby code into ISeq cache, breaks coverage reporting.
-  compile_cache_yaml:   true                  # Compile YAML into a cache
+  compile_cache_yaml:   true,                 # Compile YAML into a cache
 )
 ```
 
@@ -77,6 +78,9 @@ well together, and are both included in a newly-generated Rails applications by
 - `DISABLE_BOOTSNAP_LOAD_PATH_CACHE` allows to disable load path caching.
 - `DISABLE_BOOTSNAP_COMPILE_CACHE` allows to disable ISeq and YAML caches.
 - `BOOTSNAP_LOG` configure bootsnap to log all caches misses to STDERR.
+- `BOOTSNAP_IGNORE_DIRECTORIES` a comma separated list of directories that shouldn't be scanned.
+  Useful when you have large directories of non-ruby files inside `$LOAD_PATH`.
+  It default to ignore any directory named `node_modules`.
 
 ### Environments
 

data/ext/bootsnap/extconf.rb

@@ -22,5 +22,5 @@ if RUBY_ENGINE == "ruby"
 
   create_makefile("bootsnap/bootsnap")
 else
-  File.write("Makefile", dummy_makefile($srcdir).join(""))
+  File.write("Makefile", dummy_makefile($srcdir).join)
 end

data/lib/bootsnap/cli.rb

@@ -66,7 +66,7 @@ module Bootsnap
 
           # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
-          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
           gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
           precompile_json_files(gem_paths, exclude: gem_exclude)
@@ -138,8 +138,8 @@ module Bootsnap
 
     def precompile_yaml(*yaml_files)
       Array(yaml_files).each do |yaml_file|
-        if CompileCache::YAML.precompile(yaml_file)
-          STDERR.puts(yaml_file) if verbose
+        if CompileCache::YAML.precompile(yaml_file) && verbose
+          $stderr.puts(yaml_file)
         end
       end
     end
@@ -161,8 +161,8 @@ module Bootsnap
 
     def precompile_json(*json_files)
       Array(json_files).each do |json_file|
-        if CompileCache::JSON.precompile(json_file)
-          STDERR.puts(json_file) if verbose
+        if CompileCache::JSON.precompile(json_file) && verbose
+          $stderr.puts(json_file)
         end
       end
     end
@@ -183,8 +183,8 @@ module Bootsnap
 
     def precompile_ruby(*ruby_files)
       Array(ruby_files).each do |ruby_file|
-        if CompileCache::ISeq.precompile(ruby_file)
-          STDERR.puts(ruby_file) if verbose
+        if CompileCache::ISeq.precompile(ruby_file) && verbose
+          $stderr.puts(ruby_file)
         end
       end
     end
@@ -203,9 +203,9 @@ module Bootsnap
     end
 
     def invalid_usage!(message)
-      STDERR.puts message
-      STDERR.puts
-      STDERR.puts parser
+      $stderr.puts message
+      $stderr.puts
+      $stderr.puts parser
       1
     end
 

data/lib/bootsnap/compile_cache/iseq.rb

@@ -34,14 +34,14 @@ module Bootsnap
           begin
             iseq.to_binary
           rescue TypeError
-            return UNCOMPILABLE # ruby bug #18250
+            UNCOMPILABLE # ruby bug #18250
           end
         end
       else
         def self.input_to_storage(_, path)
           RubyVM::InstructionSequence.compile_file(path).to_binary
         rescue SyntaxError
-          return UNCOMPILABLE # syntax error
+          UNCOMPILABLE # syntax error
         end
       end
 
@@ -49,7 +49,7 @@ module Bootsnap
         RubyVM::InstructionSequence.load_from_binary(binary)
       rescue RuntimeError => error
         if error.message == "broken binary format"
-          STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+          $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
           nil
         else
           raise

data/lib/bootsnap/compile_cache/json.rb

@@ -51,13 +51,18 @@ module Bootsnap
 
           self.msgpack_factory = MessagePack::Factory.new
           self.supported_options = [:symbolize_names]
-          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
-            if MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
-              self.supported_options = [:freeze]
-            end
+          if supports_freeze?
+            self.supported_options = [:freeze]
           end
           supported_options.freeze
         end
+
+        private
+
+        def supports_freeze?
+          ::JSON.parse('["foo"]', freeze: true).first.frozen? &&
+            MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
+        end
       end
 
       module Patch

data/lib/bootsnap/compile_cache/yaml.rb

@@ -64,6 +64,19 @@ module Bootsnap
             @implementation::Patch.send(:remove_method, :unsafe_load_file)
           end
 
+          unless const_defined?(:NoTagsVisitor)
+            visitor = Class.new(Psych::Visitors::ToRuby) do
+              def visit(target)
+                if target.tag
+                  raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
+                end
+
+                super
+              end
+            end
+            const_set(:NoTagsVisitor, visitor)
+          end
+
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
@@ -102,10 +115,8 @@ module Bootsnap
           if params.include?([:key, :symbolize_names])
             supported_options << :symbolize_names
           end
-          if params.include?([:key, :freeze])
-            if factory.load(factory.dump("yaml"), freeze: true).frozen?
-              supported_options << :freeze
-            end
+          if params.include?([:key, :freeze]) && factory.load(factory.dump("yaml"), freeze: true).frozen?
+            supported_options << :freeze
           end
           supported_options.freeze
         end
@@ -118,19 +129,7 @@ module Bootsnap
           ast = ::YAML.parse(payload)
           return ast unless ast
 
-          strict_visitor.create.visit(ast)
-        end
-
-        def strict_visitor
-          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
-            def visit(target)
-              if target.tag
-                raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
-              end
-
-              super
-            end
-          end
+          NoTagsVisitor.create.visit(ast)
         end
       end
 

data/lib/bootsnap/load_path_cache/cache.rb

@@ -45,20 +45,19 @@ module Bootsnap
 
       # Try to resolve this feature to an absolute path without traversing the
       # loadpath.
-      def find(feature, try_extensions: true)
+      def find(feature)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
         feature = feature.to_s.freeze
 
         return feature if Bootsnap.absolute_path?(feature)
 
         if feature.start_with?("./", "../")
-          return try_extensions ? expand_path(feature) : File.expand_path(feature).freeze
+          return expand_path(feature)
         end
 
         @mutex.synchronize do
-          x = search_index(feature, try_extensions: try_extensions)
+          x = search_index(feature)
           return x if x
-          return unless try_extensions
 
           # Ruby has some built-in features that require lies about.
           # For example, 'enumerator' is built in. If you require it, ruby
@@ -115,7 +114,7 @@ module Bootsnap
       def reinitialize(path_obj = @path_obj)
         @mutex.synchronize do
           @path_obj = path_obj
-          ChangeObserver.register(self, @path_obj)
+          ChangeObserver.register(@path_obj, self)
           @index = {}
           @dirs = {}
           @generated_at = now
@@ -183,15 +182,11 @@ module Bootsnap
       end
 
       if DLEXT2
-        def search_index(feature, try_extensions: true)
-          if try_extensions
-            try_index(feature + DOT_RB) ||
-              try_index(feature + DLEXT) ||
-              try_index(feature + DLEXT2) ||
-              try_index(feature)
-          else
+        def search_index(feature)
+          try_index(feature + DOT_RB) ||
+            try_index(feature + DLEXT) ||
+            try_index(feature + DLEXT2) ||
             try_index(feature)
-          end
         end
 
         def maybe_append_extension(feature)
@@ -201,12 +196,8 @@ module Bootsnap
             feature
         end
       else
-        def search_index(feature, try_extensions: true)
-          if try_extensions
-            try_index(feature + DOT_RB) || try_index(feature + DLEXT) || try_index(feature)
-          else
-            try_index(feature)
-          end
+        def search_index(feature)
+          try_index(feature + DOT_RB) || try_index(feature + DLEXT) || try_index(feature)
         end
 
         def maybe_append_extension(feature)
@@ -216,7 +207,7 @@ module Bootsnap
 
       s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
       if s.respond_to?(:-@)
-        if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= "2.7"
+        if ((-s).equal?(s) && (-s.dup).equal?(s)) || RUBY_VERSION >= "2.7"
           def try_index(feature)
             if (path = @index[feature])
               -File.join(path, feature).freeze

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -56,11 +56,22 @@ module Bootsnap
         end
       end
 
-      def self.register(observer, arr)
+      def self.register(arr, observer)
         return if arr.frozen? # can't register observer, but no need to.
 
         arr.instance_variable_set(:@lpc_observer, observer)
-        arr.extend(ArrayMixin)
+        ArrayMixin.instance_methods.each do |method_name|
+          arr.singleton_class.send(:define_method, method_name, ArrayMixin.instance_method(method_name))
+        end
+      end
+
+      def self.unregister(arr)
+        return unless arr.instance_variable_defined?(:@lpc_observer) && arr.instance_variable_get(:@lpc_observer)
+
+        ArrayMixin.instance_methods.each do |method_name|
+          arr.singleton_class.send(:remove_method, method_name)
+        end
+        arr.instance_variable_set(:@lpc_observer, nil)
       end
     end
   end

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -6,6 +6,8 @@ module Kernel
   alias_method(:require_without_bootsnap, :require)
 
   def require(path)
+    return require_without_bootsnap(path) unless Bootsnap::LoadPathCache.enabled?
+
     string_path = Bootsnap.rb_get_path(path)
     return false if Bootsnap::LoadPathCache.loaded_features_index.key?(string_path)
 
@@ -32,13 +34,4 @@ module Kernel
       return ret
     end
   end
-
-  alias_method(:load_without_bootsnap, :load)
-  def load(path, wrap = false)
-    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(Bootsnap.rb_get_path(path), try_extensions: false))
-      load_without_bootsnap(resolved, wrap)
-    else
-      load_without_bootsnap(path, wrap)
-    end
-  end
 end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -40,7 +40,7 @@ module Bootsnap
 
             # /a/b/lib/my/foo.rb
             #          ^^^^^^^^^
-            short = feat[(lpe.length + 1)..-1]
+            short = feat[(lpe.length + 1)..]
             stripped = strip_extension_if_elidable(short)
             @lfi[short] = hash
             @lfi[stripped] = hash
@@ -76,7 +76,7 @@ module Bootsnap
       end
 
       def identify(short, cursor)
-        $LOADED_FEATURES[cursor..-1].detect do |feat|
+        $LOADED_FEATURES[cursor..].detect do |feat|
           offset = 0
           while (offset = feat.index(short, offset))
             if feat.index(".", offset + 1) && !feat.index("/", offset + 2)

data/lib/bootsnap/load_path_cache/path.rb

@@ -35,18 +35,18 @@ module Bootsnap
           return self
         end
 
-        if realpath != path
-          Path.new(realpath, real: true)
-        else
+        if realpath == path
           @real = true
           self
+        else
+          Path.new(realpath, real: true)
         end
       end
 
       # True if the path exists, but represents a non-directory object
       def non_directory?
         !File.stat(path).directory?
-      rescue Errno::ENOENT, Errno::ENOTDIR
+      rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
         false
       end
 
@@ -101,7 +101,7 @@ module Bootsnap
         ["", *dirs].each do |dir|
           curr = begin
             File.mtime("#{path}/#{dir}").to_i
-                 rescue Errno::ENOENT, Errno::ENOTDIR
+                 rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EINVAL
                    -1
           end
           max = curr if curr > max
@@ -121,16 +121,14 @@ module Bootsnap
       RUBY_SITEDIR = RbConfig::CONFIG["sitedir"]
 
       def stability
-        @stability ||= begin
-          if Gem.path.detect { |p| expanded_path.start_with?(p.to_s) }
-            STABLE
-          elsif Bootsnap.bundler? && expanded_path.start_with?(Bundler.bundle_path.to_s)
-            STABLE
-          elsif expanded_path.start_with?(RUBY_LIBDIR) && !expanded_path.start_with?(RUBY_SITEDIR)
-            STABLE
-          else
-            VOLATILE
-          end
+        @stability ||= if Gem.path.detect { |p| expanded_path.start_with?(p.to_s) }
+          STABLE
+        elsif Bootsnap.bundler? && expanded_path.start_with?(Bundler.bundle_path.to_s)
+          STABLE
+        elsif expanded_path.start_with?(RUBY_LIBDIR) && !expanded_path.start_with?(RUBY_SITEDIR)
+          STABLE
+        else
+          VOLATILE
         end
       end
     end

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -15,7 +15,11 @@ module Bootsnap
         ""
       end
 
+      @ignored_directories = %w(node_modules)
+
       class << self
+        attr_accessor :ignored_directories
+
         def call(path)
           path = File.expand_path(path.to_s).freeze
           return [[], []] unless File.directory?(path)
@@ -50,6 +54,8 @@ module Bootsnap
 
             absolute_path = "#{absolute_dir_path}/#{name}"
             if File.directory?(absolute_path)
+              next if ignored_directories.include?(name)
+
               if yield relative_path, absolute_path, true
                 walk(absolute_path, relative_path, &block)
               end

data/lib/bootsnap/load_path_cache/store.rb

@@ -49,11 +49,9 @@ module Bootsnap
         raise(NestedTransactionError) if @txn_mutex.owned?
 
         @txn_mutex.synchronize do
-          begin
-            yield
-          ensure
-            commit_transaction
-          end
+          yield
+        ensure
+          commit_transaction
         end
       end
 
@@ -121,11 +119,9 @@ module Bootsnap
           path = File.dirname(path)
         end
         stack.reverse_each do |dir|
-          begin
-            Dir.mkdir(dir)
-          rescue SystemCallError
-            raise unless File.directory?(dir)
-          end
+          Dir.mkdir(dir)
+        rescue SystemCallError
+          raise unless File.directory?(dir)
         end
       end
     end

data/lib/bootsnap/load_path_cache.rb

@@ -21,10 +21,14 @@ module Bootsnap
 
     CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
 
+    @enabled = false
+
     class << self
-      attr_reader(:load_path_cache, :loaded_features_index)
+      attr_reader(:load_path_cache, :loaded_features_index, :enabled)
+      alias_method :enabled?, :enabled
+      remove_method(:enabled)
 
-      def setup(cache_path:, development_mode:)
+      def setup(cache_path:, development_mode:, ignore_directories:)
         unless supported?
           warn("[bootsnap/setup] Load path caching is not supported on this implementation of Ruby") if $VERBOSE
           return
@@ -35,10 +39,20 @@ module Bootsnap
         @loaded_features_index = LoadedFeaturesIndex.new
 
         @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
+        PathScanner.ignored_directories = ignore_directories if ignore_directories
+        @enabled = true
         require_relative("load_path_cache/core_ext/kernel_require")
         require_relative("load_path_cache/core_ext/loaded_features")
       end
 
+      def unload!
+        @enabled = false
+        @loaded_features_index = nil
+        @realpath_cache = nil
+        @load_path_cache = nil
+        ChangeObserver.unregister($LOAD_PATH)
+      end
+
       def supported?
         RUBY_ENGINE == "ruby" &&
           RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.12.0"
+  VERSION = "1.14.0"
 end

data/lib/bootsnap.rb

@@ -39,48 +39,29 @@ module Bootsnap
       cache_dir:,
       development_mode: true,
       load_path_cache: true,
-      autoload_paths_cache: nil,
-      disable_trace: nil,
+      ignore_directories: nil,
       compile_cache_iseq: true,
       compile_cache_yaml: true,
       compile_cache_json: true
     )
-      unless autoload_paths_cache.nil?
-        warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
-          "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \
-          "upgrading is recommended."
-      end
-
-      unless disable_trace.nil?
-        warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \
-          "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
-      end
-
-      if compile_cache_iseq && !iseq_cache_supported?
-        warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
-          "to turn `compile_cache_iseq` off on Ruby 2.5"
-      end
-
       if load_path_cache
         Bootsnap::LoadPathCache.setup(
-          cache_path: cache_dir + "/bootsnap/load-path-cache",
+          cache_path: "#{cache_dir}/bootsnap/load-path-cache",
           development_mode: development_mode,
+          ignore_directories: ignore_directories,
         )
       end
 
       Bootsnap::CompileCache.setup(
-        cache_dir: cache_dir + "/bootsnap/compile-cache",
+        cache_dir: "#{cache_dir}/bootsnap/compile-cache",
         iseq: compile_cache_iseq,
         yaml: compile_cache_yaml,
         json: compile_cache_json,
       )
     end
 
-    def iseq_cache_supported?
-      return @iseq_cache_supported if defined? @iseq_cache_supported
-
-      ruby_version = Gem::Version.new(RUBY_VERSION)
-      @iseq_cache_supported = ruby_version < Gem::Version.new("2.5.0") || ruby_version >= Gem::Version.new("2.6.0")
+    def unload_cache!
+      LoadPathCache.unload!
     end
 
     def default_setup
@@ -109,13 +90,18 @@ module Bootsnap
           cache_dir = File.join(app_root, "tmp", "cache")
         end
 
+        ignore_directories = if ENV.key?("BOOTSNAP_IGNORE_DIRECTORIES")
+          ENV["BOOTSNAP_IGNORE_DIRECTORIES"].split(",")
+        end
+
         setup(
           cache_dir: cache_dir,
           development_mode: development_mode,
           load_path_cache: !ENV["DISABLE_BOOTSNAP_LOAD_PATH_CACHE"],
-          compile_cache_iseq: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] && iseq_cache_supported?,
+          compile_cache_iseq: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
           compile_cache_yaml: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
           compile_cache_json: !ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"],
+          ignore_directories: ignore_directories,
         )
 
         if ENV["BOOTSNAP_LOG"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.12.0
+  version: 1.14.0
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-05-30 00:00:00.000000000 Z
+date: 2022-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -76,14 +76,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: Boot large ruby/rails apps faster