bootsnap 1.4.0 → 1.7.7

data/lib/bootsnap/load_path_cache/cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative('../explicit_require')
 
 module Bootsnap
@@ -8,8 +10,8 @@ module Bootsnap
       def initialize(store, path_obj, development_mode: false)
         @development_mode = development_mode
         @store = store
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
-        @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f }
+        @mutex = Mutex.new
+        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
         @has_relative_paths = nil
         reinitialize
       end
@@ -44,9 +46,9 @@ module Bootsnap
       # loadpath.
       def find(feature)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
-        feature = feature.to_s
+        feature = feature.to_s.freeze
         return feature if absolute_path?(feature)
-        return File.expand_path(feature) if feature.start_with?('./')
+        return expand_path(feature) if feature.start_with?('./')
         @mutex.synchronize do
           x = search_index(feature)
           return x if x
@@ -65,7 +67,7 @@ module Bootsnap
           # native dynamic extension, e.g. .bundle or .so), we know it was a
           # failure and there's nothing more we can do to find the file.
           # no extension, .rb, (.bundle or .so)
-          when '', *CACHED_EXTENSIONS # rubocop:disable Performance/CaseWhenSplat
+          when '', *CACHED_EXTENSIONS
             nil
           # Ruby allows specifying native extensions as '.so' even when DLEXT
           # is '.bundle'. This is where we handle that case.
@@ -142,7 +144,7 @@ module Bootsnap
             expanded_path = p.expanded_path
             entries, dirs = p.entries_and_dirs(@store)
             # push -> low precedence -> set only if unset
-            dirs.each    { |dir| @dirs[dir]  ||= path }
+            dirs.each    { |dir| @dirs[dir] ||= path }
             entries.each { |rel| @index[rel] ||= expanded_path }
           end
         end
@@ -162,6 +164,10 @@ module Bootsnap
         end
       end
 
+      def expand_path(feature)
+        maybe_append_extension(File.expand_path(feature))
+      end
+
       def stale?
         @development_mode && @generated_at + AGE_THRESHOLD < now
       end
@@ -174,17 +180,46 @@ module Bootsnap
         def search_index(f)
           try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
         end
+
+        def maybe_append_extension(f)
+          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f
+        end
       else
         def search_index(f)
           try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
         end
+
+        def maybe_append_extension(f)
+          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || f
+        end
       end
 
-      def try_index(f)
-        if (p = @index[f])
-          p + '/' + f
+      s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
+      if s.respond_to?(:-@)
+        if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= '2.7'
+          def try_index(f)
+            if (p = @index[f])
+              -(File.join(p, f).freeze)
+            end
+          end
+        else
+          def try_index(f)
+            if (p = @index[f])
+              -File.join(p, f).untaint
+            end
+          end
+        end
+      else
+        def try_index(f)
+          if (p = @index[f])
+            File.join(p, f)
+          end
         end
       end
+
+      def try_ext(f)
+        f if File.exist?(f)
+      end
     end
   end
 end

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module ChangeObserver
@@ -14,18 +15,20 @@ module Bootsnap
           @lpc_observer.push_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :append, :push
 
         def unshift(*entries)
           @lpc_observer.unshift_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :prepend, :unshift
 
         def concat(entries)
           @lpc_observer.push_paths(self, *entries.map(&:to_s))
           super
         end
 
-        # uniq! keeps the first occurance of each path, otherwise preserving
+        # uniq! keeps the first occurrence of each path, otherwise preserving
         # order, preserving the effective load path
         def uniq!(*args)
           ret = super
@@ -53,6 +56,7 @@ module Bootsnap
       end
 
       def self.register(observer, arr)
+        return if arr.frozen? # can't register observer, but no need to.
         arr.instance_variable_set(:@lpc_observer, observer)
         arr.extend(ArrayMixin)
       end

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module CoreExt
       def self.make_load_error(path)
-        err = LoadError.new("cannot load such file -- #{path}")
+        err = LoadError.new(+"cannot load such file -- #{path}")
+        err.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
         err.define_singleton_method(:path) { path }
         err
       end
@@ -30,16 +32,24 @@ module Kernel
     end
 
     raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
+  rescue LoadError => e
+    e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
+    raise(e)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    require_with_bootsnap_lfi(path)
+    fallback = true
+  ensure
+    if fallback
+      require_with_bootsnap_lfi(path)
+    end
   end
 
   alias_method(:require_relative_without_bootsnap, :require_relative)
   def require_relative(path)
+    location = caller_locations(1..1).first
     realpath = Bootsnap::LoadPathCache.realpath_cache.call(
-      caller_locations(1..1).first.absolute_path, path
+      location.absolute_path || location.path, path
     )
     require(realpath)
   end
@@ -51,15 +61,22 @@ module Kernel
     end
 
     # load also allows relative paths from pwd even when not in $:
-    if File.exist?(relative = File.expand_path(path))
+    if File.exist?(relative = File.expand_path(path).freeze)
       return load_without_bootsnap(relative, wrap)
     end
 
     raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
+  rescue LoadError => e
+    e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
+    raise(e)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_bootsnap(path, wrap)
+    fallback = true
+  ensure
+    if fallback
+      load_without_bootsnap(path, wrap)
+    end
   end
 end
 
@@ -74,9 +91,16 @@ class Module
     # added to $LOADED_FEATURES and won't be able to hook that modification
     # since it's done in C-land.
     autoload_without_bootsnap(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
+  rescue LoadError => e
+    e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
+    raise(e)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    autoload_without_bootsnap(const, path)
+    fallback = true
+  ensure
+    if fallback
+      autoload_without_bootsnap(const, path)
+    end
   end
 end

data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb

@@ -1,7 +1,18 @@
+# frozen_string_literal: true
 class << $LOADED_FEATURES
   alias_method(:delete_without_bootsnap, :delete)
   def delete(key)
     Bootsnap::LoadPathCache.loaded_features_index.purge(key)
     delete_without_bootsnap(key)
   end
+
+  alias_method(:reject_without_bootsnap!, :reject!)
+  def reject!(&block)
+    backup = dup
+
+    # FIXME: if no block is passed we'd need to return a decorated iterator
+    reject_without_bootsnap!(&block)
+
+    Bootsnap::LoadPathCache.loaded_features_index.purge_multi(backup - self)
+  end
 end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bootsnap
   module LoadPathCache
     # LoadedFeaturesIndex partially mirrors an internal structure in ruby that
@@ -24,7 +26,7 @@ module Bootsnap
     class LoadedFeaturesIndex
       def initialize
         @lfi = {}
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+        @mutex = Mutex.new
 
         # In theory the user could mutate $LOADED_FEATURES and invalidate our
         # cache. If this ever comes up in practice — or if you, the
@@ -38,7 +40,7 @@ module Bootsnap
             # /a/b/lib/my/foo.rb
             #          ^^^^^^^^^
             short = feat[(lpe.length + 1)..-1]
-            stripped = strip_extension(short)
+            stripped = strip_extension_if_elidable(short)
             @lfi[short] = hash
             @lfi[stripped] = hash
           end
@@ -55,6 +57,13 @@ module Bootsnap
         end
       end
 
+      def purge_multi(features)
+        rejected_hashes = features.each_with_object({}) { |f, h| h[f.hash] = true }
+        @mutex.synchronize do
+          @lfi.reject! { |_, hash| rejected_hashes.key?(hash) }
+        end
+      end
+
       def key?(feature)
         @mutex.synchronize { @lfi.key?(feature) }
       end
@@ -85,13 +94,14 @@ module Bootsnap
 
         hash = long.hash
 
-        # do we have 'bundler' or 'bundler.rb'?
-        altname = if File.extname(short) != ''
-          # strip the path from 'bundler.rb' -> 'bundler'
-          strip_extension(short)
-        elsif long && (ext = File.extname(long))
-          # get the extension from the expanded path if given
-          # 'bundler' + '.rb'
+        # Do we have a filename with an elidable extension, e.g.,
+        # 'bundler.rb', or 'libgit2.so'?
+        altname = if extension_elidable?(short)
+          # Strip the extension off, e.g. 'bundler.rb' -> 'bundler'.
+          strip_extension_if_elidable(short)
+        elsif long && (ext = File.extname(long.freeze))
+          # We already know the extension of the actual file this
+          # resolves to, so put that back on.
           short + ext
         end
 
@@ -108,8 +118,30 @@ module Bootsnap
       STRIP_EXTENSION = /\.[^.]*?$/
       private_constant(:STRIP_EXTENSION)
 
-      def strip_extension(f)
-        f.sub(STRIP_EXTENSION, '')
+      # Might Ruby automatically search for this extension if
+      # someone tries to 'require' the file without it? E.g. Ruby
+      # will implicitly try 'x.rb' if you ask for 'x'.
+      #
+      # This is complex and platform-dependent, and the Ruby docs are a little
+      # handwavy about what will be tried when and in what order.
+      # So optimistically pretend that all known elidable extensions
+      # will be tried on all platforms, and that people are unlikely
+      # to name files in a way that assumes otherwise.
+      # (E.g. It's unlikely that someone will know that their code
+      # will _never_ run on MacOS, and therefore think they can get away
+      # with calling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.)
+      #
+      # See <https://ruby-doc.org/core-2.6.4/Kernel.html#method-i-require>.
+      def extension_elidable?(f)
+        f.to_s.end_with?('.rb', '.so', '.o', '.dll', '.dylib')
+      end
+
+      def strip_extension_if_elidable(f)
+        if extension_elidable?(f)
+          f.sub(STRIP_EXTENSION, '')
+        else
+          f
+        end
       end
     end
   end

data/lib/bootsnap/load_path_cache/path.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative('path_scanner')
 
 module Bootsnap
@@ -20,7 +21,7 @@ module Bootsnap
       attr_reader(:path)
 
       def initialize(path)
-        @path = path.to_s
+        @path = path.to_s.freeze
       end
 
       # True if the path exists, but represents a non-directory object
@@ -59,7 +60,7 @@ module Bootsnap
       end
 
       def expanded_path
-        File.expand_path(path)
+        File.expand_path(path).freeze
       end
 
       private

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -1,9 +1,10 @@
+# frozen_string_literal: true
+
 require_relative('../explicit_require')
 
 module Bootsnap
   module LoadPathCache
     module PathScanner
-      ALL_FILES = "/{,**/*/**/}*"
       REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS
       NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
       ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/
@@ -11,37 +12,62 @@ module Bootsnap
       BUNDLE_PATH = if Bootsnap.bundler?
         (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
       else
-        ''.freeze
+        ''
       end
 
-      def self.call(path)
-        path = path.to_s
-
-        relative_slice = (path.size + 1)..-1
-        # If the bundle path is a descendent of this path, we do additional
-        # checks to prevent recursing into the bundle path as we recurse
-        # through this path. We don't want to scan the bundle path because
-        # anything useful in it will be present on other load path items.
-        #
-        # This can happen if, for example, the user adds '.' to the load path,
-        # and the bundle path is '.bundle'.
-        contains_bundle_path = BUNDLE_PATH.start_with?(path)
-
-        dirs = []
-        requirables = []
-
-        Dir.glob(path + ALL_FILES).each do |absolute_path|
-          next if contains_bundle_path && absolute_path.start_with?(BUNDLE_PATH)
-          relative_path = absolute_path.slice(relative_slice)
-
-          if File.directory?(absolute_path)
-            dirs << relative_path
-          elsif REQUIRABLE_EXTENSIONS.include?(File.extname(relative_path))
-            requirables << relative_path
+      class << self
+        def call(path)
+          path = File.expand_path(path.to_s).freeze
+          return [[], []] unless File.directory?(path)
+
+          # If the bundle path is a descendent of this path, we do additional
+          # checks to prevent recursing into the bundle path as we recurse
+          # through this path. We don't want to scan the bundle path because
+          # anything useful in it will be present on other load path items.
+          #
+          # This can happen if, for example, the user adds '.' to the load path,
+          # and the bundle path is '.bundle'.
+          contains_bundle_path = BUNDLE_PATH.start_with?(path)
+
+          dirs = []
+          requirables = []
+          walk(path, nil) do |relative_path, absolute_path, is_directory|
+            if is_directory
+              dirs << os_path(relative_path)
+              !contains_bundle_path || !absolute_path.start_with?(BUNDLE_PATH)
+            elsif relative_path.end_with?(*REQUIRABLE_EXTENSIONS)
+              requirables << os_path(relative_path)
+            end
+          end
+          [requirables, dirs]
+        end
+
+        def walk(absolute_dir_path, relative_dir_path, &block)
+          Dir.foreach(absolute_dir_path) do |name|
+            next if name.start_with?('.')
+            relative_path = relative_dir_path ? File.join(relative_dir_path, name) : name
+
+            absolute_path = "#{absolute_dir_path}/#{name}"
+            if File.directory?(absolute_path)
+              if yield relative_path, absolute_path, true
+                walk(absolute_path, relative_path, &block)
+              end
+            else
+              yield relative_path, absolute_path, false
+            end
           end
         end
 
-        [requirables, dirs]
+        if RUBY_VERSION >= '3.1'
+          def os_path(path)
+            path.freeze
+          end
+        else
+          def os_path(path)
+            path.force_encoding(Encoding::US_ASCII) if path.ascii_only?
+            path.freeze
+          end
+        end
       end
     end
   end

data/lib/bootsnap/load_path_cache/realpath_cache.rb

@@ -15,15 +15,15 @@ module Bootsnap
 
       def realpath(caller_location, path)
         base = File.dirname(caller_location)
-        file = find_file(File.expand_path(path, base))
-        dir = File.dirname(file)
-        File.join(dir, File.basename(file))
+        abspath = File.expand_path(path, base).freeze
+        find_file(abspath)
       end
 
       def find_file(name)
-        ['', *CACHED_EXTENSIONS].each do |ext|
+        return File.realpath(name).freeze if File.exist?(name)
+        CACHED_EXTENSIONS.each do |ext|
           filename = "#{name}#{ext}"
-          return File.realpath(filename) if File.exist?(filename)
+          return File.realpath(filename).freeze if File.exist?(filename)
         end
         name
       end

data/lib/bootsnap/load_path_cache/store.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative('../explicit_require')
 
 Bootsnap::ExplicitRequire.with_gems('msgpack') { require('msgpack') }
@@ -11,7 +12,7 @@ module Bootsnap
 
       def initialize(store_path)
         @store_path = store_path
-        @in_txn = false
+        @txn_mutex = Mutex.new
         @dirty = false
         load_data
       end
@@ -21,7 +22,7 @@ module Bootsnap
       end
 
       def fetch(key)
-        raise(SetOutsideTransactionNotAllowed) unless @in_txn
+        raise(SetOutsideTransactionNotAllowed) unless @txn_mutex.owned?
         v = get(key)
         unless v
           @dirty = true
@@ -32,7 +33,7 @@ module Bootsnap
       end
 
       def set(key, value)
-        raise(SetOutsideTransactionNotAllowed) unless @in_txn
+        raise(SetOutsideTransactionNotAllowed) unless @txn_mutex.owned?
         if value != @data[key]
           @dirty = true
           @data[key] = value
@@ -40,12 +41,14 @@ module Bootsnap
       end
 
       def transaction
-        raise(NestedTransactionError) if @in_txn
-        @in_txn = true
-        yield
-      ensure
-        commit_transaction
-        @in_txn = false
+        raise(NestedTransactionError) if @txn_mutex.owned?
+        @txn_mutex.synchronize do
+          begin
+            yield
+          ensure
+            commit_transaction
+          end
+        end
       end
 
       private
@@ -59,10 +62,18 @@ module Bootsnap
 
       def load_data
         @data = begin
-          MessagePack.load(File.binread(@store_path))
-                # handle malformed data due to upgrade incompatability
-                rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
-                  {}
+          File.open(@store_path, encoding: Encoding::BINARY) do |io|
+            MessagePack.load(io)
+          end
+        # handle malformed data due to upgrade incompatibility
+        rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
+          {}
+        rescue ArgumentError => error
+          if error.message =~ /negative array size/
+            {}
+          else
+            raise
+          end
         end
       end
 
@@ -74,10 +85,13 @@ module Bootsnap
         exclusive_write = File::Constants::CREAT | File::Constants::EXCL | File::Constants::WRONLY
         # `encoding:` looks redundant wrt `binwrite`, but necessary on windows
         # because binary is part of mode.
-        File.binwrite(tmp, MessagePack.dump(@data), mode: exclusive_write, encoding: Encoding::BINARY)
+        File.open(tmp, mode: exclusive_write, encoding: Encoding::BINARY) do |io|
+          MessagePack.dump(@data, io, freeze: true)
+        end
         FileUtils.mv(tmp, @store_path)
       rescue Errno::EEXIST
         retry
+      rescue SystemCallError
       end
     end
   end

data/lib/bootsnap/load_path_cache.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Bootsnap
   module LoadPathCache
     ReturnFalse = Class.new(StandardError)
@@ -7,6 +9,11 @@ module Bootsnap
     DOT_SO = '.so'
     SLASH  = '/'
 
+    # If a NameError happens several levels deep, don't re-handle it
+    # all the way up the chain: mark it once and bubble it up without
+    # more retries.
+    ERROR_TAG_IVAR = :@__bootsnap_rescued
+
     DL_EXTENSIONS = ::RbConfig::CONFIG
       .values_at('DLEXT', 'DLEXT2')
       .reject { |ext| !ext || ext.empty? }
@@ -21,10 +28,9 @@ module Bootsnap
     CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
 
     class << self
-      attr_reader(:load_path_cache, :autoload_paths_cache,
-        :loaded_features_index, :realpath_cache)
+      attr_reader(:load_path_cache, :loaded_features_index, :realpath_cache)
 
-      def setup(cache_path:, development_mode:, active_support: true)
+      def setup(cache_path:, development_mode:)
         unless supported?
           warn("[bootsnap/setup] Load path caching is not supported on this implementation of Ruby") if $VERBOSE
           return
@@ -38,23 +44,11 @@ module Bootsnap
         @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
         require_relative('load_path_cache/core_ext/kernel_require')
         require_relative('load_path_cache/core_ext/loaded_features')
-
-        if active_support
-          # this should happen after setting up the initial cache because it
-          # loads a lot of code. It's better to do after +require+ is optimized.
-          require('active_support/dependencies')
-          @autoload_paths_cache = Cache.new(
-            store,
-            ::ActiveSupport::Dependencies.autoload_paths,
-            development_mode: development_mode
-          )
-          require_relative('load_path_cache/core_ext/active_support')
-        end
       end
 
       def supported?
         RUBY_ENGINE == 'ruby' &&
-        RUBY_PLATFORM =~ /darwin|linux|bsd/
+        RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/
       end
     end
   end

data/lib/bootsnap/setup.rb

@@ -1,35 +1,4 @@
+# frozen_string_literal: true
 require_relative('../bootsnap')
 
-env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
-development_mode = ['', nil, 'development'].include?(env)
-
-cache_dir = ENV['BOOTSNAP_CACHE_DIR']
-unless cache_dir
-  config_dir_frame = caller.detect do |line|
-    line.include?('/config/')
-  end
-
-  unless config_dir_frame
-    $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
-    $stderr.puts("[bootsnap/setup]   1. require bootsnap/setup from your application's config directory; or")
-    $stderr.puts("[bootsnap/setup]   2. Define the environment variable BOOTSNAP_CACHE_DIR")
-
-    raise("couldn't infer bootsnap cache directory")
-  end
-
-  path = config_dir_frame.split(/:\d+:/).first
-  path = File.dirname(path) until File.basename(path) == 'config'
-  app_root = File.dirname(path)
-
-  cache_dir = File.join(app_root, 'tmp', 'cache')
-end
-
-Bootsnap.setup(
-  cache_dir:            cache_dir,
-  development_mode:     development_mode,
-  load_path_cache:      true,
-  autoload_paths_cache: true, # assume rails. open to PRs to impl. detection
-  disable_trace:        false,
-  compile_cache_iseq:   true,
-  compile_cache_yaml:   true,
-)
+Bootsnap.default_setup

data/lib/bootsnap/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
-  VERSION = "1.4.0"
+  VERSION = "1.7.7"
 end