bootsnap 1.1.8 → 1.4.0

data/lib/bootsnap/load_path_cache/core_ext/active_support.rb

@@ -10,6 +10,14 @@ module Bootsnap
           Thread.current[:without_bootsnap_cache] = prev
         end
 
+        def self.allow_bootsnap_retry(allowed)
+          prev = Thread.current[:without_bootsnap_retry] || false
+          Thread.current[:without_bootsnap_retry] = !allowed
+          yield
+        ensure
+          Thread.current[:without_bootsnap_retry] = prev
+        end
+
         module ClassMethods
           def autoload_paths=(o)
             super
@@ -22,17 +30,25 @@ module Bootsnap
               Bootsnap::LoadPathCache.autoload_paths_cache.find(path)
             rescue Bootsnap::LoadPathCache::ReturnFalse
               nil # doesn't really apply here
+            rescue Bootsnap::LoadPathCache::FallbackScan
+              nil # doesn't really apply here
             end
           end
 
           def autoloadable_module?(path_suffix)
-            Bootsnap::LoadPathCache.autoload_paths_cache.has_dir?(path_suffix)
+            Bootsnap::LoadPathCache.autoload_paths_cache.load_dir(path_suffix)
           end
 
           def remove_constant(const)
             CoreExt::ActiveSupport.without_bootsnap_cache { super }
           end
 
+          def require_or_load(*)
+            CoreExt::ActiveSupport.allow_bootsnap_retry(true) do
+              super
+            end
+          end
+
           # If we can't find a constant using the patched implementation of
           # search_for_file, try again with the default implementation.
           #
@@ -40,8 +56,15 @@ module Bootsnap
           # behaviour.  The gymnastics here are a bit awkward, but it prevents
           # 200+ lines of monkeypatches.
           def load_missing_constant(from_mod, const_name)
-            super
+            CoreExt::ActiveSupport.allow_bootsnap_retry(false) do
+              super
+            end
           rescue NameError => e
+            # This function can end up called recursively, we only want to
+            # retry at the top-level.
+            raise if Thread.current[:without_bootsnap_retry]
+            # If we already had cache disabled, there's no use retrying
+            raise if Thread.current[:without_bootsnap_cache]
             # NoMethodError is a NameError, but we only want to handle actual
             # NameError instances.
             raise unless e.class == NameError
@@ -58,6 +81,8 @@ module Bootsnap
           def depend_on(*)
             super
           rescue LoadError
+            # If we already had cache disabled, there's no use retrying
+            raise if Thread.current[:without_bootsnap_cache]
             CoreExt::ActiveSupport.without_bootsnap_cache { super }
           end
         end
@@ -69,7 +94,7 @@ end
 module ActiveSupport
   module Dependencies
     class << self
-      prepend Bootsnap::LoadPathCache::CoreExt::ActiveSupport::ClassMethods
+      prepend(Bootsnap::LoadPathCache::CoreExt::ActiveSupport::ClassMethods)
     end
   end
 end

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -11,78 +11,72 @@ module Bootsnap
 end
 
 module Kernel
-  alias_method :require_without_cache, :require
-  def require(path)
-    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      require_without_cache(resolved)
-    else
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
-    end
-  rescue Bootsnap::LoadPathCache::ReturnFalse
-    return false
-  rescue Bootsnap::LoadPathCache::FallbackScan
-    require_without_cache(path)
-  end
+  module_function # rubocop:disable Style/ModuleFunction
 
-  alias_method :load_without_cache, :load
-  def load(path, wrap = false)
-    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      load_without_cache(resolved, wrap)
-    else
-      # load also allows relative paths from pwd even when not in $:
-      relative = File.expand_path(path)
-      if File.exist?(File.expand_path(path))
-        return load_without_cache(relative, wrap)
-      end
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+  alias_method(:require_without_bootsnap, :require)
+
+  # Note that require registers to $LOADED_FEATURES while load does not.
+  def require_with_bootsnap_lfi(path, resolved = nil)
+    Bootsnap::LoadPathCache.loaded_features_index.register(path, resolved) do
+      require_without_bootsnap(resolved || path)
     end
-  rescue Bootsnap::LoadPathCache::ReturnFalse
-    return false
-  rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_cache(path, wrap)
   end
-end
 
-class << Kernel
-  alias_method :require_without_cache, :require
   def require(path)
-    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      require_without_cache(resolved)
-    else
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+    return false if Bootsnap::LoadPathCache.loaded_features_index.key?(path)
+
+    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path))
+      return require_with_bootsnap_lfi(path, resolved)
     end
+
+    raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
   rescue Bootsnap::LoadPathCache::ReturnFalse
-    return false
+    false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    require_without_cache(path)
+    require_with_bootsnap_lfi(path)
+  end
+
+  alias_method(:require_relative_without_bootsnap, :require_relative)
+  def require_relative(path)
+    realpath = Bootsnap::LoadPathCache.realpath_cache.call(
+      caller_locations(1..1).first.absolute_path, path
+    )
+    require(realpath)
   end
 
-  alias_method :load_without_cache, :load
+  alias_method(:load_without_bootsnap, :load)
   def load(path, wrap = false)
-    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      load_without_cache(resolved, wrap)
-    else
-      # load also allows relative paths from pwd even when not in $:
-      relative = File.expand_path(path)
-      if File.exist?(relative)
-        return load_without_cache(relative, wrap)
-      end
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path))
+      return load_without_bootsnap(resolved, wrap)
+    end
+
+    # load also allows relative paths from pwd even when not in $:
+    if File.exist?(relative = File.expand_path(path))
+      return load_without_bootsnap(relative, wrap)
     end
+
+    raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
   rescue Bootsnap::LoadPathCache::ReturnFalse
-    return false
+    false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_cache(path, wrap)
+    load_without_bootsnap(path, wrap)
   end
 end
 
 class Module
-  alias_method :autoload_without_cache, :autoload
+  alias_method(:autoload_without_bootsnap, :autoload)
   def autoload(const, path)
-    autoload_without_cache(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
+    # NOTE: This may defeat LoadedFeaturesIndex, but it's not immediately
+    # obvious how to make it work. This feels like a pretty niche case, unclear
+    # if it will ever burn anyone.
+    #
+    # The challenge is that we don't control the point at which the entry gets
+    # added to $LOADED_FEATURES and won't be able to hook that modification
+    # since it's done in C-land.
+    autoload_without_bootsnap(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
-    return false
+    false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    autoload_without_cache(const, path)
+    autoload_without_bootsnap(const, path)
   end
 end

data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb

@@ -0,0 +1,7 @@
+class << $LOADED_FEATURES
+  alias_method(:delete_without_bootsnap, :delete)
+  def delete(key)
+    Bootsnap::LoadPathCache.loaded_features_index.purge(key)
+    delete_without_bootsnap(key)
+  end
+end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -0,0 +1,116 @@
+module Bootsnap
+  module LoadPathCache
+    # LoadedFeaturesIndex partially mirrors an internal structure in ruby that
+    # we can't easily obtain an interface to.
+    #
+    # This works around an issue where, without bootsnap, *ruby* knows that it
+    # has already required a file by its short name (e.g. require 'bundler') if
+    # a new instance of bundler is added to the $LOAD_PATH which resolves to a
+    # different absolute path. This class makes bootsnap smart enough to
+    # realize that it has already loaded 'bundler', and not just
+    # '/path/to/bundler'.
+    #
+    # If you disable LoadedFeaturesIndex, you can see the problem this solves by:
+    #
+    # 1. `require 'a'`
+    # 2. Prepend a new $LOAD_PATH element containing an `a.rb`
+    # 3. `require 'a'`
+    #
+    # Ruby returns false from step 3.
+    # With bootsnap but with no LoadedFeaturesIndex, this loads two different
+    #   `a.rb`s.
+    # With bootsnap and with LoadedFeaturesIndex, this skips the second load,
+    #   returning false like ruby.
+    class LoadedFeaturesIndex
+      def initialize
+        @lfi = {}
+        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+
+        # In theory the user could mutate $LOADED_FEATURES and invalidate our
+        # cache. If this ever comes up in practice — or if you, the
+        # enterprising reader, feels inclined to solve this problem — we could
+        # parallel the work done with ChangeObserver on $LOAD_PATH to mirror
+        # updates to our @lfi.
+        $LOADED_FEATURES.each do |feat|
+          hash = feat.hash
+          $LOAD_PATH.each do |lpe|
+            next unless feat.start_with?(lpe)
+            # /a/b/lib/my/foo.rb
+            #          ^^^^^^^^^
+            short = feat[(lpe.length + 1)..-1]
+            stripped = strip_extension(short)
+            @lfi[short] = hash
+            @lfi[stripped] = hash
+          end
+        end
+      end
+
+      # We've optimized for initialize and register to be fast, and purge to be tolerable.
+      # If access patterns make this not-okay, we can lazy-invert the LFI on
+      # first purge and work from there.
+      def purge(feature)
+        @mutex.synchronize do
+          feat_hash = feature.hash
+          @lfi.reject! { |_, hash| hash == feat_hash }
+        end
+      end
+
+      def key?(feature)
+        @mutex.synchronize { @lfi.key?(feature) }
+      end
+
+      # There is a relatively uncommon case where we could miss adding an
+      # entry:
+      #
+      # If the user asked for e.g. `require 'bundler'`, and we went through the
+      # `FallbackScan` pathway in `kernel_require.rb` and therefore did not
+      # pass `long` (the full expanded absolute path), then we did are not able
+      # to confidently add the `bundler.rb` form to @lfi.
+      #
+      # We could either:
+      #
+      # 1. Just add `bundler.rb`, `bundler.so`, and so on, which is close but
+      #    not quite right; or
+      # 2. Inspect $LOADED_FEATURES upon return from yield to find the matching
+      #    entry.
+      def register(short, long = nil)
+        if long.nil?
+          pat = %r{/#{Regexp.escape(short)}(\.[^/]+)?$}
+          len = $LOADED_FEATURES.size
+          ret = yield
+          long = $LOADED_FEATURES[len..-1].detect { |feat| feat =~ pat }
+        else
+          ret = yield
+        end
+
+        hash = long.hash
+
+        # do we have 'bundler' or 'bundler.rb'?
+        altname = if File.extname(short) != ''
+          # strip the path from 'bundler.rb' -> 'bundler'
+          strip_extension(short)
+        elsif long && (ext = File.extname(long))
+          # get the extension from the expanded path if given
+          # 'bundler' + '.rb'
+          short + ext
+        end
+
+        @mutex.synchronize do
+          @lfi[short] = hash
+          (@lfi[altname] = hash) if altname
+        end
+
+        ret
+      end
+
+      private
+
+      STRIP_EXTENSION = /\.[^.]*?$/
+      private_constant(:STRIP_EXTENSION)
+
+      def strip_extension(f)
+        f.sub(STRIP_EXTENSION, '')
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/path.rb

@@ -1,4 +1,4 @@
-require_relative 'path_scanner'
+require_relative('path_scanner')
 
 module Bootsnap
   module LoadPathCache
@@ -17,7 +17,7 @@ module Bootsnap
         stability == VOLATILE
       end
 
-      attr_reader :path
+      attr_reader(:path)
 
       def initialize(path)
         @path = path.to_s
@@ -26,7 +26,7 @@ module Bootsnap
       # True if the path exists, but represents a non-directory object
       def non_directory?
         !File.stat(path).directory?
-      rescue Errno::ENOENT
+      rescue Errno::ENOENT, Errno::ENOTDIR
         false
       end
 
@@ -76,8 +76,8 @@ module Bootsnap
         ["", *dirs].each do |dir|
           curr = begin
             File.mtime("#{path}/#{dir}").to_i
-          rescue Errno::ENOENT
-            -1
+                 rescue Errno::ENOENT, Errno::ENOTDIR
+                   -1
           end
           max = curr if curr > max
         end

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -1,22 +1,18 @@
-require_relative '../explicit_require'
+require_relative('../explicit_require')
 
 module Bootsnap
   module LoadPathCache
     module PathScanner
-      # Glob pattern to find requirable files and subdirectories in given path.
-      # It expands to:
-      #
-      #   * `/*{.rb,.so,/}` - It matches requirable files, directories and
-      #     symlinks to directories at given path.
-      #   * `/*/**/*{.rb,.so,/}` - It matches requirable files and
-      #     subdirectories in any (even symlinked) directory at given path at
-      #     any directory tree depth.
-      #
-      REQUIRABLES_AND_DIRS = "/{,*/**/}*{#{DOT_RB},#{DL_EXTENSIONS.join(',')},/}"
+      ALL_FILES = "/{,**/*/**/}*"
+      REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS
       NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
       ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/
-      BUNDLE_PATH = Bootsnap.bundler? ?
-        (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze : ''.freeze
+
+      BUNDLE_PATH = if Bootsnap.bundler?
+        (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
+      else
+        ''.freeze
+      end
 
       def self.call(path)
         path = path.to_s
@@ -34,13 +30,13 @@ module Bootsnap
         dirs = []
         requirables = []
 
-        Dir.glob(path + REQUIRABLES_AND_DIRS).each do |absolute_path|
+        Dir.glob(path + ALL_FILES).each do |absolute_path|
           next if contains_bundle_path && absolute_path.start_with?(BUNDLE_PATH)
-          relative_path = absolute_path.slice!(relative_slice)
+          relative_path = absolute_path.slice(relative_slice)
 
-          if relative_path.end_with?('/')
-            dirs << relative_path[0..-2]
-          else
+          if File.directory?(absolute_path)
+            dirs << relative_path
+          elsif REQUIRABLE_EXTENSIONS.include?(File.extname(relative_path))
             requirables << relative_path
           end
         end

data/lib/bootsnap/load_path_cache/realpath_cache.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Bootsnap
+  module LoadPathCache
+    class RealpathCache
+      def initialize
+        @cache = Hash.new { |h, k| h[k] = realpath(*k) }
+      end
+
+      def call(*key)
+        @cache[key]
+      end
+
+      private
+
+      def realpath(caller_location, path)
+        base = File.dirname(caller_location)
+        file = find_file(File.expand_path(path, base))
+        dir = File.dirname(file)
+        File.join(dir, File.basename(file))
+      end
+
+      def find_file(name)
+        ['', *CACHED_EXTENSIONS].each do |ext|
+          filename = "#{name}#{ext}"
+          return File.realpath(filename) if File.exist?(filename)
+        end
+        name
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/store.rb

@@ -1,6 +1,6 @@
-require_relative '../explicit_require'
+require_relative('../explicit_require')
 
-Bootsnap::ExplicitRequire.with_gems('msgpack') { require 'msgpack' }
+Bootsnap::ExplicitRequire.with_gems('msgpack') { require('msgpack') }
 Bootsnap::ExplicitRequire.from_rubylibdir('fileutils')
 
 module Bootsnap
@@ -21,7 +21,7 @@ module Bootsnap
       end
 
       def fetch(key)
-        raise SetOutsideTransactionNotAllowed unless @in_txn
+        raise(SetOutsideTransactionNotAllowed) unless @in_txn
         v = get(key)
         unless v
           @dirty = true
@@ -32,7 +32,7 @@ module Bootsnap
       end
 
       def set(key, value)
-        raise SetOutsideTransactionNotAllowed unless @in_txn
+        raise(SetOutsideTransactionNotAllowed) unless @in_txn
         if value != @data[key]
           @dirty = true
           @data[key] = value
@@ -40,7 +40,7 @@ module Bootsnap
       end
 
       def transaction
-        raise NestedTransactionError if @in_txn
+        raise(NestedTransactionError) if @in_txn
         @in_txn = true
         yield
       ensure
@@ -60,9 +60,9 @@ module Bootsnap
       def load_data
         @data = begin
           MessagePack.load(File.binread(@store_path))
-        # handle malformed data due to upgrade incompatability
-        rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
-          {}
+                # handle malformed data due to upgrade incompatability
+                rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
+                  {}
         end
       end