bootsnap-pr-184 1.3.1.pr.pre.184.1

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -0,0 +1,53 @@
+module Bootsnap
+  module LoadPathCache
+    module ChangeObserver
+      module ArrayMixin
+        # For each method that adds items to one end or another of the array
+        # (<<, push, unshift, concat), override that method to also notify the
+        # observer of the change.
+        def <<(entry)
+          @lpc_observer.push_paths(self, entry.to_s)
+          super
+        end
+
+        def push(*entries)
+          @lpc_observer.push_paths(self, *entries.map(&:to_s))
+          super
+        end
+
+        def unshift(*entries)
+          @lpc_observer.unshift_paths(self, *entries.map(&:to_s))
+          super
+        end
+
+        def concat(entries)
+          @lpc_observer.push_paths(self, *entries.map(&:to_s))
+          super
+        end
+
+        # For each method that modifies the array more aggressively, override
+        # the method to also have the observer completely reconstruct its state
+        # after the modification. Many of these could be made to modify the
+        # internal state of the LoadPathCache::Cache more efficiently, but the
+        # accounting cost would be greater than the hit from these, since we
+        # actively discourage calling them.
+        %i(
+          []= clear collect! compact! delete delete_at delete_if fill flatten!
+          insert keep_if map! pop reject! replace reverse! rotate! select!
+          shift shuffle! slice! sort! sort_by! uniq!
+        ).each do |method_name|
+          define_method(method_name) do |*args, &block|
+            ret = super(*args, &block)
+            @lpc_observer.reinitialize
+            ret
+          end
+        end
+      end
+
+      def self.register(observer, arr)
+        arr.instance_variable_set(:@lpc_observer, observer)
+        arr.extend(ArrayMixin)
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/core_ext/active_support.rb

@@ -0,0 +1,75 @@
+module Bootsnap
+  module LoadPathCache
+    module CoreExt
+      module ActiveSupport
+        def self.without_bootsnap_cache
+          prev = Thread.current[:without_bootsnap_cache] || false
+          Thread.current[:without_bootsnap_cache] = true
+          yield
+        ensure
+          Thread.current[:without_bootsnap_cache] = prev
+        end
+
+        module ClassMethods
+          def autoload_paths=(o)
+            super
+            Bootsnap::LoadPathCache.autoload_paths_cache.reinitialize(o)
+          end
+
+          def search_for_file(path)
+            return super if Thread.current[:without_bootsnap_cache]
+            begin
+              Bootsnap::LoadPathCache.autoload_paths_cache.find(path)
+            rescue Bootsnap::LoadPathCache::ReturnFalse
+              nil # doesn't really apply here
+            end
+          end
+
+          def autoloadable_module?(path_suffix)
+            Bootsnap::LoadPathCache.autoload_paths_cache.has_dir?(path_suffix)
+          end
+
+          def remove_constant(const)
+            CoreExt::ActiveSupport.without_bootsnap_cache { super }
+          end
+
+          # If we can't find a constant using the patched implementation of
+          # search_for_file, try again with the default implementation.
+          #
+          # These methods call search_for_file, and we want to modify its
+          # behaviour.  The gymnastics here are a bit awkward, but it prevents
+          # 200+ lines of monkeypatches.
+          def load_missing_constant(from_mod, const_name)
+            super
+          rescue NameError => e
+            # NoMethodError is a NameError, but we only want to handle actual
+            # NameError instances.
+            raise unless e.class == NameError
+            # We can only confidently handle cases when *this* constant fails
+            # to load, not other constants referred to by it.
+            raise unless e.name == const_name
+            # If the constant was actually loaded, something else went wrong?
+            raise if from_mod.const_defined?(const_name)
+            CoreExt::ActiveSupport.without_bootsnap_cache { super }
+          end
+
+          # Signature has changed a few times over the years; easiest to not
+          # reiterate it with version polymorphism here...
+          def depend_on(*)
+            super
+          rescue LoadError
+            CoreExt::ActiveSupport.without_bootsnap_cache { super }
+          end
+        end
+      end
+    end
+  end
+end
+
+module ActiveSupport
+  module Dependencies
+    class << self
+      prepend Bootsnap::LoadPathCache::CoreExt::ActiveSupport::ClassMethods
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -0,0 +1,82 @@
+module Bootsnap
+  module LoadPathCache
+    module CoreExt
+      def self.make_load_error(path)
+        err = LoadError.new("cannot load such file -- #{path}")
+        err.define_singleton_method(:path) { path }
+        err
+      end
+    end
+  end
+end
+
+module Kernel
+  module_function
+
+  alias_method :require_without_bootsnap, :require
+
+  # Note that require registers to $LOADED_FEATURES while load does not.
+  def require_with_bootsnap_lfi(path, resolved = nil)
+    Bootsnap::LoadPathCache.loaded_features_index.register(path, resolved) do
+      require_without_bootsnap(resolved || path)
+    end
+  end
+
+  def require(path)
+    return false if Bootsnap::LoadPathCache.loaded_features_index.key?(path)
+
+    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
+      return require_with_bootsnap_lfi(path, resolved)
+    end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+  rescue Bootsnap::LoadPathCache::ReturnFalse
+    return false
+  rescue Bootsnap::LoadPathCache::FallbackScan
+    require_with_bootsnap_lfi(path)
+  end
+
+  alias_method :require_relative_without_bootsnap, :require_relative
+  def require_relative(path)
+    realpath = Bootsnap::LoadPathCache.realpath_cache.call(
+      caller_locations(1..1).first.absolute_path, path
+    )
+    require(realpath)
+  end
+
+  alias_method :load_without_bootsnap, :load
+  def load(path, wrap = false)
+    if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
+      return load_without_bootsnap(resolved, wrap)
+    end
+
+    # load also allows relative paths from pwd even when not in $:
+    if File.exist?(relative = File.expand_path(path))
+      return load_without_bootsnap(relative, wrap)
+    end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+  rescue Bootsnap::LoadPathCache::ReturnFalse
+    return false
+  rescue Bootsnap::LoadPathCache::FallbackScan
+    load_without_bootsnap(path, wrap)
+  end
+end
+
+class Module
+  alias_method :autoload_without_bootsnap, :autoload
+  def autoload(const, path)
+    # NOTE: This may defeat LoadedFeaturesIndex, but it's not immediately
+    # obvious how to make it work. This feels like a pretty niche case, unclear
+    # if it will ever burn anyone.
+    #
+    # The challenge is that we don't control the point at which the entry gets
+    # added to $LOADED_FEATURES and won't be able to hook that modification
+    # since it's done in C-land.
+    autoload_without_bootsnap(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
+  rescue Bootsnap::LoadPathCache::ReturnFalse
+    return false
+  rescue Bootsnap::LoadPathCache::FallbackScan
+    autoload_without_bootsnap(const, path)
+  end
+end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -0,0 +1,95 @@
+module Bootsnap
+  module LoadPathCache
+    # LoadedFeaturesIndex partially mirrors an internal structure in ruby that
+    # we can't easily obtain an interface to.
+    #
+    # This works around an issue where, without bootsnap, *ruby* knows that it
+    # has already required a file by its short name (e.g. require 'bundler') if
+    # a new instance of bundler is added to the $LOAD_PATH which resolves to a
+    # different absolute path. This class makes bootsnap smart enough to
+    # realize that it has already loaded 'bundler', and not just
+    # '/path/to/bundler'.
+    #
+    # If you disable LoadedFeaturesIndex, you can see the problem this solves by:
+    #
+    # 1. `require 'a'`
+    # 2. Prepend a new $LOAD_PATH element containing an `a.rb`
+    # 3. `require 'a'`
+    #
+    # Ruby returns false from step 3.
+    # With bootsnap but with no LoadedFeaturesIndex, this loads two different
+    #   `a.rb`s.
+    # With bootsnap and with LoadedFeaturesIndex, this skips the second load,
+    #   returning false like ruby.
+    class LoadedFeaturesIndex
+      def initialize
+        @lfi = {}
+        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+
+        # In theory the user could mutate $LOADED_FEATURES and invalidate our
+        # cache. If this ever comes up in practice — or if you, the
+        # enterprising reader, feels inclined to solve this problem — we could
+        # parallel the work done with ChangeObserver on $LOAD_PATH to mirror
+        # updates to our @lfi.
+        $LOADED_FEATURES.each do |feat|
+          $LOAD_PATH.each do |lpe|
+            next unless feat.start_with?(lpe)
+            # /a/b/lib/my/foo.rb
+            #          ^^^^^^^^^
+            short = feat[(lpe.length + 1)..-1]
+            @lfi[short] = true
+            @lfi[strip_extension(short)] = true
+          end
+        end
+      end
+
+      def key?(feature)
+        @mutex.synchronize { @lfi.key?(feature) }
+      end
+
+      # There is a relatively uncommon case where we could miss adding an
+      # entry:
+      #
+      # If the user asked for e.g. `require 'bundler'`, and we went through the
+      # `FallbackScan` pathway in `kernel_require.rb` and therefore did not
+      # pass `long` (the full expanded absolute path), then we did are not able
+      # to confidently add the `bundler.rb` form to @lfi.
+      #
+      # We could either:
+      #
+      # 1. Just add `bundler.rb`, `bundler.so`, and so on, which is close but
+      #    not quite right; or
+      # 2. Inspect $LOADED_FEATURES upon return from yield to find the matching
+      #    entry.
+      def register(short, long = nil)
+        ret = yield
+
+        # do we have 'bundler' or 'bundler.rb'?
+        altname = if File.extname(short) != ''
+          # strip the path from 'bundler.rb' -> 'bundler'
+          strip_extension(short)
+        elsif long && ext = File.extname(long)
+          # get the extension from the expanded path if given
+          # 'bundler' + '.rb'
+          short + ext
+        end
+
+        @mutex.synchronize do
+          @lfi[short] = true
+          (@lfi[altname] = true) if altname
+        end
+
+        ret
+      end
+
+      private
+
+      STRIP_EXTENSION = /\.[^.]*?$/
+      private_constant :STRIP_EXTENSION
+
+      def strip_extension(f)
+        f.sub(STRIP_EXTENSION, '')
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/path.rb

@@ -0,0 +1,113 @@
+require_relative 'path_scanner'
+
+module Bootsnap
+  module LoadPathCache
+    class Path
+      # A path is considered 'stable' if it is part of a Gem.path or the ruby
+      # distribution. When adding or removing files in these paths, the cache
+      # must be cleared before the change will be noticed.
+      def stable?
+        stability == STABLE
+      end
+
+      # A path is considered volatile if it doesn't live under a Gem.path or
+      # the ruby distribution root. These paths are scanned for new additions
+      # more frequently.
+      def volatile?
+        stability == VOLATILE
+      end
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path.to_s
+      end
+
+      # True if the path exists, but represents a non-directory object
+      def non_directory?
+        !File.stat(path).directory?
+      rescue Errno::ENOENT
+        false
+      end
+
+      def relative?
+        !path.start_with?(SLASH)
+      end
+
+      # Return a list of all the requirable files and all of the subdirectories
+      # of this +Path+.
+      def entries_and_dirs(store)
+        if stable?
+          # the cached_mtime field is unused for 'stable' paths, but is
+          # set to zero anyway, just in case we change the stability heuristics.
+          _, entries, dirs = store.get(expanded_path)
+          return [entries, dirs] if entries # cache hit
+          entries, dirs = scan!
+          store.set(expanded_path, [0, entries, dirs])
+          return [entries, dirs]
+        end
+
+        cached_mtime, entries, dirs = store.get(expanded_path)
+
+        current_mtime = latest_mtime(expanded_path, dirs || [])
+        return [[], []]        if current_mtime == -1 # path does not exist
+        return [entries, dirs] if cached_mtime == current_mtime
+
+        entries, dirs = scan!
+        store.set(expanded_path, [current_mtime, entries, dirs])
+        [entries, dirs]
+      end
+
+      def expanded_path
+        File.expand_path(path)
+      end
+
+      private
+
+      def scan! # (expensive) returns [entries, dirs]
+        PathScanner.call(expanded_path)
+      end
+
+      # last time a directory was modified in this subtree. +dirs+ should be a
+      # list of relative paths to directories under +path+. e.g. for /a/b and
+      # /a/b/c, pass ('/a/b', ['c'])
+      def latest_mtime(path, dirs)
+        max = -1
+        ["", *dirs].each do |dir|
+          curr = begin
+            File.mtime("#{path}/#{dir}").to_i
+          rescue Errno::ENOENT
+            -1
+          end
+          max = curr if curr > max
+        end
+        max
+      end
+
+      # a Path can be either stable of volatile, depending on how frequently we
+      # expect its contents may change. Stable paths aren't rescanned nearly as
+      # often.
+      STABLE   = :stable
+      VOLATILE = :volatile
+
+      # Built-in ruby lib stuff doesn't change, but things can occasionally be
+      # installed into sitedir, which generally lives under libdir.
+      RUBY_LIBDIR  = RbConfig::CONFIG['libdir']
+      RUBY_SITEDIR = RbConfig::CONFIG['sitedir']
+
+      def stability
+        @stability ||= begin
+          if Gem.path.detect { |p| expanded_path.start_with?(p.to_s) }
+            STABLE
+          elsif Bootsnap.bundler? && expanded_path.start_with?(Bundler.bundle_path.to_s)
+            STABLE
+          elsif expanded_path.start_with?(RUBY_LIBDIR) && !expanded_path.start_with?(RUBY_SITEDIR)
+            STABLE
+          else
+            VOLATILE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -0,0 +1,44 @@
+require_relative '../explicit_require'
+
+module Bootsnap
+  module LoadPathCache
+    module PathScanner
+      ALL_FILES = "/{,**/*/**/}*"
+      REQUIRABLE_EXTENSIONS = [DOT_RB] + DL_EXTENSIONS
+      NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
+      ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/
+      BUNDLE_PATH = Bootsnap.bundler? ?
+        (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze : ''.freeze
+
+      def self.call(path)
+        path = path.to_s
+
+        relative_slice = (path.size + 1)..-1
+        # If the bundle path is a descendent of this path, we do additional
+        # checks to prevent recursing into the bundle path as we recurse
+        # through this path. We don't want to scan the bundle path because
+        # anything useful in it will be present on other load path items.
+        #
+        # This can happen if, for example, the user adds '.' to the load path,
+        # and the bundle path is '.bundle'.
+        contains_bundle_path = BUNDLE_PATH.start_with?(path)
+
+        dirs = []
+        requirables = []
+
+        Dir.glob(path + ALL_FILES).each do |absolute_path|
+          next if contains_bundle_path && absolute_path.start_with?(BUNDLE_PATH)
+          relative_path = absolute_path.slice(relative_slice)
+
+          if File.directory?(absolute_path)
+            dirs << relative_path
+          elsif REQUIRABLE_EXTENSIONS.include?(File.extname(relative_path))
+            requirables << relative_path
+          end
+        end
+
+        [requirables, dirs]
+      end
+    end
+  end
+end