zeitwerk 2.5.0 → 2.6.12

data/lib/zeitwerk/loader.rb

@@ -1,36 +1,37 @@
 # frozen_string_literal: true
 
+require "monitor"
 require "set"
-require "securerandom"
 
 module Zeitwerk
   class Loader
     require_relative "loader/helpers"
     require_relative "loader/callbacks"
     require_relative "loader/config"
+    require_relative "loader/eager_load"
+
+    extend Internal
 
     include RealModName
     include Callbacks
     include Helpers
     include Config
+    include EagerLoad
 
-    # Keeps track of autoloads defined by the loader which have not been
-    # executed so far.
-    #
-    # This metadata helps us implement a few things:
-    #
-    # 1. When autoloads are triggered, ensure they define the expected constant
-    #    and invoke user callbacks. If reloading is enabled, remember cref and
-    #    abspath for later unloading logic.
-    #
-    # 2. When unloading, remove autoloads that have not been executed.
+    MUTEX = Mutex.new
+    private_constant :MUTEX
+
+    # Maps absolute paths for which an autoload has been set ---and not
+    # executed--- to their corresponding parent class or module and constant
+    # name.
     #
-    # 3. Eager load with a recursive const_get, rather than a recursive require,
-    #    for consistency with lazy loading.
+    #   "/Users/fxn/blog/app/models/user.rb"          => [Object, :User],
+    #   "/Users/fxn/blog/app/models/hotel/pricing.rb" => [Hotel, :Pricing]
+    #   ...
     #
-    # @private
-    # @sig Zeitwerk::Autoloads
+    # @sig Hash[String, [Module, Symbol]]
     attr_reader :autoloads
+    internal :autoloads
 
     # We keep track of autoloaded directories to remove them from the registry
     # at the end of eager loading.
@@ -38,9 +39,9 @@ module Zeitwerk
     # Files are removed as they are autoloaded, but directories need to wait due
     # to concurrency (see why in Zeitwerk::Loader::Callbacks#on_dir_autoloaded).
     #
-    # @private
     # @sig Array[String]
     attr_reader :autoloaded_dirs
+    internal :autoloaded_dirs
 
     # Stores metadata needed for unloading. Its entries look like this:
     #
@@ -54,11 +55,11 @@ module Zeitwerk
     # If reloading is enabled, this hash is filled as constants are autoloaded
     # or eager loaded. Otherwise, the collection remains empty.
     #
-    # @private
     # @sig Hash[String, [String, [Module, Symbol]]]
     attr_reader :to_unload
+    internal :to_unload
 
-    # Maps constant paths of namespaces to arrays of corresponding directories.
+    # Maps namespace constant paths to their respective directories.
     #
     # For example, given this mapping:
     #
@@ -68,46 +69,59 @@ module Zeitwerk
     #     ...
     #   ]
     #
-    # when `Admin` gets defined we know that it plays the role of a namespace and
-    # that its children are spread over those directories. We'll visit them to set
-    # up the corresponding autoloads.
+    # when `Admin` gets defined we know that it plays the role of a namespace
+    # and that its children are spread over those directories. We'll visit them
+    # to set up the corresponding autoloads.
     #
-    # @private
     # @sig Hash[String, Array[String]]
-    attr_reader :lazy_subdirs
+    attr_reader :namespace_dirs
+    internal :namespace_dirs
+
+    # A shadowed file is a file managed by this loader that is ignored when
+    # setting autoloads because its matching constant is already taken.
+    #
+    # This private set is populated as we descend. For example, if the loader
+    # has only scanned the top-level, `shadowed_files` does not have shadowed
+    # files that may exist deep in the project tree yet.
+    #
+    # @sig Set[String]
+    attr_reader :shadowed_files
+    internal :shadowed_files
 
-    # @private
     # @sig Mutex
     attr_reader :mutex
+    private :mutex
 
-    # @private
-    # @sig Mutex
-    attr_reader :mutex2
+    # @sig Monitor
+    attr_reader :dirs_autoload_monitor
+    private :dirs_autoload_monitor
 
     def initialize
       super
 
-      @autoloads       = Autoloads.new
+      @autoloads       = {}
       @autoloaded_dirs = []
       @to_unload       = {}
-      @lazy_subdirs    = Hash.new { |h, cpath| h[cpath] = [] }
-      @mutex           = Mutex.new
-      @mutex2          = Mutex.new
+      @namespace_dirs  = Hash.new { |h, cpath| h[cpath] = [] }
+      @shadowed_files  = Set.new
       @setup           = false
       @eager_loaded    = false
 
+      @mutex = Mutex.new
+      @dirs_autoload_monitor = Monitor.new
+
       Registry.register_loader(self)
     end
 
-    # Sets autoloads in the root namespace.
+    # Sets autoloads in the root namespaces.
     #
     # @sig () -> void
     def setup
       mutex.synchronize do
         break if @setup
 
-        actual_root_dirs.each do |root_dir, namespace|
-          set_autoloads_in_dir(root_dir, namespace)
+        actual_roots.each do |root_dir, root_namespace|
+          define_autoloads_for_dir(root_dir, root_namespace)
         end
 
         on_setup_callbacks.each(&:call)
@@ -124,12 +138,14 @@ module Zeitwerk
     # unload them.
     #
     # This method is public but undocumented. Main interface is `reload`, which
-    # means `unload` + `setup`. This one is avaiable to be used together with
+    # means `unload` + `setup`. This one is available to be used together with
     # `unregister`, which is undocumented too.
     #
     # @sig () -> void
     def unload
       mutex.synchronize do
+        raise SetupRequired unless @setup
+
         # We are going to keep track of the files that were required by our
         # autoloads to later remove them from $LOADED_FEATURES, thus making them
         # loadable by Kernel#require again.
@@ -138,7 +154,7 @@ module Zeitwerk
         # is enough.
         unloaded_files = Set.new
 
-        autoloads.each do |(parent, cname), abspath|
+        autoloads.each do |abspath, (parent, cname)|
           if parent.autoload?(cname)
             unload_autoload(parent, cname)
           else
@@ -152,8 +168,15 @@ module Zeitwerk
 
         to_unload.each do |cpath, (abspath, (parent, cname))|
           unless on_unload_callbacks.empty?
-            value = parent.const_get(cname)
-            run_on_unload_callbacks(cpath, value, abspath)
+            begin
+              value = cget(parent, cname)
+            rescue ::NameError
+              # Perhaps the user deleted the constant by hand, or perhaps an
+              # autoload failed to define the expected constant but the user
+              # rescued the exception.
+            else
+              run_on_unload_callbacks(cpath, value, abspath)
+            end
           end
 
           unload_cref(parent, cname)
@@ -178,10 +201,11 @@ module Zeitwerk
         autoloads.clear
         autoloaded_dirs.clear
         to_unload.clear
-        lazy_subdirs.clear
+        namespace_dirs.clear
+        shadowed_files.clear
 
         Registry.on_unload(self)
-        ExplicitNamespace.unregister_loader(self)
+        ExplicitNamespace.__unregister_loader(self)
 
         @setup        = false
         @eager_loaded = false
@@ -197,67 +221,62 @@ module Zeitwerk
     # @raise [Zeitwerk::Error]
     # @sig () -> void
     def reload
-      if reloading_enabled?
-        unload
-        recompute_ignored_paths
-        recompute_collapse_dirs
-        setup
-      else
-        raise ReloadingDisabledError, "can't reload, please call loader.enable_reloading before setup"
-      end
+      raise ReloadingDisabledError unless reloading_enabled?
+      raise SetupRequired unless @setup
+
+      unload
+      recompute_ignored_paths
+      recompute_collapse_dirs
+      setup
     end
 
-    # Eager loads all files in the root directories, recursively. Files do not
-    # need to be in `$LOAD_PATH`, absolute file names are used. Ignored files
-    # are not eager loaded. You can opt-out specifically in specific files and
-    # directories with `do_not_eager_load`, and that can be overridden passing
-    # `force: true`.
-    #
-    # @sig (true | false) -> void
-    def eager_load(force: false)
-      mutex.synchronize do
-        break if @eager_loaded
+  # @sig (String | Pathname) -> String?
+  def cpath_expected_at(path)
+    abspath = File.expand_path(path)
 
-        log("eager load start") if logger
+    raise Zeitwerk::Error.new("#{abspath} does not exist") unless File.exist?(abspath)
 
-        honour_exclusions = !force
+    return unless dir?(abspath) || ruby?(abspath)
+    return if ignored_path?(abspath)
 
-        queue = []
-        actual_root_dirs.each do |root_dir, namespace|
-          queue << [namespace, root_dir] unless honour_exclusions && excluded_from_eager_load?(root_dir)
-        end
+    paths = []
 
-        while to_eager_load = queue.shift
-          namespace, dir = to_eager_load
-
-          ls(dir) do |basename, abspath|
-            next if honour_exclusions && excluded_from_eager_load?(abspath)
-
-            if ruby?(abspath)
-              if cref = autoloads.cref_for(abspath)
-                cget(*cref)
-              end
-            elsif dir?(abspath) && !root_dirs.key?(abspath)
-              if collapse?(abspath)
-                queue << [namespace, abspath]
-              else
-                cname = inflector.camelize(basename, abspath)
-                queue << [cget(namespace, cname), abspath]
-              end
-            end
-          end
-        end
+    if ruby?(abspath)
+      basename = File.basename(abspath, ".rb")
+      return if hidden?(basename)
 
-        autoloaded_dirs.each do |autoloaded_dir|
-          Registry.unregister_autoload(autoloaded_dir)
-        end
-        autoloaded_dirs.clear
+      paths << [basename, abspath]
+      walk_up_from = File.dirname(abspath)
+    else
+      walk_up_from = abspath
+    end
 
-        @eager_loaded = true
+    root_namespace = nil
 
-        log("eager load end") if logger
+    walk_up(walk_up_from) do |dir|
+      break if root_namespace = roots[dir]
+      return if ignored_path?(dir)
+
+      basename = File.basename(dir)
+      return if hidden?(basename)
+
+      paths << [basename, abspath] unless collapse?(dir)
+    end
+
+    return unless root_namespace
+
+    if paths.empty?
+      real_mod_name(root_namespace)
+    else
+      cnames = paths.reverse_each.map { |b, a| cname_for(b, a) }
+
+      if root_namespace == Object
+        cnames.join("::")
+      else
+        "#{real_mod_name(root_namespace)}::#{cnames.join("::")}"
       end
     end
+  end
 
     # Says if the given constant path would be unloaded on reload. This
     # predicate returns `false` if reloading is disabled.
@@ -281,22 +300,29 @@ module Zeitwerk
     # @sig () -> void
     def unregister
       Registry.unregister_loader(self)
-      ExplicitNamespace.unregister_loader(self)
+      ExplicitNamespace.__unregister_loader(self)
+    end
+
+    # The return value of this predicate is only meaningful if the loader has
+    # scanned the file. This is the case in the spots where we use it.
+    #
+    # @sig (String) -> Boolean
+    internal def shadowed_file?(file)
+      shadowed_files.member?(file)
     end
 
     # --- Class methods ---------------------------------------------------------------------------
 
     class << self
+      include RealModName
+
       # @sig #call | #debug | nil
       attr_accessor :default_logger
 
-      # @private
-      # @sig Mutex
-      attr_accessor :mutex
-
       # This is a shortcut for
       #
       #   require "zeitwerk"
+      #
       #   loader = Zeitwerk::Loader.new
       #   loader.tag = File.basename(__FILE__, ".rb")
       #   loader.inflector = Zeitwerk::GemInflector.new(__FILE__)
@@ -305,17 +331,70 @@ module Zeitwerk
       # except that this method returns the same object in subsequent calls from
       # the same file, in the unlikely case the gem wants to be able to reload.
       #
-      # @sig () -> Zeitwerk::Loader
-      def for_gem
+      # This method returns a subclass of Zeitwerk::Loader, but the exact type
+      # is private, client code can only rely on the interface.
+      #
+      # @sig (bool) -> Zeitwerk::GemLoader
+      def for_gem(warn_on_extra_files: true)
         called_from = caller_locations(1, 1).first.path
-        Registry.loader_for_gem(called_from)
+        Registry.loader_for_gem(called_from, namespace: Object, warn_on_extra_files: warn_on_extra_files)
       end
 
-      # Broadcasts `eager_load` to all loaders.
+      # This is a shortcut for
+      #
+      #   require "zeitwerk"
+      #
+      #   loader = Zeitwerk::Loader.new
+      #   loader.tag = namespace.name + "-" + File.basename(__FILE__, ".rb")
+      #   loader.inflector = Zeitwerk::GemInflector.new(__FILE__)
+      #   loader.push_dir(__dir__, namespace: namespace)
+      #
+      # except that this method returns the same object in subsequent calls from
+      # the same file, in the unlikely case the gem wants to be able to reload.
+      #
+      # This method returns a subclass of Zeitwerk::Loader, but the exact type
+      # is private, client code can only rely on the interface.
+      #
+      # @sig (bool) -> Zeitwerk::GemLoader
+      def for_gem_extension(namespace)
+        unless namespace.is_a?(Module) # Note that Class < Module.
+          raise Zeitwerk::Error, "#{namespace.inspect} is not a class or module object, should be"
+        end
+
+        unless real_mod_name(namespace)
+          raise Zeitwerk::Error, "extending anonymous namespaces is unsupported"
+        end
+
+        called_from = caller_locations(1, 1).first.path
+        Registry.loader_for_gem(called_from, namespace: namespace, warn_on_extra_files: false)
+      end
+
+      # Broadcasts `eager_load` to all loaders. Those that have not been setup
+      # are skipped.
       #
       # @sig () -> void
       def eager_load_all
-        Registry.loaders.each(&:eager_load)
+        Registry.loaders.each do |loader|
+          begin
+            loader.eager_load
+          rescue SetupRequired
+            # This is fine, we eager load what can be eager loaded.
+          end
+        end
+      end
+
+      # Broadcasts `eager_load_namespace` to all loaders. Those that have not
+      # been setup are skipped.
+      #
+      # @sig (Module) -> void
+      def eager_load_namespace(mod)
+        Registry.loaders.each do |loader|
+          begin
+            loader.eager_load_namespace(mod)
+          rescue SetupRequired
+            # This is fine, we eager load what can be eager loaded.
+          end
+        end
       end
 
       # Returns an array with the absolute paths of the root directories of all
@@ -327,79 +406,58 @@ module Zeitwerk
       end
     end
 
-    self.mutex = Mutex.new
-
-    private # -------------------------------------------------------------------------------------
-
     # @sig (String, Module) -> void
-    def set_autoloads_in_dir(dir, parent)
+    private def define_autoloads_for_dir(dir, parent)
       ls(dir) do |basename, abspath|
-        begin
-          if ruby?(basename)
-            basename.delete_suffix!(".rb")
-            cname = inflector.camelize(basename, abspath).to_sym
-            autoload_file(parent, cname, abspath)
-          elsif dir?(abspath)
-            # In a Rails application, `app/models/concerns` is a subdirectory of
-            # `app/models`, but both of them are root directories.
-            #
-            # To resolve the ambiguity file name -> constant path this introduces,
-            # the `app/models/concerns` directory is totally ignored as a namespace,
-            # it counts only as root. The guard checks that.
-            unless root_dir?(abspath)
-              cname = inflector.camelize(basename, abspath).to_sym
-              if collapse?(abspath)
-                set_autoloads_in_dir(abspath, parent)
-              else
-                autoload_subdir(parent, cname, abspath)
-              end
-            end
+        if ruby?(basename)
+          basename.delete_suffix!(".rb")
+          autoload_file(parent, cname_for(basename, abspath), abspath)
+        else
+          if collapse?(abspath)
+            define_autoloads_for_dir(abspath, parent)
+          else
+            autoload_subdir(parent, cname_for(basename, abspath), abspath)
           end
-        rescue ::NameError => error
-          path_type = ruby?(abspath) ? "file" : "directory"
-
-          raise NameError.new(<<~MESSAGE, error.name)
-            #{error.message} inferred by #{inflector.class} from #{path_type}
-
-              #{abspath}
-
-            Possible ways to address this:
-
-              * Tell Zeitwerk to ignore this particular #{path_type}.
-              * Tell Zeitwerk to ignore one of its parent directories.
-              * Rename the #{path_type} to comply with the naming conventions.
-              * Modify the inflector to handle this case.
-          MESSAGE
         end
       end
     end
 
     # @sig (Module, Symbol, String) -> void
-    def autoload_subdir(parent, cname, subdir)
-      if autoload_path = autoloads.abspath_for(parent, cname)
+    private def autoload_subdir(parent, cname, subdir)
+      if autoload_path = autoload_path_set_by_me_for?(parent, cname)
         cpath = cpath(parent, cname)
-        register_explicit_namespace(cpath) if ruby?(autoload_path)
-        # We do not need to issue another autoload, the existing one is enough
-        # no matter if it is for a file or a directory. Just remember the
-        # subdirectory has to be visited if the namespace is used.
-        lazy_subdirs[cpath] << subdir
+        if ruby?(autoload_path)
+          # Scanning visited a Ruby file first, and now a directory for the same
+          # constant has been found. This means we are dealing with an explicit
+          # namespace whose definition was seen first.
+          #
+          # Registering is idempotent, and we have to keep the autoload pointing
+          # to the file. This may run again if more directories are found later
+          # on, no big deal.
+          register_explicit_namespace(cpath)
+        end
+        # If the existing autoload points to a file, it has to be preserved, if
+        # not, it is fine as it is. In either case, we do not need to override.
+        # Just remember the subdirectory conforms this namespace.
+        namespace_dirs[cpath] << subdir
       elsif !cdef?(parent, cname)
         # First time we find this namespace, set an autoload for it.
-        lazy_subdirs[cpath(parent, cname)] << subdir
-        set_autoload(parent, cname, subdir)
+        namespace_dirs[cpath(parent, cname)] << subdir
+        define_autoload(parent, cname, subdir)
       else
         # For whatever reason the constant that corresponds to this namespace has
         # already been defined, we have to recurse.
         log("the namespace #{cpath(parent, cname)} already exists, descending into #{subdir}") if logger
-        set_autoloads_in_dir(subdir, cget(parent, cname))
+        define_autoloads_for_dir(subdir, cget(parent, cname))
       end
     end
 
     # @sig (Module, Symbol, String) -> void
-    def autoload_file(parent, cname, file)
+    private def autoload_file(parent, cname, file)
       if autoload_path = strict_autoload_path(parent, cname) || Registry.inception?(cpath(parent, cname))
         # First autoload for a Ruby file wins, just ignore subsequent ones.
         if ruby?(autoload_path)
+          shadowed_files << file
           log("file #{file} is ignored because #{autoload_path} has precedence") if logger
         else
           promote_namespace_from_implicit_to_explicit(
@@ -410,9 +468,10 @@ module Zeitwerk
           )
         end
       elsif cdef?(parent, cname)
+        shadowed_files << file
         log("file #{file} is ignored because #{cpath(parent, cname)} is already defined") if logger
       else
-        set_autoload(parent, cname, file)
+        define_autoload(parent, cname, file)
       end
     end
 
@@ -420,19 +479,19 @@ module Zeitwerk
     # the file where we've found the namespace is explicitly defined.
     #
     # @sig (dir: String, file: String, parent: Module, cname: Symbol) -> void
-    def promote_namespace_from_implicit_to_explicit(dir:, file:, parent:, cname:)
+    private def promote_namespace_from_implicit_to_explicit(dir:, file:, parent:, cname:)
       autoloads.delete(dir)
       Registry.unregister_autoload(dir)
 
       log("earlier autoload for #{cpath(parent, cname)} discarded, it is actually an explicit namespace defined in #{file}") if logger
 
-      set_autoload(parent, cname, file)
+      define_autoload(parent, cname, file)
       register_explicit_namespace(cpath(parent, cname))
     end
 
     # @sig (Module, Symbol, String) -> void
-    def set_autoload(parent, cname, abspath)
-      autoloads.define(parent, cname, abspath)
+    private def define_autoload(parent, cname, abspath)
+      parent.autoload(cname, abspath)
 
       if logger
         if ruby?(abspath)
@@ -442,6 +501,7 @@ module Zeitwerk
         end
       end
 
+      autoloads[abspath] = [parent, cname]
       Registry.register_autoload(self, abspath)
 
       # See why in the documentation of Zeitwerk::Registry.inceptions.
@@ -450,29 +510,38 @@ module Zeitwerk
       end
     end
 
+    # @sig (Module, Symbol) -> String?
+    private def autoload_path_set_by_me_for?(parent, cname)
+      if autoload_path = strict_autoload_path(parent, cname)
+        autoload_path if autoloads.key?(autoload_path)
+      else
+        Registry.inception?(cpath(parent, cname))
+      end
+    end
+
     # @sig (String) -> void
-    def register_explicit_namespace(cpath)
-      ExplicitNamespace.register(cpath, self)
+    private def register_explicit_namespace(cpath)
+      ExplicitNamespace.__register(cpath, self)
     end
 
     # @sig (String) -> void
-    def raise_if_conflicting_directory(dir)
-      self.class.mutex.synchronize do
+    private def raise_if_conflicting_directory(dir)
+      MUTEX.synchronize do
+        dir_slash = dir + "/"
+
         Registry.loaders.each do |loader|
           next if loader == self
-          next if loader.ignores?(dir)
+          next if loader.__ignores?(dir)
 
-          dir = dir + "/"
-          loader.root_dirs.each do |root_dir, _namespace|
+          loader.__roots.each_key do |root_dir|
             next if ignores?(root_dir)
 
-            root_dir = root_dir + "/"
-            if dir.start_with?(root_dir) || root_dir.start_with?(dir)
+            root_dir_slash = root_dir + "/"
+            if dir_slash.start_with?(root_dir_slash) || root_dir_slash.start_with?(dir_slash)
               require "pp" # Needed for pretty_inspect, even in Ruby 2.5.
               raise Error,
-                "loader\n\n#{pretty_inspect}\n\nwants to manage directory #{dir.chop}," \
+                "loader\n\n#{pretty_inspect}\n\nwants to manage directory #{dir}," \
                 " which is already managed by\n\n#{loader.pretty_inspect}\n"
-              EOS
             end
           end
         end
@@ -480,23 +549,23 @@ module Zeitwerk
     end
 
     # @sig (String, Object, String) -> void
-    def run_on_unload_callbacks(cpath, value, abspath)
+    private def run_on_unload_callbacks(cpath, value, abspath)
       # Order matters. If present, run the most specific one.
       on_unload_callbacks[cpath]&.each { |c| c.call(value, abspath) }
       on_unload_callbacks[:ANY]&.each { |c| c.call(cpath, value, abspath) }
     end
 
     # @sig (Module, Symbol) -> void
-    def unload_autoload(parent, cname)
-      parent.__send__(:remove_const, cname)
+    private def unload_autoload(parent, cname)
+      crem(parent, cname)
       log("autoload for #{cpath(parent, cname)} removed") if logger
     end
 
     # @sig (Module, Symbol) -> void
-    def unload_cref(parent, cname)
+    private def unload_cref(parent, cname)
       # Let's optimistically remove_const. The way we use it, this is going to
       # succeed always if all is good.
-      parent.__send__(:remove_const, cname)
+      crem(parent, cname)
     rescue ::NameError
       # There are a few edge scenarios in which this may happen. If the constant
       # is gone, that is OK, anyway.