r10k 3.9.3 → 3.12.1

data/lib/r10k/module_loader/puppetfile.rb

@@ -0,0 +1,276 @@
+require 'r10k/errors'
+require 'r10k/logging'
+require 'r10k/module'
+require 'r10k/module_loader/puppetfile/dsl'
+
+require 'pathname'
+
+module R10K
+  module ModuleLoader
+    class Puppetfile
+
+      include R10K::Logging
+
+      DEFAULT_MODULEDIR = 'modules'
+      DEFAULT_PUPPETFILE_NAME = 'Puppetfile'
+
+      attr_accessor :default_branch_override, :environment
+      attr_reader :modules, :moduledir, :puppetfile_path,
+        :managed_directories, :desired_contents, :purge_exclusions
+
+      # @param basedir [String] The path that contains the moduledir &
+      #     Puppetfile by default. May be an environment, project, or
+      #     simple directory.
+      # @param puppetfile [String] The path to the Puppetfile, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param moduledir [String] The path to the moduledir, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param forge [String] The url (without protocol) to the Forge
+      # @param overrides [Hash] Configuration for loaded modules' behavior
+      # @param environment [R10K::Environment] When provided, the environment
+      #     in which loading takes place
+      def initialize(basedir:,
+                     moduledir: DEFAULT_MODULEDIR,
+                     puppetfile: DEFAULT_PUPPETFILE_NAME,
+                     overrides: {},
+                     environment: nil)
+
+        @basedir     = cleanpath(basedir)
+        @moduledir   = resolve_path(@basedir, moduledir)
+        @puppetfile_path  = resolve_path(@basedir, puppetfile)
+        @overrides   = overrides
+        @environment = environment
+        @default_branch_override = @overrides.dig(:environments, :default_branch_override)
+        @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
+
+        @existing_module_metadata = []
+        @existing_module_versions_by_name = {}
+        @modules = []
+
+        @managed_directories = []
+        @desired_contents = []
+        @purge_exclusions = []
+      end
+
+      def load
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load!
+        end
+      end
+
+      def load!
+        logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
+        logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir}
+
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        validate_no_duplicate_names(@modules)
+
+        managed_content = @modules.group_by(&:dirname)
+
+        @managed_directories = determine_managed_directories(managed_content)
+        @desired_contents = determine_desired_contents(managed_content)
+        @purge_exclusions = determine_purge_exclusions(@managed_directories)
+
+        {
+          modules: @modules,
+          managed_directories: @managed_directories,
+          desired_contents: @desired_contents,
+          purge_exclusions: @purge_exclusions
+        }
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
+      end
+
+      def load_metadata
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load_metadata!
+        end
+      end
+
+      def load_metadata!
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h
+        empty_load_output.merge(modules: @existing_module_metadata)
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message })
+      end
+
+      def add_module_metadata(name, info)
+        install_path, metadata_info, _ = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+
+        @existing_module_metadata << mod
+      end
+
+      ##
+      ## set_forge, set_moduledir, and add_module are used directly by the DSL class
+      ##
+
+      # @param [String] forge
+      def set_forge(forge)
+        if @allow_puppetfile_forge
+          logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge }
+          PuppetForge.host = forge
+        else
+          logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host }
+        end
+      end
+
+      # @param [String] moduledir
+      def set_moduledir(moduledir)
+        @moduledir = resolve_path(@basedir, moduledir)
+      end
+
+      # @param [String] name
+      # @param [Hash, String, Symbol, nil] info Calling with
+      #   anything but a Hash is deprecated. The DSL will now convert
+      #   String and Symbol versions to Hashes of the shape
+      #     { version: <String or Symbol> }
+      #
+      #   String inputs should be valid module versions, the Symbol
+      #   `:latest` is allowed, as well as `nil`.
+      #
+      #   Non-Hash inputs are only ever used by Forge modules. In
+      #   future versions this method will require the caller (the
+      #   DSL class, not the Puppetfile author) to do this conversion
+      #   itself.
+      #
+      def add_module(name, info)
+        install_path, metadata_info, spec_deletable = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+        mod.origin = :puppetfile
+        mod.spec_deletable = spec_deletable
+
+        # Do not save modules if they would conflict with the attached
+        # environment
+        if @environment && @environment.module_conflicts?(mod)
+          return @modules
+        end
+
+        # If this module's metadata has a static version and that version
+        # matches the existing module declaration use it, otherwise create
+        # a regular module to sync.
+        unless mod.version && (mod.version == @existing_module_versions_by_name[mod.name])
+          mod = mod.to_implementation
+        end
+
+        @modules << mod
+      end
+
+     private
+
+      def empty_load_output
+        {
+          modules: [],
+          managed_directories: [],
+          desired_contents: [],
+          purge_exclusions: []
+        }
+      end
+
+      def with_readable_puppetfile(puppetfile_path, &block)
+        if File.readable?(puppetfile_path)
+          block.call
+        else
+          logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
+
+          empty_load_output
+        end
+      end
+
+      def parse_module_definition(name, info)
+        if !info.is_a?(Hash)
+          info = { version: info }
+        end
+
+        info[:overrides] = @overrides
+
+        if @default_branch_override
+          info[:default_branch_override] = @default_branch_override
+        end
+
+        spec_deletable = false
+        if install_path = info.delete(:install_path)
+          install_path = resolve_path(@basedir, install_path)
+          validate_install_path(install_path, name)
+        else
+          install_path = @moduledir
+          spec_deletable = true
+        end
+
+        return [ install_path, info, spec_deletable ]
+      end
+
+      # @param [Array<R10K::Module>] modules
+      def validate_no_duplicate_names(modules)
+        dupes = modules
+                .group_by { |mod| mod.name }
+                .select { |_, mods| mods.size > 1 }
+                .map(&:first)
+        unless dupes.empty?
+          msg = _('Puppetfiles cannot contain duplicate module names.')
+          msg += ' '
+          msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
+          raise R10K::Error.new(msg)
+        end
+      end
+
+      def resolve_path(base, path)
+        if Pathname.new(path).absolute?
+          cleanpath(path)
+        else
+          cleanpath(File.join(base, path))
+        end
+      end
+
+      def validate_install_path(path, modname)
+        unless /^#{Regexp.escape(@basedir)}.*/ =~ path
+          raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}")
+        end
+
+        true
+      end
+
+      def determine_managed_directories(managed_content)
+        managed_content.keys.reject { |dir| dir == @basedir }
+      end
+
+      # Returns an array of the full paths to all the content being managed.
+      # @return [Array<String>]
+      def determine_desired_contents(managed_content)
+        managed_content.flat_map do |install_path, mods|
+          mods.collect { |mod| File.join(install_path, mod.name) }
+        end
+      end
+
+      def determine_purge_exclusions(managed_dirs)
+        if environment && environment.respond_to?(:desired_contents)
+          managed_dirs + environment.desired_contents
+        else
+          managed_dirs
+        end
+      end
+
+      # .cleanpath is as close to a canonical path as we can do without touching
+      # the filesystem. The .realpath methods will choke if some of the
+      # intermediate paths are missing, even though in some cases we will create
+      # them later as needed.
+      def cleanpath(path)
+        Pathname.new(path).cleanpath.to_s
+      end
+
+      # For testing purposes only
+      def puppetfile_content(path)
+        File.read(path)
+      end
+    end
+  end
+end

data/lib/r10k/puppetfile.rb

@@ -4,10 +4,17 @@ require 'r10k/module'
 require 'r10k/util/purgeable'
 require 'r10k/errors'
 require 'r10k/content_synchronizer'
+require 'r10k/module_loader/puppetfile/dsl'
+require 'r10k/module_loader/puppetfile'
 
 module R10K
+
+# Deprecated, use R10K::ModuleLoader::Puppetfile#load to load content,
+# provide the `:modules` key of the returned Hash to
+# R10K::ContentSynchronizer (either the `serial_sync` or `concurrent_sync`)
+# and the remaining keys (`:managed_directories`, `:desired_contents`, and
+# `:purge_exclusions`) to R10K::Util::Cleaner.
 class Puppetfile
-  # Defines the data members of a Puppetfile
 
   include R10K::Settings::Mixin
 
@@ -19,25 +26,13 @@ class Puppetfile
   #   @return [String] The URL to use for the Puppet Forge
   attr_reader :forge
 
-  # @!attribute [r] modules
-  #   @return [Array<R10K::Module>]
-  attr_reader :modules
-
   # @!attribute [r] basedir
   #   @return [String] The base directory that contains the Puppetfile
   attr_reader :basedir
 
-  # @!attribute [r] moduledir
-  #   @return [String] The directory to install the modules #{basedir}/modules
-  attr_reader :moduledir
-
-  # @!attrbute [r] puppetfile_path
-  #   @return [String] The path to the Puppetfile
-  attr_reader :puppetfile_path
-
-  # @!attribute [rw] environment
+  # @!attribute [r] environment
   #   @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to.
-  attr_accessor :environment
+  attr_reader :environment
 
   # @!attribute [rw] force
   #   @return [Boolean] Overwrite any locally made changes
@@ -47,6 +42,10 @@ class Puppetfile
   #   @return [Hash] Various settings overridden from normal configs
   attr_reader :overrides
 
+  # @!attribute [r] loader
+  #   @return [R10K::ModuleLoader::Puppetfile] The internal module loader
+  attr_reader :loader
+
   # @param [String] basedir
   # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options.
   #         Usage as moduledir is deprecated. Only use as options, defaults to nil
@@ -65,144 +64,122 @@ class Puppetfile
 
     @force           = deprecated_force_arg     || options.delete(:force)           || false
     @moduledir       = deprecated_moduledir_arg || options.delete(:moduledir)       || File.join(basedir, 'modules')
-    @puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
-    @puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
+    puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
+    puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path)
+    @puppetfile = puppetfile_path || puppetfile_name
+    @environment     = options.delete(:environment)
 
     @overrides       = options.delete(:overrides) || {}
+    @default_branch_override = @overrides.dig(:environments, :default_branch_override)
 
-    logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
-
-    @modules = []
-    @managed_content = {}
     @forge   = 'forgeapi.puppetlabs.com'
 
+    @loader = ::R10K::ModuleLoader::Puppetfile.new(
+      basedir: @basedir,
+      moduledir: @moduledir,
+      puppetfile: @puppetfile,
+      overrides: @overrides,
+      environment: @environment
+    )
+
+    @loaded_content = {
+      modules: [],
+      managed_directories: [],
+      desired_contents: [],
+      purge_exclusions: []
+    }
+
     @loaded = false
   end
 
+  # @param [String] default_branch_override The default branch to use
+  #   instead of one specified in the module declaration, if applicable.
+  #   Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
+  #   the default_branch_override as an option on initialization.
   def load(default_branch_override = nil)
-    return true if self.loaded?
-    if File.readable? @puppetfile_path
-      self.load!(default_branch_override)
+    if self.loaded?
+      return @loaded_content
     else
-      logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
+      if !File.readable?(puppetfile_path)
+        logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
+      else
+        self.load!(default_branch_override)
+      end
     end
   end
 
+  # @param [String] default_branch_override The default branch to use
+  #   instead of one specified in the module declaration, if applicable.
+  #   Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
+  #   the default_branch_override as an option on initialization.
   def load!(default_branch_override = nil)
-    @default_branch_override = default_branch_override
 
-    dsl = R10K::Puppetfile::DSL.new(self)
-    dsl.instance_eval(puppetfile_contents, @puppetfile_path)
+    if default_branch_override && (default_branch_override != @default_branch_override)
+      logger.warn("Mismatch between passed and initialized default branch overrides, preferring passed value.")
+      @loader.default_branch_override = default_branch_override
+    end
 
-    validate_no_duplicate_names(@modules)
+    @loaded_content = @loader.load!
     @loaded = true
-  rescue SyntaxError, LoadError, ArgumentError, NameError => e
-    raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
+
+    @loaded_content
   end
 
   def loaded?
     @loaded
   end
 
-  # @param [Array<String>] modules
-  def validate_no_duplicate_names(modules)
-    dupes = modules
-            .group_by { |mod| mod.name }
-            .select { |_, v| v.size > 1 }
-            .map(&:first)
-    unless dupes.empty?
-      msg = _('Puppetfiles cannot contain duplicate module names.')
-      msg += ' '
-      msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
-      raise R10K::Error.new(msg)
-    end
+  def modules
+    @loaded_content[:modules]
   end
 
-  # @param [String] forge
-  def set_forge(forge)
-    @forge = forge
+  # @see R10K::ModuleLoader::Puppetfile#add_module for upcoming signature changes
+  def add_module(name, args)
+    @loader.add_module(name, args)
   end
 
-  # @param [String] moduledir
-  def set_moduledir(moduledir)
-    @moduledir = if Pathname.new(moduledir).absolute?
-      moduledir
-    else
-      File.join(basedir, moduledir)
-    end
+  def set_moduledir(dir)
+    @loader.set_moduledir(dir)
   end
 
-  # @param [String] name
-  # @param [Hash, String, Symbol] args Calling with anything but a Hash is
-  #   deprecated. The DSL will now convert String and Symbol versions to
-  #   Hashes of the shape
-  #     { version: <String or Symbol> }
-  #
-  def add_module(name, args)
-    if !args.is_a?(Hash)
-      args = { version: args }
-    end
-
-    args[:overrides] = @overrides
-
-    if install_path = args.delete(:install_path)
-      install_path = resolve_install_path(install_path)
-      validate_install_path(install_path, name)
-    else
-      install_path = @moduledir
-    end
-
-    if @default_branch_override != nil
-      args[:default_branch_override] = @default_branch_override
-    end
-
-
-    mod = R10K::Module.new(name, install_path, args, @environment)
-    mod.origin = :puppetfile
+  def set_forge(forge)
+    @loader.set_forge(forge)
+  end
 
-    # Do not load modules if they would conflict with the attached
-    # environment
-    if environment && environment.module_conflicts?(mod)
-      mod = nil
-      return @modules
-    end
+  def moduledir
+    @loader.moduledir
+  end
 
-    # Keep track of all the content this Puppetfile is managing to enable purging.
-    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
-    @managed_content[install_path] << mod.name
+  def puppetfile_path
+    @loader.puppetfile_path
+  end
 
-    @modules << mod
+  def environment=(env)
+    @loader.environment = env
+    @environment = env
   end
 
   include R10K::Util::Purgeable
 
   def managed_directories
-    self.load unless @loaded
+    self.load
 
-    dirs = @managed_content.keys
-    dirs.delete(real_basedir)
-    dirs
+    @loaded_content[:managed_directories]
   end
 
   # Returns an array of the full paths to all the content being managed.
   # @note This implements a required method for the Purgeable mixin
   # @return [Array<String>]
   def desired_contents
-    self.load unless @loaded
+    self.load
 
-    @managed_content.flat_map do |install_path, modnames|
-      modnames.collect { |name| File.join(install_path, name) }
-    end
+    @loaded_content[:desired_contents]
   end
 
   def purge_exclusions
-    exclusions = managed_directories
-
-    if environment && environment.respond_to?(:desired_contents)
-      exclusions += environment.desired_contents
-    end
+    self.load
 
-    exclusions
+    @loaded_content[:purge_exclusions]
   end
 
   def accept(visitor)
@@ -225,65 +202,10 @@ class Puppetfile
 
   private
 
-  def puppetfile_contents
-    File.read(@puppetfile_path)
-  end
-
-  def resolve_install_path(path)
-    pn = Pathname.new(path)
-
-    unless pn.absolute?
-      pn = Pathname.new(File.join(basedir, path))
-    end
-
-    # .cleanpath is as good as we can do without touching the filesystem.
-    # The .realpath methods will also choke if some of the intermediate
-    # paths are missing, even though we will create them later as needed.
-    pn.cleanpath.to_s
-  end
-
-  def validate_install_path(path, modname)
-    unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
-      raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
-    end
-
-    true
-  end
-
   def real_basedir
     Pathname.new(basedir).cleanpath.to_s
   end
 
-  class DSL
-    # A barebones implementation of the Puppetfile DSL
-    #
-    # @api private
-
-    def initialize(librarian)
-      @librarian = librarian
-    end
-
-    def mod(name, args = nil)
-      if args.is_a?(Hash)
-        opts = args
-      else
-        opts = { version: args }
-      end
-
-      @librarian.add_module(name, opts)
-    end
-
-    def forge(location)
-      @librarian.set_forge(location)
-    end
-
-    def moduledir(location)
-      @librarian.set_moduledir(location)
-    end
-
-    def method_missing(method, *args)
-      raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
-    end
-  end
+  DSL = R10K::ModuleLoader::Puppetfile::DSL
 end
 end

data/lib/r10k/settings/definition.rb

@@ -90,7 +90,7 @@ module R10K
       def resolve
         if !@value.nil?
           @value
-        elsif @default
+        elsif !@default.nil?
           if @default == :inherit
             # walk all the way up to root, starting with grandparent
             ancestor = parent