r10k 3.10.0 → 3.13.0

data/lib/r10k/module.rb

@@ -22,17 +22,35 @@ module R10K::Module
   #
   # @return [Object < R10K::Module] A member of the implementing subclass
   def self.new(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      implementation.new(name, basedir, args, environment)
+    end
+  end
+
+  # Takes the same signature as Module.new but returns an metadata module
+  def self.from_metadata(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      R10K::Module::Definition.new(name,
+                                   dirname: basedir,
+                                   args: args,
+                                   implementation: implementation,
+                                   environment: environment)
+    end
+  end
+
+  def self.with_implementation(name, args, &block)
     if implementation = @klasses.find { |klass| klass.implement?(name, args) }
-      obj = implementation.new(name, basedir, args, environment)
-      obj
+      block.call(implementation)
     else
       raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect}
     end
   end
 
+
   require 'r10k/module/base'
   require 'r10k/module/git'
   require 'r10k/module/svn'
   require 'r10k/module/local'
   require 'r10k/module/forge'
+  require 'r10k/module/definition'
 end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -6,8 +6,9 @@ module R10K
         #
         # @api private
 
-        def initialize(librarian)
-          @librarian = librarian
+        def initialize(librarian, metadata_only: false)
+          @librarian     = librarian
+          @metadata_only = metadata_only
         end
 
         def mod(name, args = nil)
@@ -17,7 +18,11 @@ module R10K
             opts = { version: args }
           end
 
-          @librarian.add_module(name, opts)
+          if @metadata_only
+            @librarian.add_module_metadata(name, opts)
+          else
+            @librarian.add_module(name, opts)
+          end
         end
 
         def forge(location)

data/lib/r10k/module_loader/puppetfile.rb

@@ -1,13 +1,21 @@
+require 'r10k/errors'
+require 'r10k/logging'
+require 'r10k/module'
+require 'r10k/module_loader/puppetfile/dsl'
+
+require 'pathname'
+
 module R10K
   module ModuleLoader
     class Puppetfile
 
+      include R10K::Logging
+
       DEFAULT_MODULEDIR = 'modules'
       DEFAULT_PUPPETFILE_NAME = 'Puppetfile'
-      DEFAULT_FORGE_API = 'forgeapi.puppetlabs.com'
 
       attr_accessor :default_branch_override, :environment
-      attr_reader :modules, :moduledir,
+      attr_reader :modules, :moduledir, :puppetfile_path,
         :managed_directories, :desired_contents, :purge_exclusions
 
       # @param basedir [String] The path that contains the moduledir &
@@ -24,18 +32,19 @@ module R10K
       def initialize(basedir:,
                      moduledir: DEFAULT_MODULEDIR,
                      puppetfile: DEFAULT_PUPPETFILE_NAME,
-                     forge: DEFAULT_FORGE_API,
                      overrides: {},
                      environment: nil)
 
         @basedir     = cleanpath(basedir)
         @moduledir   = resolve_path(@basedir, moduledir)
-        @puppetfile  = resolve_path(@basedir, puppetfile)
-        @forge       = forge
+        @puppetfile_path  = resolve_path(@basedir, puppetfile)
         @overrides   = overrides
         @environment = environment
         @default_branch_override = @overrides.dig(:environments, :default_branch_override)
+        @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
 
+        @existing_module_metadata = []
+        @existing_module_versions_by_name = {}
         @modules = []
 
         @managed_directories = []
@@ -44,11 +53,19 @@ module R10K
       end
 
       def load
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load!
+        end
+      end
+
+      def load!
+        logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
+        logger.debug _("Using moduledir '%{moduledir}'") % {moduledir: @moduledir}
+
         dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self)
-        dsl.instance_eval(puppetfile_content(@puppetfile), @puppetfile)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
 
         validate_no_duplicate_names(@modules)
-        @modules
 
         managed_content = @modules.group_by(&:dirname)
 
@@ -64,9 +81,33 @@ module R10K
         }
 
       rescue SyntaxError, LoadError, ArgumentError, NameError => e
-        raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile})
+        raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
+      end
+
+      def load_metadata
+        with_readable_puppetfile(@puppetfile_path) do
+          self.load_metadata!
+        end
+      end
+
+      def load_metadata!
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self, metadata_only: true)
+        dsl.instance_eval(puppetfile_content(@puppetfile_path), @puppetfile_path)
+
+        @existing_module_versions_by_name = @existing_module_metadata.map {|mod| [ mod.name, mod.version ] }.to_h
+        empty_load_output.merge(modules: @existing_module_metadata)
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        logger.warn _("Unable to preload Puppetfile because of %{msg}" % { msg: e.message })
       end
 
+      def add_module_metadata(name, info)
+        install_path, metadata_info, _ = parse_module_definition(name, info)
+
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
+
+        @existing_module_metadata << mod
+      end
 
       ##
       ## set_forge, set_moduledir, and add_module are used directly by the DSL class
@@ -74,7 +115,12 @@ module R10K
 
       # @param [String] forge
       def set_forge(forge)
-        @forge = forge
+        if @allow_puppetfile_forge
+          logger.debug _("Using Forge from Puppetfile: %{forge}") % { forge: forge }
+          PuppetForge.host = forge
+        else
+          logger.debug _("Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}.") % { forge: PuppetForge.host }
+        end
       end
 
       # @param [String] moduledir
@@ -83,7 +129,7 @@ module R10K
       end
 
       # @param [String] name
-      # @param [Hash, String, Symbol, nil] module_info Calling with
+      # @param [Hash, String, Symbol, nil] info Calling with
       #   anything but a Hash is deprecated. The DSL will now convert
       #   String and Symbol versions to Hashes of the shape
       #     { version: <String or Symbol> }
@@ -96,26 +142,12 @@ module R10K
       #   DSL class, not the Puppetfile author) to do this conversion
       #   itself.
       #
-      def add_module(name, module_info)
-        if !module_info.is_a?(Hash)
-          module_info = { version: module_info }
-        end
-
-        module_info[:overrides] = @overrides
-
-        if install_path = module_info.delete(:install_path)
-          install_path = resolve_path(@basedir, install_path)
-          validate_install_path(install_path, name)
-        else
-          install_path = @moduledir
-        end
+      def add_module(name, info)
+        install_path, metadata_info, spec_deletable = parse_module_definition(name, info)
 
-        if @default_branch_override
-          module_info[:default_branch_override] = @default_branch_override
-        end
-
-        mod = R10K::Module.new(name, install_path, module_info, @environment)
+        mod = R10K::Module.from_metadata(name, install_path, metadata_info, @environment)
         mod.origin = :puppetfile
+        mod.spec_deletable = spec_deletable
 
         # Do not save modules if they would conflict with the attached
         # environment
@@ -123,11 +155,60 @@ module R10K
           return @modules
         end
 
+        # If this module's metadata has a static version and that version
+        # matches the existing module declaration use it, otherwise create
+        # a regular module to sync.
+        unless mod.version && (mod.version == @existing_module_versions_by_name[mod.name])
+          mod = mod.to_implementation
+        end
+
         @modules << mod
       end
 
      private
 
+      def empty_load_output
+        {
+          modules: [],
+          managed_directories: [],
+          desired_contents: [],
+          purge_exclusions: []
+        }
+      end
+
+      def with_readable_puppetfile(puppetfile_path, &block)
+        if File.readable?(puppetfile_path)
+          block.call
+        else
+          logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
+
+          empty_load_output
+        end
+      end
+
+      def parse_module_definition(name, info)
+        if !info.is_a?(Hash)
+          info = { version: info }
+        end
+
+        info[:overrides] = @overrides
+
+        if @default_branch_override
+          info[:default_branch_override] = @default_branch_override
+        end
+
+        spec_deletable = false
+        if install_path = info.delete(:install_path)
+          install_path = resolve_path(@basedir, install_path)
+          validate_install_path(install_path, name)
+        else
+          install_path = @moduledir
+          spec_deletable = true
+        end
+
+        return [ install_path, info, spec_deletable ]
+      end
+
       # @param [Array<R10K::Module>] modules
       def validate_no_duplicate_names(modules)
         dupes = modules

data/lib/r10k/puppetfile.rb

@@ -30,10 +30,6 @@ class Puppetfile
   #   @return [String] The base directory that contains the Puppetfile
   attr_reader :basedir
 
-  # @!attrbute [r] puppetfile_path
-  #   @return [String] The path to the Puppetfile
-  attr_reader :puppetfile_path
-
   # @!attribute [r] environment
   #   @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to.
   attr_reader :environment
@@ -68,22 +64,20 @@ class Puppetfile
 
     @force           = deprecated_force_arg     || options.delete(:force)           || false
     @moduledir       = deprecated_moduledir_arg || options.delete(:moduledir)       || File.join(basedir, 'modules')
-    @puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
-    @puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
+    puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
+    puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path)
+    @puppetfile = puppetfile_path || puppetfile_name
     @environment     = options.delete(:environment)
 
     @overrides       = options.delete(:overrides) || {}
     @default_branch_override = @overrides.dig(:environments, :default_branch_override)
 
-    logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
-
     @forge   = 'forgeapi.puppetlabs.com'
 
     @loader = ::R10K::ModuleLoader::Puppetfile.new(
       basedir: @basedir,
       moduledir: @moduledir,
-      puppetfile: @puppetfile_path,
-      forge: @forge,
+      puppetfile: @puppetfile,
       overrides: @overrides,
       environment: @environment
     )
@@ -106,8 +100,8 @@ class Puppetfile
     if self.loaded?
       return @loaded_content
     else
-      if !File.readable?(@puppetfile_path)
-        logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
+      if !File.readable?(puppetfile_path)
+        logger.debug _("Puppetfile %{path} missing or unreadable") % {path: puppetfile_path.inspect}
       else
         self.load!(default_branch_override)
       end
@@ -125,7 +119,7 @@ class Puppetfile
       @loader.default_branch_override = default_branch_override
     end
 
-    @loaded_content = @loader.load
+    @loaded_content = @loader.load!
     @loaded = true
 
     @loaded_content
@@ -156,6 +150,10 @@ class Puppetfile
     @loader.moduledir
   end
 
+  def puppetfile_path
+    @loader.puppetfile_path
+  end
+
   def environment=(env)
     @loader.environment = env
     @environment = env

data/lib/r10k/settings/definition.rb

@@ -90,7 +90,7 @@ module R10K
       def resolve
         if !@value.nil?
           @value
-        elsif @default
+        elsif !@default.nil?
           if @default == :inherit
             # walk all the way up to root, starting with grandparent
             ancestor = parent

data/lib/r10k/settings.rb

@@ -42,6 +42,22 @@ module R10K
                     Only used by the 'rugged' Git provider."
         }),
 
+        Definition.new(:github_app_id, {
+          :desc => "The Github App id for Git SSL remotes.
+                    Only used by the 'rugged' Git provider."
+        }),
+
+        Definition.new(:github_app_key, {
+          :desc => "The Github App private key for Git SSL remotes.
+                    Only used by the 'rugged' Git provider."
+        }),
+
+        Definition.new(:github_app_ttl, {
+          :desc => "The ttl expiration for SSL tokens.
+                    Only used by the 'rugged' Git provider.",
+          :default => "120",
+        }),
+
         URIDefinition.new(:proxy, {
           :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
           :default => :inherit,
@@ -65,6 +81,24 @@ module R10K
               :default => :inherit
             }),
 
+            Definition.new(:github_app_id, {
+              :desc => "The Github App id for Git SSL remotes.
+                        Only used by the 'rugged' Git provider.",
+              :default => :inherit
+            }),
+
+            Definition.new(:github_app_key, {
+              :desc => "The Github App private key for Git SSL remotes.
+                        Only used by the 'rugged' Git provider.",
+              :default => :inherit
+            }),
+
+            Definition.new(:github_app_ttl, {
+              :desc => "The ttl expiration for Git SSL tokens.
+                        Only used by the 'rugged' Git provider.",
+              :default => :inherit
+            }),
+
             URIDefinition.new(:proxy, {
               :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
               :default => :inherit,
@@ -92,8 +126,19 @@ module R10K
         URIDefinition.new(:baseurl, {
           :desc => "The URL to the Puppet Forge to use for downloading modules."
         }),
+
         Definition.new(:authorization_token, {
           :desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections."
+        }),
+
+        Definition.new(:allow_puppetfile_override, {
+          :desc => "Whether to use `forge` declarations in the Puppetfile as an override of `baseurl`.",
+          :default => false,
+          :validate => lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`allow_puppetfile_override` can only be a boolean value, not '#{value}'"
+            end
+          end
         })
       ])
     end
@@ -161,6 +206,47 @@ module R10K
             end
           end
         }),
+        Definition.new(:exclude_spec, {
+          :desc => "Whether or not to deploy the spec dir of a module. Defaults to false.",
+          :default => false,
+          :validate => lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`exclude_spec` can only be a boolean value, not '#{value}'"
+            end
+          end
+        })])
+    end
+
+    def self.logging_settings
+      R10K::Settings::Collection.new(:logging, [
+        Definition.new(:level, {
+          desc: 'What logging level should R10k run on if not specified at runtime.',
+          validate: lambda do |value|
+            if R10K::Logging.parse_level(value).nil?
+              raise ArgumentError, "`level` must be a valid log level.
+                                    Valid levels are #{R10K::Logging::LOG_LEVELS.map(&:downcase).inspect}"
+            end
+          end
+        }),
+
+        Definition.new(:outputs, {
+          desc: 'Additional log outputs to use.',
+          validate: lambda do |value|
+            unless value.is_a?(Array)
+              raise ArgumentError, "The `outputs` setting should be an array of outputs, not a #{value.class}"
+            end
+          end
+        }),
+
+        Definition.new(:disable_default_stderr, {
+          desc: 'Disable the default stderr logging output',
+          default: false,
+          validate: lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`disable_default_stderr` can only be a boolean value, not '#{value}'"
+            end
+          end
+        })
       ])
     end
 
@@ -180,7 +266,7 @@ module R10K
         }),
 
         Definition.new(:postrun, {
-          :desc => "The command r10k should run after deploying environments.",
+          :desc => "The command r10k should run after deploying environments or modules.",
           :validate => lambda do |value|
             if !value.is_a?(Array)
               raise ArgumentError, "The postrun setting should be an array of strings, not a #{value.class}"
@@ -218,6 +304,8 @@ module R10K
         R10K::Settings.git_settings,
 
         R10K::Settings.deploy_settings,
+
+        R10K::Settings.logging_settings
       ])
     end
   end

data/lib/r10k/source/yaml.rb

@@ -7,7 +7,7 @@ class R10K::Source::Yaml < R10K::Source::Hash
     begin
       contents = ::YAML.load_file(config)
     rescue => e
-      raise ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
+      raise R10K::ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
     end
 
     # Set the environments key for the parent class to consume

data/lib/r10k/util/purgeable.rb

@@ -1,3 +1,5 @@
+require 'r10k/logging'
+
 require 'fileutils'
 
 module R10K
@@ -9,6 +11,8 @@ module R10K
     #   {#desired_contents}
     module Purgeable
 
+      include R10K::Logging
+
       HIDDEN_FILE = /\.[^.]+/
 
       FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH
@@ -95,8 +99,8 @@ module R10K
         end
 
         children.flat_map do |child|
-          if File.directory?(child) && recurse
-            potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
+          if File.directory?(child) && !File.symlink?(child) && recurse
+            potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) << child.to_s
           else
             child.to_s
           end

data/lib/r10k/util/setopts.rb

@@ -1,3 +1,5 @@
+require 'r10k/logging'
+
 module R10K
   module Util
 

data/lib/r10k/util/subprocess.rb

@@ -1,3 +1,4 @@
+require 'r10k/logging'
 require 'r10k/util/platform'
 
 module R10K

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.10.0'
+  VERSION = '3.13.0'
 end