r10k 3.12.0 → 3.14.1

data/lib/r10k/git/cache.rb

@@ -3,6 +3,7 @@ require 'r10k/git'
 require 'r10k/settings'
 require 'r10k/instance_cache'
 require 'forwardable'
+require 'r10k/util/cacheable'
 
 # Cache Git repository mirrors for object database reuse.
 #
@@ -15,18 +16,9 @@ require 'forwardable'
 class R10K::Git::Cache
 
   include R10K::Settings::Mixin
+  include R10K::Util::Cacheable
 
-  #@api private
-  def self.determine_cache_root
-    if R10K::Util::Platform.windows?
-      File.join(ENV['LOCALAPPDATA'], 'r10k', 'git')
-    else
-      File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
-    end
-  end
-  private_class_method :determine_cache_root
-
-  def_setting_attr :cache_root, determine_cache_root
+  def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir('git')
 
   @instance_cache = R10K::InstanceCache.new(self)
 
@@ -109,8 +101,7 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= super(@remote)
   end
 end

data/lib/r10k/git/rugged/bare_repository.rb

@@ -64,7 +64,7 @@ class R10K::Git::Rugged::BareRepository < R10K::Git::Rugged::BaseRepository
     results = nil
 
     R10K::Git.with_proxy(proxy) do
-      results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) }
+      results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) }
     end
 
     report_transfer(results, remote_name)

data/lib/r10k/git/rugged/base_repository.rb

@@ -20,7 +20,8 @@ class R10K::Git::Rugged::BaseRepository
     else
       object.oid
     end
-  rescue ::Rugged::ReferenceError
+  rescue ::Rugged::ReferenceError, ::Rugged::OdbError => e
+    logger.debug2(_("Unable to resolve %{pattern}: %{e} ") % {pattern: pattern, e: e })
     nil
   end
 
@@ -60,6 +61,16 @@ class R10K::Git::Rugged::BaseRepository
     remotes_hash
   end
 
+  # Update a remote URL
+  # @param [String] The remote URL of the git repository
+  # @param [String] An optional remote name for the git repository
+  def update_remote(remote, remote_name='origin')
+    if @_rugged_repo
+      logger.debug2(_("Remote URL is different from cache, updating %{orig} to %{update}") % {orig: remotes[remote_name], update: remote})
+      @_rugged_repo.remotes.set_url(remote_name, remote)
+    end
+  end
+
   private
 
   def with_repo(opts={})

data/lib/r10k/git/rugged/cache.rb

@@ -8,4 +8,12 @@ class R10K::Git::Rugged::Cache < R10K::Git::Cache
   def self.bare_repository
     R10K::Git::Rugged::BareRepository
   end
+
+  # Update the remote URL if the cache differs from the current configuration
+  def sync!
+    if cached? && @repo.remotes['origin'] != @remote
+      @repo.update_remote(@remote)
+    end
+    super
+  end
 end

data/lib/r10k/git/rugged/credentials.rb

@@ -130,7 +130,7 @@ class R10K::Git::Rugged::Credentials
 
     user = nil
 
-    if !username_from_url.nil?
+    if !username_from_url.nil? && !username_from_url.empty?
       user = username_from_url
       logger.debug2 _("URL %{url} includes the username %{username}, using that user for authentication.") % {url: url.inspect, username: username_from_url}
     elsif git_user

data/lib/r10k/git/rugged/working_repository.rb

@@ -93,7 +93,7 @@ class R10K::Git::Rugged::WorkingRepository < R10K::Git::Rugged::BaseRepository
     results = nil
 
     R10K::Git.with_proxy(proxy) do
-      results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) }
+      results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) }
     end
 
     report_transfer(results, remote)

data/lib/r10k/git/stateful_repository.rb

@@ -86,6 +86,8 @@ class R10K::Git::StatefulRepository
       :mismatched
     elsif !(@repo.origin == @remote)
       :mismatched
+    elsif @repo.head.nil?
+      :mismatched
     elsif @repo.dirty?
       :dirty
     elsif !(@repo.head == @cache.resolve(ref))

data/lib/r10k/initializers.rb

@@ -4,6 +4,7 @@ require 'r10k/git'
 require 'r10k/git/cache'
 
 require 'r10k/forge/module_release'
+require 'r10k/tarball'
 
 module R10K
   module Initializers
@@ -30,14 +31,27 @@ module R10K
           logger.warn(_("the purgedirs key in r10k.yaml is deprecated. it is currently ignored."))
         end
 
+        with_setting(:logging) { |value| LoggingInitializer.new(value).call }
+
         with_setting(:deploy) { |value| DeployInitializer.new(value).call }
 
         with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value }
         with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value }
+        with_setting(:cachedir) { |value| R10K::Tarball.settings[:cache_root] = value }
         with_setting(:pool_size) { |value| R10K::Puppetfile.settings[:pool_size] = value }
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
 
         with_setting(:git) { |value| GitInitializer.new(value).call }
         with_setting(:forge) { |value| ForgeInitializer.new(value).call }
+        with_setting(:tarball) { |value| TarballInitializer.new(value).call }
+      end
+    end
+
+    class LoggingInitializer < BaseInitializer
+      def call
+        with_setting(:level) { |value| R10K::Logging.level = value }
+        with_setting(:disable_default_stderr) { |value| R10K::Logging.disable_default_stderr = value }
+        with_setting(:outputs) { |value| R10K::Logging.add_outputters(value) }
       end
     end
 
@@ -69,5 +83,11 @@ module R10K
         with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value }
       end
     end
+
+    class TarballInitializer < BaseInitializer
+      def call
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
+      end
+    end
   end
 end

data/lib/r10k/logging.rb

@@ -8,6 +8,16 @@ require 'r10k/logging/terminaloutputter'
 module R10K::Logging
 
   LOG_LEVELS = %w{DEBUG2 DEBUG1 DEBUG INFO NOTICE WARN ERROR FATAL}
+  SYSLOG_LEVELS_MAP = {
+    'DEBUG2' => 'DEBUG',
+    'DEBUG1' => 'DEBUG',
+    'DEBUG' => 'DEBUG',
+    'INFO' => 'INFO',
+    'NOTICE' => 'INFO',
+    'WARN' => 'WARN',
+    'ERROR' => 'ERROR',
+    'FATAL' => 'FATAL',
+  }.freeze
 
   def logger_name
     self.class.to_s
@@ -21,6 +31,9 @@ module R10K::Logging
       else
         @logger = Log4r::Logger.new(name)
         @logger.add(R10K::Logging.outputter)
+        R10K::Logging.outputters.each do |output|
+          @logger.add(output)
+        end
       end
     end
     @logger
@@ -59,7 +72,7 @@ module R10K::Logging
       if level.nil?
         raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % {val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect}
       end
-      outputter.level = level
+      outputter.level = level unless @disable_default_stderr
       @level = level
 
       if level < Log4r::INFO
@@ -69,6 +82,58 @@ module R10K::Logging
       end
     end
 
+    def disable_default_stderr=(val)
+      @disable_default_stderr = val
+      outputter.level = val ? Log4r::OFF : @level
+    end
+
+    def add_outputters(outputs)
+      outputs.each do |output|
+        type = output.fetch(:type)
+        # Support specifying both short as well as full names
+        type = type.to_s[0..-10] if type.to_s.downcase.end_with? 'outputter'
+
+        name = output.fetch(:name, 'r10k')
+        if output[:level]
+          level = parse_level(output[:level])
+          if level.nil?
+            raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: output[:level], log_levels: LOG_LEVELS.map(&:downcase).inspect }
+          end
+        else
+          level = self.level
+        end
+        only_at = output[:only_at]
+        only_at&.map! do |val|
+          lv = parse_level(val)
+          if lv.nil?
+            raise ArgumentError, _("Invalid log level '%{val}'. Valid levels are %{log_levels}") % { val: val, log_levels: LOG_LEVELS.map(&:downcase).inspect }
+          end
+
+          lv
+        end
+        parameters = output.fetch(:parameters, {}).merge({ level: level })
+
+        begin
+          # Try to load the outputter file if possible
+          require "log4r/outputter/#{type.to_s.downcase}outputter"
+        rescue LoadError
+          false
+        end
+        outputtertype = Log4r.constants
+                             .select { |klass| klass.to_s.end_with? 'Outputter' }
+                             .find { |klass| klass.to_s.downcase == "#{type.to_s.downcase}outputter" }
+        raise ArgumentError, "Unable to find a #{output[:type]} outputter." unless outputtertype
+
+        outputter = Log4r.const_get(outputtertype).new(name, parameters)
+        outputter.only_at(*only_at) if only_at
+        # Handle log4r's syslog mapping correctly
+        outputter.map_levels_by_name_to_syslog(SYSLOG_LEVELS_MAP) if outputter.respond_to? :map_levels_by_name_to_syslog
+
+        @outputters << outputter
+        Log4r::Logger.global.add outputter
+      end
+    end
+
     extend Forwardable
     def_delegators :@outputter, :use_color, :use_color=
 
@@ -87,6 +152,16 @@ module R10K::Logging
     #   @return [Log4r::Outputter]
     attr_reader :outputter
 
+    # @!attribute [r] outputters
+    #   @api private
+    #   @return [Array[Log4r::Outputter]]
+    attr_reader :outputters
+
+    # @!attribute [r] disable_default_stderr
+    #   @api private
+    #   @return [Boolean]
+    attr_reader :disable_default_stderr
+
     def default_formatter
       Log4r::PatternFormatter.new(:pattern => '%l\t -> %m')
     end
@@ -106,4 +181,6 @@ module R10K::Logging
   @level     = Log4r::WARN
   @formatter = default_formatter
   @outputter = default_outputter
+  @outputters = []
+  @disable_default_stderr = false
 end

data/lib/r10k/module/base.rb

@@ -60,8 +60,12 @@ class R10K::Module::Base
     @environment = environment
     @overrides = args.delete(:overrides) || {}
     @spec_deletable = true
-    @exclude_spec = args.delete(:exclude_spec)
-    @exclude_spec = @overrides[:modules].delete(:exclude_spec) if @overrides.dig(:modules, :exclude_spec)
+    @exclude_spec = false
+    @exclude_spec = @overrides.dig(:modules, :exclude_spec) if @overrides.dig(:modules, :exclude_spec)
+    if args.has_key?(:exclude_spec)
+      logger.debug2 _("Overriding :exclude_spec setting with per module setting for #{@title}")
+      @exclude_spec = args.delete(:exclude_spec)
+    end
     @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
 
     @requested_modules = @overrides.dig(:modules, :requested_modules) || []

data/lib/r10k/module/forge.rb

@@ -13,16 +13,7 @@ class R10K::Module::Forge < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args[:type].to_s == 'forge' ||
-      (!!
-       ((args.keys & %i{git svn type}).empty? &&
-        args.has_key?(:version) &&
-        name.match(%r[\w+[/-]\w+]) &&
-        valid_version?(args[:version])))
-  end
-
-  def self.valid_version?(expected_version)
-    expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
+    args[:type].to_s == 'forge'
   end
 
   def self.statically_defined_version(name, args)
@@ -54,11 +45,20 @@ class R10K::Module::Forge < R10K::Module::Base
       :type    => ::R10K::Util::Setopts::Ignore,
     }, :raise_on_unhandled => false)
 
+    # Validate version and raise on issue. Title is validated by base class.
+    unless valid_version?(@expected_version)
+      raise ArgumentError, _("Module version %{ver} is not a valid Forge module version") % {ver: @expected_version}
+    end
+
     @expected_version ||= current_version || :latest
 
     @v3_module = PuppetForge::V3::Module.new(:slug => @title)
   end
 
+  def valid_version?(version)
+    version == :latest || version.nil? || PuppetForge::Util.version_valid?(version)
+  end
+
   # @param [Hash] opts Deprecated
   # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})

data/lib/r10k/module/git.rb

@@ -9,8 +9,6 @@ class R10K::Module::Git < R10K::Module::Base
 
   def self.implement?(name, args)
     args.has_key?(:git) || args[:type].to_s == 'git'
-  rescue
-    false
   end
 
   # Will be called if self.implement? above returns true. Will return
@@ -69,7 +67,7 @@ class R10K::Module::Git < R10K::Module::Base
       :default_branch_override => :default_override_ref,
     }, :raise_on_unhandled => false)
 
-    force = @overrides.dig(:modules, :force)
+    force = @overrides[:force]
     @force = force == false ? false : true
 
     @desired_ref ||= 'master'

data/lib/r10k/module/local.rb

@@ -8,13 +8,17 @@ class R10K::Module::Local < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a?(Hash) && (args[:local] || args[:type].to_s == 'local')
+    args[:local] || args[:type].to_s == 'local'
   end
 
-  def version
+  def self.statically_defined_version(*)
     "0.0.0"
   end
 
+  def version
+    self.class.statically_defined_version
+  end
+
   def properties
     {
       :expected => "0.0.0 (local)",

data/lib/r10k/module/tarball.rb

@@ -0,0 +1,101 @@
+require 'r10k/module'
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+
+# This class defines a tarball source module implementation
+class R10K::Module::Tarball < R10K::Module::Base
+
+  R10K::Module.register(self)
+
+  def self.implement?(name, args)
+    args.is_a?(Hash) && args[:type].to_s == 'tarball'
+  rescue
+    false
+  end
+
+  def self.statically_defined_version(name, args)
+    args[:version] || args[:checksum]
+  end
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  def initialize(name, dirname, opts, environment=nil)
+    super
+    setopts(opts, {
+      # Standard option interface
+      :source    => :self,
+      :version   => :checksum,
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :overrides => :self,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  # Return the status of the currently installed module.
+  #
+  # @return [Symbol]
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  # Synchronize this module with the indicated state.
+  # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
+  def sync(opts={})
+    tarball.get unless tarball.cache_valid?
+    if should_sync?
+      case status
+      when :absent
+        tarball.unpack(path.to_s)
+      when :mismatched
+        path.rmtree
+        tarball.unpack(path.to_s)
+      end
+      maybe_delete_spec_dir
+      true
+    else
+      false
+    end
+  end
+
+  # Return the desired version of this module
+  def version
+    @checksum || '(present)'
+  end
+
+  # Return the properties of the module
+  #
+  # @return [Hash]
+  # @abstract
+  def properties
+    {
+      :expected => version,
+      :actual   => ((state = status) == :insync) ? version : state,
+      :type     => :tarball,
+    }
+  end
+
+  # Tarball caches are files, not directories. An important purpose of this
+  # method is to indicate where the cache "path" is, for locking/parallelism,
+  # so for the Tarball module type, the relevant path location is returned.
+  #
+  # @return [String] The path this module will cache its tarball source to
+  def cachedir
+    tarball.cache_path
+  end
+end

data/lib/r10k/module.rb

@@ -53,4 +53,5 @@ module R10K::Module
   require 'r10k/module/local'
   require 'r10k/module/forge'
   require 'r10k/module/definition'
+  require 'r10k/module/tarball'
 end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -15,7 +15,7 @@ module R10K
           if args.is_a?(Hash)
             opts = args
           else
-            opts = { version: args }
+            opts = { type: 'forge', version: args }
           end
 
           if @metadata_only

data/lib/r10k/module_loader/puppetfile.rb

@@ -1,5 +1,9 @@
+require 'r10k/errors'
 require 'r10k/logging'
 require 'r10k/module'
+require 'r10k/module_loader/puppetfile/dsl'
+
+require 'pathname'
 
 module R10K
   module ModuleLoader
@@ -12,7 +16,8 @@ module R10K
 
       attr_accessor :default_branch_override, :environment
       attr_reader :modules, :moduledir, :puppetfile_path,
-        :managed_directories, :desired_contents, :purge_exclusions
+        :managed_directories, :desired_contents, :purge_exclusions,
+        :environment_name
 
       # @param basedir [String] The path that contains the moduledir &
       #     Puppetfile by default. May be an environment, project, or
@@ -36,6 +41,7 @@ module R10K
         @puppetfile_path  = resolve_path(@basedir, puppetfile)
         @overrides   = overrides
         @environment = environment
+        @environment_name = @environment&.name
         @default_branch_override = @overrides.dig(:environments, :default_branch_override)
         @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
 
@@ -161,6 +167,13 @@ module R10K
         @modules << mod
       end
 
+      # @deprecated
+      # @return [String] The base directory that contains the Puppetfile
+      def basedir
+        logger.warn _('"basedir" is deprecated. Please use "environment_name" instead. "basedir" will be removed in a future version.')
+        @basedir
+      end
+
      private
 
       def empty_load_output
@@ -183,8 +196,10 @@ module R10K
       end
 
       def parse_module_definition(name, info)
+        # The only valid (deprecated) way a module can be defined with a
+        # non-hash info is if it is a Forge module.
         if !info.is_a?(Hash)
-          info = { version: info }
+          info = { type: 'forge', version: info }
         end
 
         info[:overrides] = @overrides

data/lib/r10k/settings.rb

@@ -217,6 +217,39 @@ module R10K
         })])
     end
 
+    def self.logging_settings
+      R10K::Settings::Collection.new(:logging, [
+        Definition.new(:level, {
+          desc: 'What logging level should R10k run on if not specified at runtime.',
+          validate: lambda do |value|
+            if R10K::Logging.parse_level(value).nil?
+              raise ArgumentError, "`level` must be a valid log level.
+                                    Valid levels are #{R10K::Logging::LOG_LEVELS.map(&:downcase).inspect}"
+            end
+          end
+        }),
+
+        Definition.new(:outputs, {
+          desc: 'Additional log outputs to use.',
+          validate: lambda do |value|
+            unless value.is_a?(Array)
+              raise ArgumentError, "The `outputs` setting should be an array of outputs, not a #{value.class}"
+            end
+          end
+        }),
+
+        Definition.new(:disable_default_stderr, {
+          desc: 'Disable the default stderr logging output',
+          default: false,
+          validate: lambda do |value|
+            unless !!value == value
+              raise ArgumentError, "`disable_default_stderr` can only be a boolean value, not '#{value}'"
+            end
+          end
+        })
+      ])
+    end
+
     def self.global_settings
       R10K::Settings::Collection.new(:global, [
         Definition.new(:sources, {
@@ -271,6 +304,8 @@ module R10K
         R10K::Settings.git_settings,
 
         R10K::Settings.deploy_settings,
+
+        R10K::Settings.logging_settings
       ])
     end
   end

data/lib/r10k/source/git.rb

@@ -100,26 +100,26 @@ class R10K::Source::Git < R10K::Source::Base
 
   def generate_environments
     envs = []
-    branch_names.each do |bn|
-      if bn.valid?
-        envs << R10K::Environment::Git.new(bn.name,
+    environment_names.each do |en|
+      if en.valid?
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.correct?
-       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname}
-        envs << R10K::Environment::Git.new(bn.name,
+      elsif en.correct?
+       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: en.name.inspect, corrected_env_name: en.dirname}
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.validate?
-       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect}
+      elsif en.validate?
+       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: en.name.inspect}
       end
     end
 
@@ -157,22 +157,22 @@ class R10K::Source::Git < R10K::Source::Base
 
   private
 
-  def branch_names
+  def environment_names
     opts = {prefix: @prefix,
             invalid: @invalid_branches,
             source: @name,
             strip_component: @strip_component}
-    branches = @cache.branches
+    branch_names = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+      branch_names = filter_branches_by_regexp(branch_names, @ignore_branch_prefixes)
     end
 
     if @filter_command && !@filter_command.empty?
-      branches = filter_branches_by_command(branches, @filter_command)
+      branch_names = filter_branches_by_command(branch_names, @filter_command)
     end
 
-    branches.map do |branch|
-      R10K::Environment::Name.new(branch, opts)
+    branch_names.map do |branch_name|
+      R10K::Environment::Name.new(branch_name, opts)
     end
   end
 end

data/lib/r10k/source/yaml.rb

@@ -7,7 +7,7 @@ class R10K::Source::Yaml < R10K::Source::Hash
     begin
       contents = ::YAML.load_file(config)
     rescue => e
-      raise ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
+      raise R10K::ConfigError, _("Couldn't open environments file %{file}: %{err}") % {file: config, err: e.message}
     end
 
     # Set the environments key for the parent class to consume