r10k 3.9.0 → 3.9.1

data/lib/r10k/module/git.rb

@@ -8,7 +8,7 @@ class R10K::Module::Git < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a?(Hash) && (args.has_key?(:git) || args[:type].to_s == 'git')
+    args.has_key?(:git) || args[:type].to_s == 'git'
   rescue
     false
   end
@@ -36,27 +36,35 @@ class R10K::Module::Git < R10K::Module::Base
   include R10K::Util::Setopts
 
   def initialize(title, dirname, opts, environment=nil)
+
     super
     setopts(opts, {
       # Standard option interface
-      :version => :desired_ref,
-      :source  => :remote,
-      :type    => ::R10K::Util::Setopts::Ignore,
+      :version                 => :desired_ref,
+      :source                  => :remote,
+      :type                    => ::R10K::Util::Setopts::Ignore,
 
       # Type-specific options
-      :branch  => :desired_ref,
-      :tag     => :desired_ref,
-      :commit  => :desired_ref,
-      :ref     => :desired_ref,
-      :git     => :remote,
+      :branch                  => :desired_ref,
+      :tag                     => :desired_ref,
+      :commit                  => :desired_ref,
+      :ref                     => :desired_ref,
+      :git                     => :remote,
       :default_branch          => :default_ref,
       :default_branch_override => :default_override_ref,
     })
 
+    force = @overrides.dig(:modules, :force)
+    @force = force == false ? false : true
+
     @desired_ref ||= 'master'
 
-    if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref)
-      @desired_ref = @environment.ref
+    if @desired_ref == :control_branch
+      if @environment && @environment.respond_to?(:ref)
+        @desired_ref = @environment.ref
+      else
+        logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." % {name: name})
+      end
     end
 
     @repo = R10K::Git::StatefulRepository.new(@remote, @dirname, @name)
@@ -74,9 +82,10 @@ class R10K::Module::Git < R10K::Module::Base
     }
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
-    force = opts && opts.fetch(:force, true)
-    @repo.sync(version, force)
+    force = opts[:force] || @force
+    @repo.sync(version, force) if should_sync?
   end
 
   def status

data/lib/r10k/module/local.rb

@@ -30,6 +30,7 @@ class R10K::Module::Local < R10K::Module::Base
     :insync
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
     logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title}
   end

data/lib/r10k/module/svn.rb

@@ -7,7 +7,7 @@ class R10K::Module::SVN < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a?(Hash) && (args.has_key?(:svn) || args[:type].to_s == 'svn')
+    args.has_key?(:svn) || args[:type].to_s == 'svn'
   end
 
   # @!attribute [r] expected_revision
@@ -69,14 +69,17 @@ class R10K::Module::SVN < R10K::Module::Base
     end
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
-    case status
-    when :absent
-      install
-    when :mismatched
-      reinstall
-    when :outdated
-      update
+    if should_sync?
+      case status
+      when :absent
+        install
+      when :mismatched
+        reinstall
+      when :outdated
+        update
+      end
     end
   end
 

data/lib/r10k/puppetfile.rb

@@ -3,6 +3,7 @@ require 'pathname'
 require 'r10k/module'
 require 'r10k/util/purgeable'
 require 'r10k/errors'
+require 'r10k/content_synchronizer'
 
 module R10K
 class Puppetfile
@@ -42,17 +43,32 @@ class Puppetfile
   #   @return [Boolean] Overwrite any locally made changes
   attr_accessor :force
 
+  # @!attribute [r] overrides
+  #   @return [Hash] Various settings overridden from normal configs
+  attr_reader :overrides
+
   # @param [String] basedir
-  # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules
-  # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile
-  # @param [String] puppetfile_name The name of the Puppetfile, default to 'Puppetfile'
-  # @param [Boolean] force Shall we overwrite locally made changes?
-  def initialize(basedir, moduledir = nil, puppetfile_path = nil, puppetfile_name = nil, force = nil )
+  # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options.
+  #         Usage as moduledir is deprecated. Only use as options, defaults to nil
+  # @param [String, nil] puppetfile_path Deprecated - The path to the Puppetfile, defaults to nil
+  # @param [String, nil] puppetfile_name Deprecated - The name of the Puppetfile, defaults to nil
+  # @param [Boolean, nil] force Deprecated - Shall we overwrite locally made changes?
+  def initialize(basedir, options_or_moduledir = nil, deprecated_path_arg = nil, deprecated_name_arg = nil, deprecated_force_arg = nil)
     @basedir         = basedir
-    @force           = force || false
-    @moduledir       = moduledir  || File.join(basedir, 'modules')
-    @puppetfile_name = puppetfile_name || 'Puppetfile'
-    @puppetfile_path = puppetfile_path || File.join(basedir, @puppetfile_name)
+    if options_or_moduledir.is_a? Hash
+      options = options_or_moduledir
+      deprecated_moduledir_arg = nil
+    else
+      options = {}
+      deprecated_moduledir_arg = options_or_moduledir
+    end
+
+    @force           = deprecated_force_arg     || options.delete(:force)           || false
+    @moduledir       = deprecated_moduledir_arg || options.delete(:moduledir)       || File.join(basedir, 'modules')
+    @puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
+    @puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
+
+    @overrides       = options.delete(:overrides) || {}
 
     logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
 
@@ -117,19 +133,30 @@ class Puppetfile
   end
 
   # @param [String] name
-  # @param [*Object] args
+  # @param [Hash, String, Symbol] args Calling with anything but a Hash is
+  #   deprecated. The DSL will now convert String and Symbol versions to
+  #   Hashes of the shape
+  #     { version: <String or Symbol> }
+  #
   def add_module(name, args)
-    if args.is_a?(Hash) && install_path = args.delete(:install_path)
+    if !args.is_a?(Hash)
+      args = { version: args }
+    end
+
+    args[:overrides] = @overrides
+
+    if install_path = args.delete(:install_path)
       install_path = resolve_install_path(install_path)
       validate_install_path(install_path, name)
     else
       install_path = @moduledir
     end
 
-    if args.is_a?(Hash) && @default_branch_override != nil
+    if @default_branch_override != nil
       args[:default_branch_override] = @default_branch_override
     end
 
+
     mod = R10K::Module.new(name, install_path, args, @environment)
     mod.origin = :puppetfile
 
@@ -181,70 +208,22 @@ class Puppetfile
   def accept(visitor)
     pool_size = self.settings[:pool_size]
     if pool_size > 1
-      concurrent_accept(visitor, pool_size)
+      R10K::ContentSynchronizer.concurrent_accept(modules, visitor, self, pool_size, logger)
     else
-      serial_accept(visitor)
-    end
-  end
-
-  private
-
-  def serial_accept(visitor)
-    visitor.visit(:puppetfile, self) do
-      modules.each do |mod|
-        mod.accept(visitor)
-      end
+      R10K::ContentSynchronizer.serial_accept(modules, visitor, self)
     end
   end
 
-  def concurrent_accept(visitor, pool_size)
-    logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
-    mods_queue = modules_queue(visitor)
-    thread_pool = pool_size.times.map { visitor_thread(visitor, mods_queue) }
-    thread_exception = nil
-
-    # If any threads raise an exception the deployment is considered a failure.
-    # In that event clear the queue, wait for other threads to finish their
-    # current work, then re-raise the first exception caught.
-    begin
-      thread_pool.each(&:join)
-    rescue => e
-      logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
-      mods_queue.clear
-      thread_exception ||= e
-      retry
-    ensure
-      raise thread_exception unless thread_exception.nil?
+  def sync
+    pool_size = self.settings[:pool_size]
+    if pool_size > 1
+      R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger)
+    else
+      R10K::ContentSynchronizer.serial_sync(modules)
     end
   end
 
-  def modules_queue(visitor)
-    Queue.new.tap do |queue|
-      visitor.visit(:puppetfile, self) do
-        modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
-        modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
-
-        modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
-        modules_by_cachedir.values.each {|mods| queue << mods }
-      end
-    end
-  end
-  public :modules_queue
-
-  def visitor_thread(visitor, mods_queue)
-    Thread.new do
-      begin
-        while mods = mods_queue.pop(true) do
-          mods.each {|mod| mod.accept(visitor) }
-        end
-      rescue ThreadError => e
-        logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
-        Thread.exit
-      rescue => e
-        Thread.main.raise(e)
-      end
-    end
-  end
+  private
 
   def puppetfile_contents
     File.read(@puppetfile_path)
@@ -285,7 +264,13 @@ class Puppetfile
     end
 
     def mod(name, args = nil)
-      @librarian.add_module(name, args)
+      if args.is_a?(Hash)
+        opts = args
+      else
+        opts = { version: args }
+      end
+
+      @librarian.add_module(name, opts)
     end
 
     def forge(location)

data/lib/r10k/source/base.rb

@@ -1,8 +1,12 @@
+require 'r10k/logging'
+
 # This class defines a common interface for source implementations.
 #
 # @since 1.3.0
 class R10K::Source::Base
 
+  include R10K::Logging
+
   # @!attribute [r] basedir
   #   @return [String] The path this source will place environments in
   attr_reader :basedir

data/lib/r10k/source/git.rb

@@ -11,8 +11,6 @@ require 'r10k/environment/name'
 # @since 1.3.0
 class R10K::Source::Git < R10K::Source::Base
 
-  include R10K::Logging
-
   R10K::Source.register(:git, self)
   # Register git as the default source
   R10K::Source.register(nil, self)
@@ -99,12 +97,22 @@ class R10K::Source::Git < R10K::Source::Base
     envs = []
     branch_names.each do |bn|
       if bn.valid?
-        envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname,
-                                       {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name })
+        envs << R10K::Environment::Git.new(bn.name,
+                                           @basedir,
+                                           bn.dirname,
+                                           {remote: remote,
+                                            ref: bn.name,
+                                            puppetfile_name: puppetfile_name,
+                                            overrides: @options[:overrides]})
       elsif bn.correct?
        logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname}
-        envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname,
-                                       {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name})
+        envs << R10K::Environment::Git.new(bn.name,
+                                           @basedir,
+                                           bn.dirname,
+                                           {remote: remote,
+                                            ref: bn.name,
+                                            puppetfile_name: puppetfile_name,
+                                            overrides: @options[:overrides]})
       elsif bn.validate?
        logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect}
       end

data/lib/r10k/source/hash.rb

@@ -120,8 +120,6 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
-  include R10K::Logging
-
   # @param hash [Hash] A hash to validate.
   # @return [Boolean] False if the hash is obviously invalid. A true return
   #   means _maybe_ it's valid.
@@ -170,7 +168,7 @@ class R10K::Source::Hash < R10K::Source::Base
 
   def environments
     @environments ||= environments_hash.map do |name, hash|
-      R10K::Environment.from_hash(name, hash)
+      R10K::Environment.from_hash(name, hash.merge({overrides: @options[:overrides]}))
     end
   end
 

data/lib/r10k/source/svn.rb

@@ -103,8 +103,6 @@ class R10K::Source::SVN < R10K::Source::Base
     @environments.map {|env| env.dirname }
   end
 
-  include R10K::Logging
-
   def filter_branches(branches, ignore_prefixes)
     filter = Regexp.new("^(#{ignore_prefixes.join('|')})")
     branches = branches.reject do |branch|

data/lib/r10k/util/cleaner.rb

@@ -0,0 +1,21 @@
+require 'r10k/logging'
+require 'r10k/util/purgeable'
+
+module R10K
+  module Util
+    class Cleaner
+
+      include R10K::Logging
+      include R10K::Util::Purgeable
+
+      attr_reader :managed_directories, :desired_contents, :purge_exclusions
+
+      def initialize(managed_directories, desired_contents, purge_exclusions = [])
+        @managed_directories = managed_directories
+        @desired_contents    = desired_contents
+        @purge_exclusions    = purge_exclusions
+      end
+
+    end
+  end
+end

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.9.0'
+  VERSION = '3.9.1'
 end

data/locales/r10k.pot

@@ -6,11 +6,11 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.4.1-151-g93d38cb\n"
+"Project-Id-Version: r10k 3.4.1-231-g5574915\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2021-03-15 17:00+0000\n"
-"PO-Revision-Date: 2021-03-15 17:00+0000\n"
+"POT-Creation-Date: 2021-05-10 23:18+0000\n"
+"PO-Revision-Date: 2021-05-10 23:18+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -31,58 +31,38 @@ msgstr ""
 msgid "Reason: %{write_lock}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:57
+#: ../lib/r10k/action/deploy/environment.rb:109
 msgid "Environment(s) \\'%{environments}\\' cannot be found in any source and will not be deployed."
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:85
+#: ../lib/r10k/action/deploy/environment.rb:139
 msgid "Environment %{env_dir} does not match environment name filter, skipping"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:93
+#: ../lib/r10k/action/deploy/environment.rb:147
 msgid "Deploying environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:96
+#: ../lib/r10k/action/deploy/environment.rb:150
 msgid "Environment %{env_dir} is now at %{env_signature}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:100
+#: ../lib/r10k/action/deploy/environment.rb:154
 msgid "Environment %{env_dir} is new, updating all modules"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:143
-msgid "Deploying %{origin} content %{path}"
+#: ../lib/r10k/action/deploy/module.rb:78
+msgid "Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/module.rb:49
-msgid "Only updating modules in environment %{opt_env} skipping environment %{env_path}"
-msgstr ""
-
-#: ../lib/r10k/action/deploy/module.rb:51
+#: ../lib/r10k/action/deploy/module.rb:80
 msgid "Updating modules %{modules} in environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/module.rb:63
-msgid "Deploying module %{mod_path}"
-msgstr ""
-
-#: ../lib/r10k/action/deploy/module.rb:70
-msgid "Only updating modules %{modules}, skipping module %{mod_name}"
-msgstr ""
-
-#: ../lib/r10k/action/puppetfile/check.rb:14
+#: ../lib/r10k/action/puppetfile/check.rb:16
 msgid "Syntax OK"
 msgstr ""
 
-#: ../lib/r10k/action/puppetfile/install.rb:30
-msgid "Updating module %{mod_path}"
-msgstr ""
-
-#: ../lib/r10k/action/puppetfile/install.rb:33
-msgid "Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available."
-msgstr ""
-
 #: ../lib/r10k/action/runner.rb:54 ../lib/r10k/deployment/config.rb:42
 msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'"
 msgstr ""
@@ -95,6 +75,18 @@ msgstr ""
 msgid "No config file explicitly given and no default config file could be found, default settings will be used."
 msgstr ""
 
+#: ../lib/r10k/content_synchronizer.rb:13
+msgid "Updating modules with %{pool_size} threads"
+msgstr ""
+
+#: ../lib/r10k/content_synchronizer.rb:24
+msgid "Error during concurrent deploy of a module: %{message}"
+msgstr ""
+
+#: ../lib/r10k/content_synchronizer.rb:52
+msgid "Module thread %{id} exiting: %{message}"
+msgstr ""
+
 #: ../lib/r10k/deployment.rb:90
 msgid "Environment collision at %{env_path} between %{source}:%{env_name} and %{osource}:%{oenv_name}"
 msgstr ""
@@ -103,7 +95,7 @@ msgstr ""
 msgid "Unable to load sources; the supplied configuration does not define the 'sources' key"
 msgstr ""
 
-#: ../lib/r10k/environment/base.rb:61 ../lib/r10k/environment/base.rb:77 ../lib/r10k/environment/base.rb:86 ../lib/r10k/source/base.rb:69
+#: ../lib/r10k/environment/base.rb:65 ../lib/r10k/environment/base.rb:81 ../lib/r10k/environment/base.rb:90 ../lib/r10k/source/base.rb:69
 msgid "%{class} has not implemented method %{method}"
 msgstr ""
 
@@ -327,19 +319,31 @@ msgstr ""
 msgid "Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)"
 msgstr ""
 
-#: ../lib/r10k/module/base.rb:118
+#: ../lib/r10k/module/base.rb:75
+msgid "Deploying module to %{path}"
+msgstr ""
+
+#: ../lib/r10k/module/base.rb:78
+msgid "Only updating modules %{modules}, skipping module %{name}"
+msgstr ""
+
+#: ../lib/r10k/module/base.rb:134
 msgid "Module name (%{title}) must match either 'modulename' or 'owner/modulename'"
 msgstr ""
 
-#: ../lib/r10k/module/forge.rb:81 ../lib/r10k/module/forge.rb:110
+#: ../lib/r10k/module/forge.rb:88 ../lib/r10k/module/forge.rb:117
 msgid "The module %{title} does not exist on %{url}."
 msgstr ""
 
-#: ../lib/r10k/module/forge.rb:185
+#: ../lib/r10k/module/forge.rb:192
 msgid "Forge module names must match 'owner/modulename', instead got #{title}"
 msgstr ""
 
-#: ../lib/r10k/module/local.rb:34
+#: ../lib/r10k/module/git.rb:66
+msgid "Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available."
+msgstr ""
+
+#: ../lib/r10k/module/local.rb:35
 msgid "Module %{title} is a local module, always indicating synced."
 msgstr ""
 
@@ -347,39 +351,27 @@ msgstr ""
 msgid "Could not read metadata.json"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:57
+#: ../lib/r10k/puppetfile.rb:73
 msgid "Using Puppetfile '%{puppetfile}'"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:71
+#: ../lib/r10k/puppetfile.rb:87
 msgid "Puppetfile %{path} missing or unreadable"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:84
+#: ../lib/r10k/puppetfile.rb:100
 msgid "Failed to evaluate %{path}"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:98
+#: ../lib/r10k/puppetfile.rb:114
 msgid "Puppetfiles cannot contain duplicate module names."
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:100
+#: ../lib/r10k/puppetfile.rb:116
 msgid "Remove the duplicates of the following modules: %{dupes}"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:201
-msgid "Updating modules with %{pool_size} threads"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:212
-msgid "Error during concurrent deploy of a module: %{message}"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:241
-msgid "Module thread %{id} exiting: %{message}"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:300
+#: ../lib/r10k/puppetfile.rb:276
 msgid "unrecognized declaration '%{method}'"
 msgstr ""
 
@@ -470,19 +462,19 @@ msgstr ""
 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:105
+#: ../lib/r10k/source/git.rb:110
 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:109
+#: ../lib/r10k/source/git.rb:119
 msgid "Environment %{env_name} contained non-word characters, ignoring it."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113
+#: ../lib/r10k/source/git.rb:138 ../lib/r10k/source/svn.rb:113
 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:139
+#: ../lib/r10k/source/git.rb:149
 msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}"
 msgstr ""