r10k 3.9.0 → 3.10.0

data/lib/r10k/source/base.rb

@@ -1,8 +1,12 @@
+require 'r10k/logging'
+
 # This class defines a common interface for source implementations.
 #
 # @since 1.3.0
 class R10K::Source::Base
 
+  include R10K::Logging
+
   # @!attribute [r] basedir
   #   @return [String] The path this source will place environments in
   attr_reader :basedir
@@ -59,6 +63,16 @@ class R10K::Source::Base
 
   end
 
+  # Perform actions to reload environments after the `preload!`. Similar
+  # to preload!, and likely to include network queries and rerunning
+  # environment generation.
+  #
+  # @api public
+  # @abstract
+  # @return [void]
+  def reload!
+  end
+
   # Enumerate the environments associated with this SVN source.
   #
   # @api public

data/lib/r10k/source/git.rb

@@ -11,8 +11,6 @@ require 'r10k/environment/name'
 # @since 1.3.0
 class R10K::Source::Git < R10K::Source::Base
 
-  include R10K::Logging
-
   R10K::Source.register(:git, self)
   # Register git as the default source
   R10K::Source.register(nil, self)
@@ -95,16 +93,31 @@ class R10K::Source::Git < R10K::Source::Base
     end
   end
 
+  def reload!
+    @cache.sync!
+    @environments = generate_environments()
+  end
+
   def generate_environments
     envs = []
     branch_names.each do |bn|
       if bn.valid?
-        envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname,
-                                       {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name })
+        envs << R10K::Environment::Git.new(bn.name,
+                                           @basedir,
+                                           bn.dirname,
+                                           {remote: remote,
+                                            ref: bn.name,
+                                            puppetfile_name: puppetfile_name,
+                                            overrides: @options[:overrides]})
       elsif bn.correct?
        logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname}
-        envs << R10K::Environment::Git.new(bn.name, @basedir, bn.dirname,
-                                       {:remote => remote, :ref => bn.name, :puppetfile_name => puppetfile_name})
+        envs << R10K::Environment::Git.new(bn.name,
+                                           @basedir,
+                                           bn.dirname,
+                                           {remote: remote,
+                                            ref: bn.name,
+                                            puppetfile_name: puppetfile_name,
+                                            overrides: @options[:overrides]})
       elsif bn.validate?
        logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect}
       end

data/lib/r10k/source/hash.rb

@@ -120,8 +120,6 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
-  include R10K::Logging
-
   # @param hash [Hash] A hash to validate.
   # @return [Boolean] False if the hash is obviously invalid. A true return
   #   means _maybe_ it's valid.
@@ -170,7 +168,7 @@ class R10K::Source::Hash < R10K::Source::Base
 
   def environments
     @environments ||= environments_hash.map do |name, hash|
-      R10K::Environment.from_hash(name, hash)
+      R10K::Environment.from_hash(name, hash.merge({overrides: @options[:overrides]}))
     end
   end
 

data/lib/r10k/source/svn.rb

@@ -65,6 +65,10 @@ class R10K::Source::SVN < R10K::Source::Base
     @ignore_branch_prefixes = options[:ignore_branch_prefixes]
   end
 
+  def reload!
+    @environments = generate_environments()
+  end
+
   # Enumerate the environments associated with this SVN source.
   #
   # @return [Array<R10K::Environment::SVN>] An array of environments created
@@ -103,8 +107,6 @@ class R10K::Source::SVN < R10K::Source::Base
     @environments.map {|env| env.dirname }
   end
 
-  include R10K::Logging
-
   def filter_branches(branches, ignore_prefixes)
     filter = Regexp.new("^(#{ignore_prefixes.join('|')})")
     branches = branches.reject do |branch|

data/lib/r10k/util/cleaner.rb

@@ -0,0 +1,21 @@
+require 'r10k/logging'
+require 'r10k/util/purgeable'
+
+module R10K
+  module Util
+    class Cleaner
+
+      include R10K::Logging
+      include R10K::Util::Purgeable
+
+      attr_reader :managed_directories, :desired_contents, :purge_exclusions
+
+      def initialize(managed_directories, desired_contents, purge_exclusions = [])
+        @managed_directories = managed_directories
+        @desired_contents    = desired_contents
+        @purge_exclusions    = purge_exclusions
+      end
+
+    end
+  end
+end

data/lib/r10k/util/purgeable.rb

@@ -9,6 +9,12 @@ module R10K
     #   {#desired_contents}
     module Purgeable
 
+      HIDDEN_FILE = /\.[^.]+/
+
+      FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH
+
+      # @deprecated
+      #
       # @!method logger
       #   @abstract Including classes must provide a logger method
       #   @return [Log4r::Logger]
@@ -38,23 +44,79 @@ module R10K
         end
       end
 
+      # @deprecated Unused helper function
+      #
       # @return [Array<String>] Directory contents that are expected but not present
       def pending_contents(recurse)
         desired_contents - current_contents(recurse)
       end
 
+      def matches?(test, path)
+        if test == path
+          true
+        elsif File.fnmatch?(test, path, FN_MATCH_OPTS)
+          true
+        else
+          false
+        end
+      end
+
+      # A method to collect potentially purgeable content without searching into
+      # ignored directories when recursively searching.
+      #
+      # @param dir [String, Pathname] The directory to search for purgeable content
+      # @param exclusion_gobs [Array<String>] A list of file paths or File globs
+      #     to exclude from recursion (These are generated by the classes that
+      #     mix this module into them and are typically programatically generated)
+      # @param allowed_gobs [Array<String>] A list of file paths or File globs to exclude
+      #     from recursion (These are passed in by the caller of purge! and typically
+      #     are user supplied configuration values)
+      # @param desireds_not_to_recurse_into [Array<String>] A list of file paths not to
+      #     recurse into. These are programatically generated, these exist to maintain
+      #     backwards compatibility with previous implementations that used File.globs
+      #     for "recursion", ie "**/{*,.[^.]*}" which would not recurse into dot directories.
+      # @param recurse [Boolean] Whether or not to recurse into child directories that do
+      #     not match other filters.
+      #
+      # @return [Array<String>] Contents which may be purged.
+      def potentially_purgeable(dir, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
+        children = Pathname.new(dir).children.reject do |path|
+          path = path.to_s
+
+          if exclusion_match = exclusion_globs.find { |exclusion| matches?(exclusion, path) }
+            logger.debug2 _("Not purging %{path} due to internal exclusion match: %{exclusion_match}") % {path: path, exclusion_match: exclusion_match}
+          elsif allowlist_match = allowed_globs.find { |allowed| matches?(allowed, path) }
+            logger.debug _("Not purging %{path} due to whitelist match: %{allowlist_match}") % {path: path, allowlist_match: allowlist_match}
+          else
+            desired_match = desireds_not_to_recurse_into.grep(path).first
+          end
+
+          !!exclusion_match || !!allowlist_match || !!desired_match
+        end
+
+        children.flat_map do |child|
+          if File.directory?(child) && recurse
+            potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
+          else
+            child.to_s
+          end
+        end
+      end
+
       # @return [Array<String>] Directory contents that are present but not expected
       def stale_contents(recurse, exclusions, whitelist)
-        fn_match_opts = File::FNM_PATHNAME | File::FNM_DOTMATCH
+        dirs = self.managed_directories
+        desireds = self.desired_contents
+        hidden_desireds, regular_desireds = desireds.partition do |desired|
+          HIDDEN_FILE.match(File.basename(desired))
+        end
 
-        (current_contents(recurse) - desired_contents).reject do |item|
-          if exclusion_match = exclusions.find { |ex_item| (ex_item == item) || File.fnmatch?(ex_item, item, fn_match_opts) }
-            logger.debug2 _("Not purging %{item} due to internal exclusion match: %{exclusion_match}") % {item: item, exclusion_match: exclusion_match}
-          elsif whitelist_match = whitelist.find { |wl_item| (wl_item == item) || File.fnmatch?(wl_item, item, fn_match_opts) }
-            logger.debug _("Not purging %{item} due to whitelist match: %{whitelist_match}") % {item: item, whitelist_match: whitelist_match}
-          end
+        initial_purgelist = dirs.flat_map do |dir|
+          potentially_purgeable(dir, exclusions, whitelist, hidden_desireds, recurse)
+        end
 
-          !!exclusion_match || !!whitelist_match
+        initial_purgelist.reject do |path|
+          regular_desireds.any? { |desired| matches?(desired, path) }
         end
       end
 

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.9.0'
+  VERSION = '3.10.0'
 end

data/locales/r10k.pot

@@ -6,11 +6,11 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.4.1-151-g93d38cb\n"
+"Project-Id-Version: r10k 3.9.3-10-gfedc81a\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2021-03-15 17:00+0000\n"
-"PO-Revision-Date: 2021-03-15 17:00+0000\n"
+"POT-Creation-Date: 2021-07-07 21:13+0000\n"
+"PO-Revision-Date: 2021-07-07 21:13+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -31,58 +31,38 @@ msgstr ""
 msgid "Reason: %{write_lock}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:57
+#: ../lib/r10k/action/deploy/environment.rb:112
 msgid "Environment(s) \\'%{environments}\\' cannot be found in any source and will not be deployed."
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:85
+#: ../lib/r10k/action/deploy/environment.rb:145
 msgid "Environment %{env_dir} does not match environment name filter, skipping"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:93
+#: ../lib/r10k/action/deploy/environment.rb:153
 msgid "Deploying environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:96
+#: ../lib/r10k/action/deploy/environment.rb:156
 msgid "Environment %{env_dir} is now at %{env_signature}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:100
+#: ../lib/r10k/action/deploy/environment.rb:160
 msgid "Environment %{env_dir} is new, updating all modules"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/environment.rb:143
-msgid "Deploying %{origin} content %{path}"
+#: ../lib/r10k/action/deploy/module.rb:81
+msgid "Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/module.rb:49
-msgid "Only updating modules in environment %{opt_env} skipping environment %{env_path}"
-msgstr ""
-
-#: ../lib/r10k/action/deploy/module.rb:51
+#: ../lib/r10k/action/deploy/module.rb:83
 msgid "Updating modules %{modules} in environment %{env_path}"
 msgstr ""
 
-#: ../lib/r10k/action/deploy/module.rb:63
-msgid "Deploying module %{mod_path}"
-msgstr ""
-
-#: ../lib/r10k/action/deploy/module.rb:70
-msgid "Only updating modules %{modules}, skipping module %{mod_name}"
-msgstr ""
-
-#: ../lib/r10k/action/puppetfile/check.rb:14
+#: ../lib/r10k/action/puppetfile/check.rb:16
 msgid "Syntax OK"
 msgstr ""
 
-#: ../lib/r10k/action/puppetfile/install.rb:30
-msgid "Updating module %{mod_path}"
-msgstr ""
-
-#: ../lib/r10k/action/puppetfile/install.rb:33
-msgid "Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available."
-msgstr ""
-
 #: ../lib/r10k/action/runner.rb:54 ../lib/r10k/deployment/config.rb:42
 msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'"
 msgstr ""
@@ -95,6 +75,18 @@ msgstr ""
 msgid "No config file explicitly given and no default config file could be found, default settings will be used."
 msgstr ""
 
+#: ../lib/r10k/content_synchronizer.rb:27
+msgid "Updating modules with %{pool_size} threads"
+msgstr ""
+
+#: ../lib/r10k/content_synchronizer.rb:37
+msgid "Error during concurrent deploy of a module: %{message}"
+msgstr ""
+
+#: ../lib/r10k/content_synchronizer.rb:75
+msgid "Module thread %{id} exiting: %{message}"
+msgstr ""
+
 #: ../lib/r10k/deployment.rb:90
 msgid "Environment collision at %{env_path} between %{source}:%{env_name} and %{osource}:%{oenv_name}"
 msgstr ""
@@ -103,7 +95,7 @@ msgstr ""
 msgid "Unable to load sources; the supplied configuration does not define the 'sources' key"
 msgstr ""
 
-#: ../lib/r10k/environment/base.rb:61 ../lib/r10k/environment/base.rb:77 ../lib/r10k/environment/base.rb:86 ../lib/r10k/source/base.rb:69
+#: ../lib/r10k/environment/base.rb:68 ../lib/r10k/environment/base.rb:84 ../lib/r10k/environment/base.rb:93 ../lib/r10k/source/base.rb:83
 msgid "%{class} has not implemented method %{method}"
 msgstr ""
 
@@ -111,15 +103,15 @@ msgstr ""
 msgid "Improper configuration value given for strip_component setting in %{src} source. Value must be a string, a /regex/, false, or omitted. Got \"%{val}\" (%{type})"
 msgstr ""
 
-#: ../lib/r10k/environment/with_modules.rb:60
+#: ../lib/r10k/environment/with_modules.rb:57
 msgid "Environment and %{src} both define the \"%{name}\" module"
 msgstr ""
 
-#: ../lib/r10k/environment/with_modules.rb:61
+#: ../lib/r10k/environment/with_modules.rb:58
 msgid "#{msg_error}. The %{src} definition will be ignored"
 msgstr ""
 
-#: ../lib/r10k/environment/with_modules.rb:71
+#: ../lib/r10k/environment/with_modules.rb:68
 msgid "Unexpected value for `module_conflicts` setting in %{env} environment: %{val}"
 msgstr ""
 
@@ -327,19 +319,35 @@ msgstr ""
 msgid "Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)"
 msgstr ""
 
-#: ../lib/r10k/module/base.rb:118
+#: ../lib/r10k/module/base.rb:75
+msgid "Deploying module to %{path}"
+msgstr ""
+
+#: ../lib/r10k/module/base.rb:78
+msgid "Only updating modules %{modules}, skipping module %{name}"
+msgstr ""
+
+#: ../lib/r10k/module/base.rb:134
 msgid "Module name (%{title}) must match either 'modulename' or 'owner/modulename'"
 msgstr ""
 
-#: ../lib/r10k/module/forge.rb:81 ../lib/r10k/module/forge.rb:110
+#: ../lib/r10k/module/forge.rb:89
+msgid "The module %{title} does not appear to have any published releases, cannot determine latest version."
+msgstr ""
+
+#: ../lib/r10k/module/forge.rb:92 ../lib/r10k/module/forge.rb:121
 msgid "The module %{title} does not exist on %{url}."
 msgstr ""
 
-#: ../lib/r10k/module/forge.rb:185
+#: ../lib/r10k/module/forge.rb:196
 msgid "Forge module names must match 'owner/modulename', instead got #{title}"
 msgstr ""
 
-#: ../lib/r10k/module/local.rb:34
+#: ../lib/r10k/module/git.rb:66
+msgid "Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available."
+msgstr ""
+
+#: ../lib/r10k/module/local.rb:35
 msgid "Module %{title} is a local module, always indicating synced."
 msgstr ""
 
@@ -347,39 +355,27 @@ msgstr ""
 msgid "Could not read metadata.json"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:57
+#: ../lib/r10k/puppetfile.rb:73
 msgid "Using Puppetfile '%{puppetfile}'"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:71
+#: ../lib/r10k/puppetfile.rb:87
 msgid "Puppetfile %{path} missing or unreadable"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:84
+#: ../lib/r10k/puppetfile.rb:100
 msgid "Failed to evaluate %{path}"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:98
+#: ../lib/r10k/puppetfile.rb:114
 msgid "Puppetfiles cannot contain duplicate module names."
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:100
+#: ../lib/r10k/puppetfile.rb:116
 msgid "Remove the duplicates of the following modules: %{dupes}"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:201
-msgid "Updating modules with %{pool_size} threads"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:212
-msgid "Error during concurrent deploy of a module: %{message}"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:241
-msgid "Module thread %{id} exiting: %{message}"
-msgstr ""
-
-#: ../lib/r10k/puppetfile.rb:300
+#: ../lib/r10k/puppetfile.rb:285
 msgid "unrecognized declaration '%{method}'"
 msgstr ""
 
@@ -462,27 +458,27 @@ msgid ""
 "Returned: %{data}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:77
+#: ../lib/r10k/source/git.rb:75
 msgid "Fetching '%{remote}' to determine current branches."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:80
+#: ../lib/r10k/source/git.rb:78
 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:105
+#: ../lib/r10k/source/git.rb:113
 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:109
+#: ../lib/r10k/source/git.rb:122
 msgid "Environment %{env_name} contained non-word characters, ignoring it."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113
+#: ../lib/r10k/source/git.rb:141 ../lib/r10k/source/svn.rb:115
 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:139
+#: ../lib/r10k/source/git.rb:152
 msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}"
 msgstr ""
 
@@ -526,23 +522,23 @@ msgstr ""
 msgid "pe_license feature is not available, PE only Puppet modules will not be downloadable."
 msgstr ""
 
-#: ../lib/r10k/util/purgeable.rb:52
-msgid "Not purging %{item} due to internal exclusion match: %{exclusion_match}"
+#: ../lib/r10k/util/purgeable.rb:87
+msgid "Not purging %{path} due to internal exclusion match: %{exclusion_match}"
 msgstr ""
 
-#: ../lib/r10k/util/purgeable.rb:54
-msgid "Not purging %{item} due to whitelist match: %{whitelist_match}"
+#: ../lib/r10k/util/purgeable.rb:89
+msgid "Not purging %{path} due to whitelist match: %{allowlist_match}"
 msgstr ""
 
-#: ../lib/r10k/util/purgeable.rb:71
+#: ../lib/r10k/util/purgeable.rb:133
 msgid "No unmanaged contents in %{managed_dirs}, nothing to purge"
 msgstr ""
 
-#: ../lib/r10k/util/purgeable.rb:76
+#: ../lib/r10k/util/purgeable.rb:138
 msgid "Removing unmanaged path %{path}"
 msgstr ""
 
-#: ../lib/r10k/util/purgeable.rb:81
+#: ../lib/r10k/util/purgeable.rb:143
 msgid "Unable to remove unmanaged path: %{path}"
 msgstr ""