r10k 3.9.3 → 3.12.1

data/lib/r10k/action/deploy/environment.rb

@@ -1,10 +1,9 @@
-require 'r10k/util/setopts'
-require 'r10k/util/cleaner'
-require 'r10k/deployment'
-require 'r10k/logging'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
 require 'r10k/action/deploy/deploy_helpers'
+require 'r10k/action/visitor'
+require 'r10k/deployment'
+require 'r10k/util/setopts'
+
 require 'json'
 
 module R10K
@@ -13,6 +12,7 @@ module R10K
       class Environment < R10K::Action::Base
 
         include R10K::Action::Deploy::DeployHelpers
+        include R10K::Action::Visitor
 
         # Deprecated
         attr_reader :force
@@ -41,11 +41,14 @@ module R10K
                 requested_environments: @argv.map { |arg| arg.gsub(/\W/,'_') },
                 default_branch_override: @default_branch_override,
                 generate_types: @generate_types || settings.dig(:deploy, :generate_types) || false,
-                preload_environments: true
+                preload_environments: true,
+                incremental: @incremental
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
                 requested_modules: [],
                 deploy_modules: @modules,
+                pool_size: @settings[:pool_size] || 4,
                 force: !@no_force, # force here is used to make it easier to reason about
               },
               purging: {
@@ -53,6 +56,9 @@ module R10K
                 purge_allowlist: read_purge_allowlist(settings.dig(:deploy, :purge_whitelist) || [],
                                                       settings.dig(:deploy, :purge_allowlist) || [])
               },
+              forge: {
+                allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false
+              },
               output: {}
             }
           })
@@ -75,8 +81,6 @@ module R10K
           @visit_ok
         end
 
-        include R10K::Action::Visitor
-
         private
 
         def read_purge_allowlist (whitelist, allowlist)
@@ -116,6 +120,9 @@ module R10K
 
           if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment)
             logger.debug("Purging unmanaged environments for deployment...")
+            deployment.sources.each do |source|
+              source.reload!
+            end
             deployment.purge!
           end
         ensure
@@ -126,6 +133,7 @@ module R10K
               envs.reject! { |e| !requested_envs.include?(e) } if requested_envs.any?
               postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs.join(' ')) }
             end
+            logger.debug _("Executing postrun command.")
             subproc = R10K::Util::Subprocess.new(postcmd)
             subproc.logger = logger
             subproc.execute
@@ -233,13 +241,18 @@ module R10K
           super.merge(puppetfile: :modules,
                       modules: :self,
                       cachedir: :self,
+                      incremental: :self,
                       'no-force': :self,
+                      'exclude-spec': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
                       'oauth-token': :self,
-                      'default-branch-override': :self)
+                      'default-branch-override': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/deploy/module.rb

@@ -1,7 +1,7 @@
-require 'r10k/deployment'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
 require 'r10k/action/deploy/deploy_helpers'
+require 'r10k/action/visitor'
+require 'r10k/deployment'
 
 module R10K
   module Action
@@ -9,6 +9,7 @@ module R10K
       class Module < R10K::Action::Base
 
         include R10K::Action::Deploy::DeployHelpers
+        include R10K::Action::Visitor
 
         # Deprecated
         attr_reader :force
@@ -29,6 +30,7 @@ module R10K
           super
 
           requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : []
+          @modified_envs = []
 
           @settings = @settings.merge({
             overrides: {
@@ -37,10 +39,15 @@ module R10K
                 generate_types: @generate_types
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
+                pool_size: @settings[:pool_size] || 4,
                 requested_modules: @argv.map.to_a,
                 # force here is used to make it easier to reason about
                 force: !@no_force
               },
+              forge: {
+                allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false
+              },
               purging: {},
               output: {}
             }
@@ -63,12 +70,27 @@ module R10K
           @visit_ok
         end
 
-        include R10K::Action::Visitor
-
         private
 
         def visit_deployment(deployment)
           yield
+        ensure
+          if (postcmd = @settings[:postrun])
+            if @modified_envs.any?
+              envs_to_run = @modified_envs.join(' ')
+              logger.debug _("Running postrun command for environments: %{envs_to_run}.") % { envs_to_run: envs_to_run }
+
+              if postcmd.grep('$modifiedenvs').any?
+                postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) }
+              end
+
+              subproc = R10K::Util::Subprocess.new(postcmd)
+              subproc.logger = logger
+              subproc.execute
+            else
+              logger.debug _("No environments were modified, not executing postrun command.")
+            end
+          end
         end
 
         def visit_source(source)
@@ -82,13 +104,17 @@ module R10K
           else
             logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @settings.dig(:overrides, :modules, :requested_modules).inspect, env_path: environment.path})
 
-            environment.deploy
+            updated_modules = environment.deploy
+
+            # We actually synced a module in this env
+            if !updated_modules.nil? && !updated_modules.empty?
+              # Record modified environment for postrun command
+              @modified_envs << environment.dirname
 
-            requested_mods = @settings.dig(:overrides, :modules, :requested_modules) || []
-            generate_types = @settings.dig(:overrides, :environments, :generate_types)
-            if generate_types && !((environment.modules.map(&:name) & requested_mods).empty?)
-              logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
-              environment.generate_types!
+              if generate_types = @settings.dig(:overrides, :environments, :generate_types)
+                logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
+                environment.generate_types!
+              end
             end
           end
         end
@@ -96,12 +122,16 @@ module R10K
         def allowed_initialize_opts
           super.merge(environment: true,
                       cachedir: :self,
+                      'exclude-spec': :self,
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
-                      'oauth-token': :self)
+                      'oauth-token': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/puppetfile/check.rb

@@ -1,6 +1,6 @@
-require 'r10k/puppetfile'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
 
 module R10K
   module Action
@@ -8,11 +8,13 @@ module R10K
       class Check < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
+          options = { basedir: @root }
+          options[:moduledir] = @moduledir if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
           begin
-            pf.load!
+            loader.load!
             $stderr.puts _("Syntax OK")
             true
           rescue => e

data/lib/r10k/action/puppetfile/install.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/errors/formatting'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
+require 'r10k/content_synchronizer'
+require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
 require 'r10k/util/cleaner'
 
 module R10K
@@ -10,25 +10,31 @@ module R10K
       class Install < R10K::Action::Base
 
         def call
-          @ok = true
           begin
-            pf = R10K::Puppetfile.new(@root,
-                                      {moduledir: @moduledir,
-                                       puppetfile_path: @puppetfile,
-                                       force: @force || false})
-            pf.load!
-            pf.sync
+            options = { basedir: @root, overrides: { force: @force || false } }
+            options[:moduledir]  = @moduledir  if @moduledir
+            options[:puppetfile] = @puppetfile if @puppetfile
+
+            loader = R10K::ModuleLoader::Puppetfile.new(**options)
+            loaded_content = loader.load!
 
-            R10K::Util::Cleaner.new(pf.managed_directories,
-                                    pf.desired_contents,
-                                    pf.purge_exclusions).purge!
+            pool_size = @settings[:pool_size] || 4
+            modules   = loaded_content[:modules]
+            if pool_size > 1
+              R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger)
+            else
+              R10K::ContentSynchronizer.serial_sync(modules, logger)
+            end
 
+            R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                    loaded_content[:desired_contents],
+                                    loaded_content[:purge_exclusions]).purge!
+
+            true
           rescue => e
-            @ok = false
             logger.error R10K::Errors::Formatting.format_exception(e, @trace)
+            false
           end
-
-          @ok
         end
 
         private

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,13 +9,16 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
-          pf.load!
-          R10K::Util::Cleaner.new(pf.managed_directories,
-                                  pf.desired_contents,
-                                  pf.purge_exclusions).purge!
+          options = { basedir: @root }
+
+          options[:moduledir]  = @moduledir  if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
+          loaded_content = loader.load!
+          R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                  loaded_content[:desired_contents],
+                                  loaded_content[:purge_exclusions]).purge!
 
           true
         rescue => e

data/lib/r10k/action/runner.rb

@@ -44,10 +44,13 @@ module R10K
 
         overrides = {}
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
-        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
-        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+        if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf')
+          overrides[:deploy] = {}
+          overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+          overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf')
+          overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+          overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash
@@ -67,15 +70,20 @@ module R10K
         exit(8)
       end
 
+      # Set up authorization from license file if it wasn't
+      # already set via the config
       def setup_authorization
-        begin
-          license = R10K::Util::License.load
+        if PuppetForge::Connection.authorization.nil?
+          begin
+            license = R10K::Util::License.load
 
-          if license.respond_to?(:authorization_token)
-            PuppetForge::Connection.authorization = license.authorization_token
+            if license.respond_to?(:authorization_token)
+              logger.debug "Using token from license to connect to the Forge."
+              PuppetForge::Connection.authorization = license.authorization_token
+            end
+          rescue R10K::Error => e
+            logger.warn e.message
           end
-        rescue R10K::Error => e
-          logger.warn e.message
         end
       end
 
@@ -100,11 +108,26 @@ module R10K
       def add_credential_overrides(overrides)
         sshkey_path = @opts[:'private-key']
         token_path = @opts[:'oauth-token']
+        app_id = @opts[:'github-app-id']
+        app_private_key_path = @opts[:'github-app-key']
+        app_ttl = @opts[:'github-app-ttl']
 
         if sshkey_path && token_path
           raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
         end
 
+        if sshkey_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy."
+        end
+
+        if token_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy."
+        end
+
+        if app_id && ! app_private_key_path || app_private_key_path && ! app_id
+          raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy."
+        end
+
         if sshkey_path
           overrides[:git] ||= {}
           overrides[:git][:private_key] = sshkey_path
@@ -121,6 +144,18 @@ module R10K
               repo[:oauth_token] = token_path
             end
           end
+        elsif app_id
+          overrides[:git] ||= {}
+          overrides[:git][:github_app_id] = app_id
+          overrides[:git][:github_app_key] = app_private_key_path
+          overrides[:git][:github_app_ttl] = app_ttl
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:github_app_id] = app_id
+              repo[:github_app_key] = app_private_key_path
+              repo[:github_app_ttl] = app_ttl
+            end
+          end
         end
 
         overrides

data/lib/r10k/action/visitor.rb

@@ -1,4 +1,5 @@
 require 'r10k/errors/formatting'
+require 'r10k/logging'
 
 module R10K
   module Action
@@ -13,6 +14,8 @@ module R10K
     # @api private
     module Visitor
 
+      include R10K::Logging
+
       # Dispatch to the type specific visitor method
       #
       # @param type [Symbol] The object type to dispatch for

data/lib/r10k/cli/deploy.rb

@@ -24,6 +24,7 @@ module R10K::CLI
         option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
+        flag nil, :'exclude-spec', 'Exclude the module\'s spec dir from deployment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
           unless File.executable? value
             $stderr.puts "The specified puppet executable #{value} is not executable."
@@ -34,6 +35,9 @@ module R10K::CLI
         option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
         option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
         option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])
@@ -65,6 +69,7 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
 
           flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
           flag :m, :modules, 'Deploy modules'
+          flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float'
           option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
                  argument: :required
 

data/lib/r10k/cli/puppetfile.rb

@@ -1,5 +1,4 @@
 require 'r10k/cli'
-require 'r10k/puppetfile'
 require 'r10k/action/puppetfile'
 
 require 'cri'

data/lib/r10k/content_synchronizer.rb

@@ -8,24 +8,31 @@ module R10K
     end
 
     def self.serial_sync(modules)
+      updated_modules = []
       modules.each do |mod|
-        mod.sync
+        updated = mod.sync
+        updated_modules << mod.name if updated
       end
+      updated_modules
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
       mods_queue = modules_visit_queue(modules, visitor, loader)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_sync(modules, pool_size, logger)
       mods_queue = modules_sync_queue(modules)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.sync_queue(mods_queue, pool_size, logger)
       logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
-      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
+      updated_modules = Queue.new
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) }
       thread_exception = nil
 
       # If any threads raise an exception the deployment is considered a failure.
@@ -33,6 +40,8 @@ module R10K
       # current work, then re-raise the first exception caught.
       begin
         thread_pool.each(&:join)
+        # Return the list of all modules that were actually updated
+        updated_modules
       rescue => e
         logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
         mods_queue.clear
@@ -65,11 +74,14 @@ module R10K
       modules_by_cachedir.values.each {|mods| queue << mods }
     end
 
-    def self.sync_thread(mods_queue, logger)
+    def self.sync_thread(mods_queue, logger, updated_modules)
       Thread.new do
         begin
           while mods = mods_queue.pop(true) do
-            mods.each { |mod| mod.sync }
+            mods.each do |mod|
+              updated = mod.sync
+              updated_modules << mod.name if updated
+            end
           end
         rescue ThreadError => e
           logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}

data/lib/r10k/environment/base.rb

@@ -1,5 +1,8 @@
-require 'r10k/util/subprocess'
+require 'r10k/content_synchronizer'
 require 'r10k/logging'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
+require 'r10k/util/subprocess'
 
 # This class defines a common interface for environment implementations.
 #
@@ -34,6 +37,10 @@ class R10K::Environment::Base
   #   @return [String] The puppetfile name (relative)
   attr_reader :puppetfile_name
 
+  attr_reader :managed_directories, :desired_contents
+
+  attr_reader :loader
+
   # Initialize the given environment.
   #
   # @param name [String] The unique name describing this environment.
@@ -57,6 +64,20 @@ class R10K::Environment::Base
                                          force: @overrides.dig(:modules, :force),
                                          puppetfile_name: @puppetfile_name})
     @puppetfile.environment = self
+
+    loader_options = { basedir: @full_path, overrides: @overrides, environment: self }
+    loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name
+
+    @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options)
+
+    if @overrides.dig(:environments, :incremental)
+      @loader.load_metadata
+    end
+
+    @base_modules = nil
+    @purge_exclusions = nil
+    @managed_directories = [ @full_path ]
+    @desired_contents = []
   end
 
   # Synchronize the given environment.
@@ -106,8 +127,11 @@ class R10K::Environment::Base
   # @return [Array<R10K::Module::Base>] All modules defined in the Puppetfile
   #   associated with this environment.
   def modules
-    @puppetfile.load
-    @puppetfile.modules
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
+
+    @base_modules
   end
 
   # @return [Array<R10K::Module::Base>] Whether or not the given module
@@ -123,29 +147,50 @@ class R10K::Environment::Base
     end
   end
 
+
+  # Returns a Queue of the names of modules actually updated
   def deploy
-    puppetfile.load(@overrides.dig(:environments, :default_branch_override))
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
 
-    puppetfile.sync
+    if ! @base_modules.empty?
+      pool_size = @overrides.dig(:modules, :pool_size)
+      updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger)
+    end
 
     if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
       logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
-      R10K::Util::Cleaner.new(puppetfile.managed_directories,
-                              puppetfile.desired_contents,
-                              puppetfile.purge_exclusions).purge!
+      @puppetfile_cleaner.purge!
     end
+
+    updated_modules
+  end
+
+  def load_puppetfile_modules
+    loaded_content = @loader.load
+    @base_modules = loaded_content[:modules]
+
+    @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories],
+                                                   loaded_content[:desired_contents])
+
+    @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                                  loaded_content[:desired_contents],
+                                                  loaded_content[:purge_exclusions])
   end
 
   def whitelist(user_whitelist=[])
     user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
   end
 
-  def purge_exclusions
+  def determine_purge_exclusions(pf_managed_dirs     = @puppetfile.managed_directories,
+                                 pf_desired_contents = @puppetfile.desired_contents)
+
     list = [File.join(@full_path, '.r10k-deploy.json')].to_set
 
-    list += @puppetfile.managed_directories
+    list += pf_managed_dirs
 
-    list += @puppetfile.desired_contents.flat_map do |item|
+    list += pf_desired_contents.flat_map do |item|
       desired_tree = []
 
       if File.directory?(item)
@@ -163,6 +208,14 @@ class R10K::Environment::Base
     list.to_a
   end
 
+  def purge_exclusions
+    if @purge_exclusions.nil?
+      load_puppetfile_modules
+    end
+
+    @purge_exclusions
+  end
+
   def generate_types!
     argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
     subproc = R10K::Util::Subprocess.new(argv)