r10k 3.9.2 → 3.12.0

data/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb

@@ -24,7 +24,7 @@ stage_env_notify_message = 'This is a different message'
 stage_env_notify_message_regex = /#{stage_env_notify_message}/
 
 #Verification for "test" Environment
-test_env_error_message_regex = /Error:.*Could not.*environment '?test'?/
+test_env_message_regex = /Environment 'test' not found on server/
 
 #Verification for "temp" Environment
 test_env_notify_message_regex = /I am in the temp environment/
@@ -157,7 +157,7 @@ agents.each do |agent|
   end
 
   step 'Attempt to Run Puppet Agent Against "test" Environment'
-  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result|
-    assert_match(test_env_error_message_regex, result.stderr, 'Expected error was not detected!')
+  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result|
+    assert_match(test_env_message_regex, result.stdout, 'Expected message not found!')
   end
 end

data/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb

@@ -15,7 +15,7 @@ initial_env_names = ['production', 'stage']
 
 #Verification
 notify_message_regex = /I am in the production environment/
-stage_env_error_message_regex = /Error:.*Could not.*environment '?stage'?/
+stage_env_message_regex = /Environment 'stage' not found on server/
 
 #Manifest
 site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
@@ -83,8 +83,8 @@ agents.each do |agent|
   end
 
   step 'Attempt to Run Puppet Agent Against "stage" Environment'
-  on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 1) do |result|
-    assert_match(stage_env_error_message_regex, result.stderr, 'Expected error was not detected!')
+  on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 2) do |result|
+    assert_match(stage_env_message_regex, result.stdout, 'Expected message not found!')
   end
 end
 

data/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb

@@ -22,7 +22,7 @@ site_pp = create_site_pp(master_certname, '  include helloworld')
 notify_message_prod_env_regex = /I am in the production environment/
 notify_message_test_env_regex = /I am in the test environment/
 removal_message_test_env_regex = /Removing unmanaged path.*test/
-error_message_regex = /Could not retrieve (catalog from remote server|information from environment test)/
+missing_message_regex = /Environment 'test' not found on server/
 
 #Teardown
 teardown do
@@ -72,7 +72,7 @@ end
 
 agents.each do |agent|
   step 'Run Puppet Agent Against "test" Environment'
-  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result|
-    assert_match(error_message_regex, result.stderr, 'Expected message not found!')
+  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result|
+    assert_match(missing_message_regex, result.stdout, 'Expected message not found!')
   end
 end

data/lib/r10k/action/deploy/environment.rb

@@ -41,11 +41,14 @@ module R10K
                 requested_environments: @argv.map { |arg| arg.gsub(/\W/,'_') },
                 default_branch_override: @default_branch_override,
                 generate_types: @generate_types || settings.dig(:deploy, :generate_types) || false,
-                preload_environments: true
+                preload_environments: true,
+                incremental: @incremental
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
                 requested_modules: [],
                 deploy_modules: @modules,
+                pool_size: @settings[:pool_size] || 4,
                 force: !@no_force, # force here is used to make it easier to reason about
               },
               purging: {
@@ -53,6 +56,9 @@ module R10K
                 purge_allowlist: read_purge_allowlist(settings.dig(:deploy, :purge_whitelist) || [],
                                                       settings.dig(:deploy, :purge_allowlist) || [])
               },
+              forge: {
+                allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false
+              },
               output: {}
             }
           })
@@ -116,6 +122,9 @@ module R10K
 
           if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment)
             logger.debug("Purging unmanaged environments for deployment...")
+            deployment.sources.each do |source|
+              source.reload!
+            end
             deployment.purge!
           end
         ensure
@@ -126,6 +135,7 @@ module R10K
               envs.reject! { |e| !requested_envs.include?(e) } if requested_envs.any?
               postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs.join(' ')) }
             end
+            logger.debug _("Executing postrun command.")
             subproc = R10K::Util::Subprocess.new(postcmd)
             subproc.logger = logger
             subproc.execute
@@ -233,13 +243,18 @@ module R10K
           super.merge(puppetfile: :modules,
                       modules: :self,
                       cachedir: :self,
+                      incremental: :self,
                       'no-force': :self,
+                      'exclude-spec': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
                       'oauth-token': :self,
-                      'default-branch-override': :self)
+                      'default-branch-override': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/deploy/module.rb

@@ -29,6 +29,7 @@ module R10K
           super
 
           requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : []
+          @modified_envs = []
 
           @settings = @settings.merge({
             overrides: {
@@ -37,10 +38,15 @@ module R10K
                 generate_types: @generate_types
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
+                pool_size: @settings[:pool_size] || 4,
                 requested_modules: @argv.map.to_a,
                 # force here is used to make it easier to reason about
                 force: !@no_force
               },
+              forge: {
+                allow_puppetfile_override: settings.dig(:forge, :allow_puppetfile_override) || false
+              },
               purging: {},
               output: {}
             }
@@ -69,6 +75,23 @@ module R10K
 
         def visit_deployment(deployment)
           yield
+        ensure
+          if (postcmd = @settings[:postrun])
+            if @modified_envs.any?
+              envs_to_run = @modified_envs.join(' ')
+              logger.debug _("Running postrun command for environments: %{envs_to_run}.") % { envs_to_run: envs_to_run }
+
+              if postcmd.grep('$modifiedenvs').any?
+                postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) }
+              end
+
+              subproc = R10K::Util::Subprocess.new(postcmd)
+              subproc.logger = logger
+              subproc.execute
+            else
+              logger.debug _("No environments were modified, not executing postrun command.")
+            end
+          end
         end
 
         def visit_source(source)
@@ -82,13 +105,17 @@ module R10K
           else
             logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @settings.dig(:overrides, :modules, :requested_modules).inspect, env_path: environment.path})
 
-            environment.deploy
+            updated_modules = environment.deploy
+
+            # We actually synced a module in this env
+            if !updated_modules.nil? && !updated_modules.empty?
+              # Record modified environment for postrun command
+              @modified_envs << environment.dirname
 
-            requested_mods = @settings.dig(:overrides, :modules, :requested_modules) || []
-            generate_types = @settings.dig(:overrides, :environments, :generate_types)
-            if generate_types && !((environment.modules.map(&:name) & requested_mods).empty?)
-              logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
-              environment.generate_types!
+              if generate_types = @settings.dig(:overrides, :environments, :generate_types)
+                logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
+                environment.generate_types!
+              end
             end
           end
         end
@@ -96,12 +123,16 @@ module R10K
         def allowed_initialize_opts
           super.merge(environment: true,
                       cachedir: :self,
+                      'exclude-spec': :self,
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
-                      'oauth-token': :self)
+                      'oauth-token': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/puppetfile/check.rb

@@ -1,6 +1,6 @@
-require 'r10k/puppetfile'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
 
 module R10K
   module Action
@@ -8,11 +8,13 @@ module R10K
       class Check < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
+          options = { basedir: @root }
+          options[:moduledir] = @moduledir if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
           begin
-            pf.load!
+            loader.load!
             $stderr.puts _("Syntax OK")
             true
           rescue => e

data/lib/r10k/action/puppetfile/install.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/errors/formatting'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
+require 'r10k/content_synchronizer'
+require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
 require 'r10k/util/cleaner'
 
 module R10K
@@ -10,25 +10,31 @@ module R10K
       class Install < R10K::Action::Base
 
         def call
-          @ok = true
           begin
-            pf = R10K::Puppetfile.new(@root,
-                                      {moduledir: @moduledir,
-                                       puppetfile_path: @puppetfile,
-                                       force: @force || false})
-            pf.load!
-            pf.sync
+            options = { basedir: @root, overrides: { force: @force || false } }
+            options[:moduledir]  = @moduledir  if @moduledir
+            options[:puppetfile] = @puppetfile if @puppetfile
+
+            loader = R10K::ModuleLoader::Puppetfile.new(**options)
+            loaded_content = loader.load!
 
-            R10K::Util::Cleaner.new(pf.managed_directories,
-                                    pf.desired_contents,
-                                    pf.purge_exclusions).purge!
+            pool_size = @settings[:pool_size] || 4
+            modules   = loaded_content[:modules]
+            if pool_size > 1
+              R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger)
+            else
+              R10K::ContentSynchronizer.serial_sync(modules, logger)
+            end
 
+            R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                    loaded_content[:desired_contents],
+                                    loaded_content[:purge_exclusions]).purge!
+
+            true
           rescue => e
-            @ok = false
             logger.error R10K::Errors::Formatting.format_exception(e, @trace)
+            false
           end
-
-          @ok
         end
 
         private

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,13 +9,16 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
-          pf.load!
-          R10K::Util::Cleaner.new(pf.managed_directories,
-                                  pf.desired_contents,
-                                  pf.purge_exclusions).purge!
+          options = { basedir: @root }
+
+          options[:moduledir]  = @moduledir  if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
+          loaded_content = loader.load!
+          R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                  loaded_content[:desired_contents],
+                                  loaded_content[:purge_exclusions]).purge!
 
           true
         rescue => e

data/lib/r10k/action/runner.rb

@@ -44,10 +44,13 @@ module R10K
 
         overrides = {}
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
-        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
-        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+        if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf')
+          overrides[:deploy] = {}
+          overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+          overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf')
+          overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+          overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash
@@ -67,15 +70,20 @@ module R10K
         exit(8)
       end
 
+      # Set up authorization from license file if it wasn't
+      # already set via the config
       def setup_authorization
-        begin
-          license = R10K::Util::License.load
+        if PuppetForge::Connection.authorization.nil?
+          begin
+            license = R10K::Util::License.load
 
-          if license.respond_to?(:authorization_token)
-            PuppetForge::Connection.authorization = license.authorization_token
+            if license.respond_to?(:authorization_token)
+              logger.debug "Using token from license to connect to the Forge."
+              PuppetForge::Connection.authorization = license.authorization_token
+            end
+          rescue R10K::Error => e
+            logger.warn e.message
           end
-        rescue R10K::Error => e
-          logger.warn e.message
         end
       end
 
@@ -100,11 +108,26 @@ module R10K
       def add_credential_overrides(overrides)
         sshkey_path = @opts[:'private-key']
         token_path = @opts[:'oauth-token']
+        app_id = @opts[:'github-app-id']
+        app_private_key_path = @opts[:'github-app-key']
+        app_ttl = @opts[:'github-app-ttl']
 
         if sshkey_path && token_path
           raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
         end
 
+        if sshkey_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy."
+        end
+
+        if token_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy."
+        end
+
+        if app_id && ! app_private_key_path || app_private_key_path && ! app_id
+          raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy."
+        end
+
         if sshkey_path
           overrides[:git] ||= {}
           overrides[:git][:private_key] = sshkey_path
@@ -121,6 +144,18 @@ module R10K
               repo[:oauth_token] = token_path
             end
           end
+        elsif app_id
+          overrides[:git] ||= {}
+          overrides[:git][:github_app_id] = app_id
+          overrides[:git][:github_app_key] = app_private_key_path
+          overrides[:git][:github_app_ttl] = app_ttl
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:github_app_id] = app_id
+              repo[:github_app_key] = app_private_key_path
+              repo[:github_app_ttl] = app_ttl
+            end
+          end
         end
 
         overrides

data/lib/r10k/cli/deploy.rb

@@ -24,6 +24,7 @@ module R10K::CLI
         option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
+        flag nil, :'exclude-spec', 'Exclude the module\'s spec dir from deployment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
           unless File.executable? value
             $stderr.puts "The specified puppet executable #{value} is not executable."
@@ -34,6 +35,9 @@ module R10K::CLI
         option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
         option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
         option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])
@@ -65,6 +69,7 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
 
           flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
           flag :m, :modules, 'Deploy modules'
+          flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float'
           option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
                  argument: :required
 

data/lib/r10k/cli/puppetfile.rb

@@ -1,5 +1,4 @@
 require 'r10k/cli'
-require 'r10k/puppetfile'
 require 'r10k/action/puppetfile'
 
 require 'cri'