r10k 3.10.0 → 3.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db8e1373846da121e13c6bdd3f74c7eadac79d356d65628157d1f0b6c0939f23
-  data.tar.gz: 7fd4694a5e33f692213339db0422fa7db58d38f84d63f2586593665f8acae8cc
+  metadata.gz: 34dc5753178d4777a6d36e6806515345b8b0e133807d46b21a9468de0325a1ed
+  data.tar.gz: 830085cca185bb00292af40f5cc13e8a46d00645a7bdc7f16e0d48b98ca4df23
 SHA512:
-  metadata.gz: 272021e80cc4ce7ba4d11f3316bd972d7498e102a1f4d4aa9c0bcf83eeef60bd283c9fb4db74ad37e854b2ebc60f6ba28d3c251ae51cb54fb2ae03a4842d4bb8
-  data.tar.gz: 3689ec04b7dd2a4597199fd2fddba3e06a903b682760aacd565f96e42a604c42a6ef340f598c37ee4e1c1cfc4f4912c1132880de98bb734d86aac39475a45342
+  metadata.gz: 26e01c3ae5cdd4b4c873aa04d233dbe8fbe1f1f9fa193add5fe03aba50d1db09604fc117fd26190b7a1c350a7a64df9bdeba1b33ee0697d90420813075fe7811
+  data.tar.gz: 84c8c37b4daa9f11a864c54df12834d7bc920207f5f1219516d49583ff88537b7a13772a0c812d2bc9ebb4ebd636da4124dd35fbc105681e36db1aee4f540961

data/.travis.yml

@@ -10,18 +10,8 @@ jdk:
 before_install: gem install bundler -v '< 2' --no-document
 matrix:
   include:
-    - stage: r10k tests
-      rvm: 2.7.0
-    - stage: r10k tests
-      rvm: 2.6.5
-    - stage: r10k tests
-      rvm: 2.5.0
-    - stage: r10k tests
-      rvm: 2.4.0
     - stage: r10k tests
       rvm: 2.3.0
-    - stage: r10k tests
-      rvm: jruby
     - stage: r10k container tests
       dist: focal
       language: ruby

data/CHANGELOG.mkd

@@ -4,6 +4,15 @@ CHANGELOG
 Unreleased
 ----------
 
+3.11.0
+------
+
+- Always sync git cache on `ref: 'HEAD'` [#1182](https://github.com/puppetlabs/r10k/pull/1182)
+- (CODEMGMT-1421, CODEMGMT-1422, CODEMGMT-1457) Add setting `exclude_spec` to remove the spec dir from module deployment[#1189](https://github.com/puppetlabs/r10k/pull/1189)[#1198](https://github.com/puppetlabs/r10k/pull/1198)[#1204](https://github.com/puppetlabs/r10k/pull/1204)
+- (RK-369) Make module deploys run the postrun command if any environments were updated. [#982](https://github.com/puppetlabs/r10k/issues/982)
+- Add support for Github App auth token. This allows r10k to authenticate under strict SSO/2FA guidelines that cannot utilize machine users for code deployment. [#1180](https://github.com/puppetlabs/r10k/pull/1180)
+- Restore the ability to load a Puppetfile from a relative `basedir`. [#1202](https://github.com/puppetlabs/r10k/pull/1202), [#1203](https://github.com/puppetlabs/r10k/pull/1203)
+
 3.10.0
 ------
 

data/README.mkd

@@ -66,6 +66,12 @@ a git repository using Bundler for dependencies:
     bundle install
     bundle exec r10k help
 
+### Arch Linux
+
+Arch Linux provides a [system package](https://archlinux.org/packages/community/any/r10k/) for r10k.
+This is built against the [system Ruby](https://archlinux.org/packages/extra/x86_64/ruby/) (which is Ruby 3.0.2 as of 2021-08-03).
+This package is maintained by [Tim Meusel](https://github.com/bastelfreak).
+
 Usage
 -----
 

data/doc/dynamic-environments/configuration.mkd

@@ -343,6 +343,20 @@ deploy:
   puppet_conf: '/opt/puppet/conf/puppet.conf'
 ```
 
+#### exclude_spec
+
+During module deployment, r10k's default behavior is to deploy the spec directory.
+This behavior can be configured for all modules using the `exclude_spec` setting in the r10k config.
+This config setting can be overridden in each module definition in the Puppetfile
+or more globally via the CLI for deploys. The following example sets all modules to
+not deploy the spec dir.
+
+
+```yaml
+deploy:
+  exclude_spec: true
+```
+
 Source options
 --------------
 

data/doc/puppetfile.mkd

@@ -130,7 +130,8 @@ with a git commit, branch reference, or a tag.
 When a module is installed using `:tag` or `:commit`, r10k assumes that the
 given object is a tag or commit and can do some optimizations around fetching
 the object. If the tag or commit is already available r10k will skip network
-operations when updating the repo, which can speed up install times.
+operations when updating the repo, which can speed up install times. When
+`:ref` is set to track `HEAD`, it will synchronize the module on each run.
 
 Module versions can also be specified using `:branch` to track a specific
 branch reference.
@@ -312,6 +313,19 @@ module.
 For more information see the [FAQ entry](faq.mkd#how-do-i-prevent-r10k-from-removing-modules-in-the-modules-directory-of-my-git-repository)
 on managing internal and external modules in the same directory.
 
+### Per-Item spec dir deployment
+
+During deployment, r10k's default behavior is to deploy the spec directory. The
+Puppetfile can modify this per module, overriding settings from the default
+r10k config. The following example sets the module to not deploy the spec
+directory.
+
+```
+mod 'apache',
+  :git => 'git@github.com:puppetlabs/puppetlabs-apache.git',
+  :exclude_spec => true
+```
+
 ### Per-Item Install Path
 
 Git and SVN content types support installing into an alternate path without changing

data/integration/Rakefile

@@ -1,4 +1,5 @@
 require 'rototiller'
+require 'fileutils'
 
 namespace :ci do
   namespace :test do
@@ -59,6 +60,7 @@ end
 desc 'Generate a host configuration used by Beaker'
 rototiller_task :beaker_hostgenerator do |t|
   if ENV['BEAKER_HOST'].nil?
+    FileUtils.mkdir_p 'configs'
     t.add_command do |c|
       c.name = 'beaker-hostgenerator'
       c.argument = '> configs/generated'

data/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb

@@ -5,9 +5,11 @@ test_name 'CODEMGMT-78 - Puppetfile Purge --puppetfile & --moduledir flag usage'
 
 #Init
 master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip
-moduledir = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip + '/production/modules'
-git_environments_path = '/root/environments'
-last_commit = git_last_commit(master, git_environments_path)
+environments_path = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip
+moduledir = File.join(environments_path, 'production', 'modules')
+puppetfile_path = File.join(environments_path, 'production', 'Puppetfile')
+git_remote_environments_path = '/root/environments'
+last_commit = git_last_commit(master, git_remote_environments_path)
 r10k_fqp = get_r10k_fqp(master)
 
 #Verification
@@ -16,11 +18,11 @@ motd_contents = 'Hello!'
 motd_contents_regex = /\A#{motd_contents}\z/
 
 #File
-puppet_file = <<-PUPPETFILE
+puppetfile = <<-PUPPETFILE
 mod "puppetlabs/xinetd"
 PUPPETFILE
 
-puppet_file_path = File.join(git_environments_path, 'Puppetfile')
+remote_puppetfile_path = File.join(git_remote_environments_path, 'Puppetfile')
 
 #Manifest
 manifest = <<-MANIFEST
@@ -29,12 +31,12 @@ manifest = <<-MANIFEST
   }
 MANIFEST
 
-site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
+remote_site_pp_path = File.join(git_remote_environments_path, 'manifests', 'site.pp')
 site_pp = create_site_pp(master_certname, manifest)
 
 #Teardown
 teardown do
-  clean_up_r10k(master, last_commit, git_environments_path)
+  clean_up_r10k(master, last_commit, git_remote_environments_path)
 
   step 'Remove "/etc/motd" File'
   on(agents, "rm -rf #{motd_path}")
@@ -45,19 +47,19 @@ step 'Stub Forge on Master'
 stub_forge_on(master)
 
 step 'Checkout "production" Branch'
-git_on(master, 'checkout production', git_environments_path)
+git_on(master, 'checkout production', git_remote_environments_path)
 
 step 'Create "Puppetfile" for the "production" Environment'
-create_remote_file(master, puppet_file_path, puppet_file)
+create_remote_file(master, remote_puppetfile_path, puppetfile)
 
 step 'Inject New "site.pp" to the "production" Environment'
-inject_site_pp(master, site_pp_path, site_pp)
+inject_site_pp(master, remote_site_pp_path, site_pp)
 
 step 'Push Changes'
-git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path)
+git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_remote_environments_path)
 
 step 'Deploy Environments via r10k'
-on(master, "#{r10k_fqp} deploy environment -v -p")
+on(master, "#{r10k_fqp} deploy environment --modules --verbose debug --trace")
 
 step 'Manually Install the "motd" Module from the Forge'
 on(master, puppet("module install puppetlabs-motd --modulepath #{moduledir}"))
@@ -76,7 +78,7 @@ agents.each do |agent|
 end
 
 step 'Use r10k to Purge Unmanaged Modules'
-on(master, "#{r10k_fqp} puppetfile purge -v --puppetfile #{puppet_file_path} --moduledir #{moduledir}")
+on(master, "#{r10k_fqp} puppetfile purge --puppetfile #{puppetfile_path} --moduledir #{moduledir} --verbose debug --trace")
 
 #Agent will fail because r10k will purge the "motd" module
 agents.each do |agent|

data/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb

@@ -24,7 +24,7 @@ stage_env_notify_message = 'This is a different message'
 stage_env_notify_message_regex = /#{stage_env_notify_message}/
 
 #Verification for "test" Environment
-test_env_error_message_regex = /Error:.*Could not.*environment '?test'?/
+test_env_message_regex = /Environment 'test' not found on server/
 
 #Verification for "temp" Environment
 test_env_notify_message_regex = /I am in the temp environment/
@@ -157,7 +157,7 @@ agents.each do |agent|
   end
 
   step 'Attempt to Run Puppet Agent Against "test" Environment'
-  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result|
-    assert_match(test_env_error_message_regex, result.stderr, 'Expected error was not detected!')
+  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result|
+    assert_match(test_env_message_regex, result.stdout, 'Expected message not found!')
   end
 end

data/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb

@@ -15,7 +15,7 @@ initial_env_names = ['production', 'stage']
 
 #Verification
 notify_message_regex = /I am in the production environment/
-stage_env_error_message_regex = /Error:.*Could not.*environment '?stage'?/
+stage_env_message_regex = /Environment 'stage' not found on server/
 
 #Manifest
 site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
@@ -83,8 +83,8 @@ agents.each do |agent|
   end
 
   step 'Attempt to Run Puppet Agent Against "stage" Environment'
-  on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 1) do |result|
-    assert_match(stage_env_error_message_regex, result.stderr, 'Expected error was not detected!')
+  on(agent, puppet('agent', '--test', '--environment stage'), :acceptable_exit_codes => 2) do |result|
+    assert_match(stage_env_message_regex, result.stdout, 'Expected message not found!')
   end
 end
 

data/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb

@@ -22,7 +22,7 @@ site_pp = create_site_pp(master_certname, '  include helloworld')
 notify_message_prod_env_regex = /I am in the production environment/
 notify_message_test_env_regex = /I am in the test environment/
 removal_message_test_env_regex = /Removing unmanaged path.*test/
-error_message_regex = /Could not retrieve (catalog from remote server|information from environment test)/
+missing_message_regex = /Environment 'test' not found on server/
 
 #Teardown
 teardown do
@@ -72,7 +72,7 @@ end
 
 agents.each do |agent|
   step 'Run Puppet Agent Against "test" Environment'
-  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 1) do |result|
-    assert_match(error_message_regex, result.stderr, 'Expected message not found!')
+  on(agent, puppet('agent', '--test', '--environment test'), :acceptable_exit_codes => 2) do |result|
+    assert_match(missing_message_regex, result.stdout, 'Expected message not found!')
   end
 end

data/lib/r10k/action/deploy/environment.rb

@@ -44,6 +44,7 @@ module R10K
                 preload_environments: true
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
                 requested_modules: [],
                 deploy_modules: @modules,
                 force: !@no_force, # force here is used to make it easier to reason about
@@ -237,12 +238,16 @@ module R10K
                       modules: :self,
                       cachedir: :self,
                       'no-force': :self,
+                      'exclude-spec': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
                       'oauth-token': :self,
-                      'default-branch-override': :self)
+                      'default-branch-override': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/deploy/module.rb

@@ -29,6 +29,7 @@ module R10K
           super
 
           requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : []
+          @modified_envs = []
 
           @settings = @settings.merge({
             overrides: {
@@ -37,6 +38,7 @@ module R10K
                 generate_types: @generate_types
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
                 requested_modules: @argv.map.to_a,
                 # force here is used to make it easier to reason about
                 force: !@no_force
@@ -69,6 +71,21 @@ module R10K
 
         def visit_deployment(deployment)
           yield
+        ensure
+          if (postcmd = @settings[:postrun])
+            if @modified_envs.any?
+              if postcmd.grep('$modifiedenvs').any?
+                envs_to_run = @modified_envs.join(' ')
+                logger.debug "Running postrun command for environments #{envs_to_run}."
+                postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) }
+              end
+              subproc = R10K::Util::Subprocess.new(postcmd)
+              subproc.logger = logger
+              subproc.execute
+            else
+              logger.debug "No environments were modified, not executing postrun command."
+            end
+          end
         end
 
         def visit_source(source)
@@ -85,10 +102,15 @@ module R10K
             environment.deploy
 
             requested_mods = @settings.dig(:overrides, :modules, :requested_modules) || []
-            generate_types = @settings.dig(:overrides, :environments, :generate_types)
-            if generate_types && !((environment.modules.map(&:name) & requested_mods).empty?)
-              logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
-              environment.generate_types!
+            # We actually synced a module in this env
+            if !((environment.modules.map(&:name) & requested_mods).empty?)
+              # Record modified environment for postrun command
+              @modified_envs << environment.dirname
+
+              if generate_types = @settings.dig(:overrides, :environments, :generate_types)
+                logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
+                environment.generate_types!
+              end
             end
           end
         end
@@ -96,12 +118,16 @@ module R10K
         def allowed_initialize_opts
           super.merge(environment: true,
                       cachedir: :self,
+                      'exclude-spec': :self,
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
-                      'oauth-token': :self)
+                      'oauth-token': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -44,10 +44,13 @@ module R10K
 
         overrides = {}
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
-        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
-        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+        if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf')
+          overrides[:deploy] = {}
+          overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+          overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf')
+          overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+          overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash
@@ -105,11 +108,26 @@ module R10K
       def add_credential_overrides(overrides)
         sshkey_path = @opts[:'private-key']
         token_path = @opts[:'oauth-token']
+        app_id = @opts[:'github-app-id']
+        app_private_key_path = @opts[:'github-app-key']
+        app_ttl = @opts[:'github-app-ttl']
 
         if sshkey_path && token_path
           raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
         end
 
+        if sshkey_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy."
+        end
+
+        if token_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy."
+        end
+
+        if app_id && ! app_private_key_path || app_private_key_path && ! app_id
+          raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy."
+        end
+
         if sshkey_path
           overrides[:git] ||= {}
           overrides[:git][:private_key] = sshkey_path
@@ -126,6 +144,18 @@ module R10K
               repo[:oauth_token] = token_path
             end
           end
+        elsif app_id
+          overrides[:git] ||= {}
+          overrides[:git][:github_app_id] = app_id
+          overrides[:git][:github_app_key] = app_private_key_path
+          overrides[:git][:github_app_ttl] = app_ttl
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:github_app_id] = app_id
+              repo[:github_app_key] = app_private_key_path
+              repo[:github_app_ttl] = app_ttl
+            end
+          end
         end
 
         overrides

data/lib/r10k/cli/deploy.rb

@@ -24,6 +24,7 @@ module R10K::CLI
         option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
+        flag nil, :'exclude-spec', 'Exclude the module\'s spec dir from deployment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
           unless File.executable? value
             $stderr.puts "The specified puppet executable #{value} is not executable."
@@ -34,6 +35,9 @@ module R10K::CLI
         option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
         option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
         option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])