r10k 3.9.0 → 3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e658209eb66330ab2c2cea4dd4fac0690b6700042e639e5ff11d2fffb120f1b
-  data.tar.gz: 870249f76ef6c2b289e3255e6c752d6fc725009a8e37f8b4219f7496a7c0a439
+  metadata.gz: db8e1373846da121e13c6bdd3f74c7eadac79d356d65628157d1f0b6c0939f23
+  data.tar.gz: 7fd4694a5e33f692213339db0422fa7db58d38f84d63f2586593665f8acae8cc
 SHA512:
-  metadata.gz: a01dfae93025bac535c4e18d5f15b800c7bc1df1aa79cf0a2ea565c034767d5c5f0b0e4b41b7954e59310090a25ca67e28e92ab416958b2f91864a3da873d93a
-  data.tar.gz: 0631fe1846185c2808bd14be2cca3b182402991476950bae18da0fb19129eb7139fbec3b1f9a3bf1568f09931d0438c9a014380378cce71b01d7b9022364cb7c
+  metadata.gz: 272021e80cc4ce7ba4d11f3316bd972d7498e102a1f4d4aa9c0bcf83eeef60bd283c9fb4db74ad37e854b2ebc60f6ba28d3c251ae51cb54fb2ae03a4842d4bb8
+  data.tar.gz: 3689ec04b7dd2a4597199fd2fddba3e06a903b682760aacd565f96e42a604c42a6ef340f598c37ee4e1c1cfc4f4912c1132880de98bb734d86aac39475a45342

data/.github/pull_request_template.md

@@ -1,4 +1,4 @@
 Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format:
 ```
-- Summary of changes. [Issue or PR #](link to issue or PR)
+- (JIRA ticket) Summary of changes. [Issue or PR #](link to issue or PR)
 ```

data/.github/workflows/rspec_tests.yml

@@ -15,7 +15,7 @@ jobs:
           - {os: ubuntu-18.04, ruby: 2.5}
           - {os: ubuntu-18.04, ruby: 2.6}
           - {os: ubuntu-18.04, ruby: 2.7}
-          - {os: ubuntu-18.04, ruby: jruby-9.2.9.0}
+          - {os: ubuntu-18.04, ruby: jruby-9.2.10.0}
           - {os: windows-2016, ruby: 2.5}
           - {os: windows-2016, ruby: 2.6}
           - {os: windows-2016, ruby: 2.7}

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues
+
+on:
+  schedule:
+  - cron: "30 1 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 60
+        days-before-close: 7
+        stale-issue-message: 'This issue has been marked stale because it has had no activity for 60 days. The Puppet Team is actively prioritizing existing bugs and new features, if this issue is still important to you please comment and we will add this to our backlog to complete. Otherwise, it will be closed in 7 days.'
+        stale-issue-label: 'stale'
+        stale-pr-message: "This PR has been marked stale because it has had no activity for 60 days. If you are still interested in getting this merged, please comment and we'll try to move it forward. Otherwise, it will be closed in 7 days."
+        stale-pr-label: 'stale'

data/CHANGELOG.mkd

@@ -4,6 +4,30 @@ CHANGELOG
 Unreleased
 ----------
 
+3.10.0
+------
+
+- Add `authorization_token` setting to allow authentication to a custom Forge server. [#1181](https://github.com/puppetlabs/r10k/pull/1181)
+- (RK-135) Attempting to download the latest version for a module that has no Forge releases will now issue a meaningful error. [#1177](https://github.com/puppetlabs/r10k/pull/1177)
+- Added an interface to R10K::Source::Base named `reload!` for updating the environments list for a given deployment; `reload!` is called before deployment purges to make r10k deploy pools more threadsafe. [#1172](https://github.com/puppetlabs/r10k/pull/1172)
+- Remove username and password from remote url in cache directory name [#1186](https://github.com/puppetlabs/r10k/pull/1186)
+- Purging efficiency is greatly improved. R10K will no longer recurse into directories that match recursive purge exclusions. This should significantly improve the deploy times for those users who enable the "environment" purge level. [#1178](https://github.com/puppetlabs/r10k/pull/1178)
+
+3.9.3
+-----
+
+- Fixes a regression when using `--default_branch_override` with Puppetfiles containing Forge modules. [#1173](https://github.com/puppetlabs/r10k/issues/1173)
+
+3.9.2
+-----
+
+- Makes the third parameter to R10K::Actions optional, restoring backwards compatability broken in 3.9.1.
+
+3.9.1
+-----
+
+- Invalid module specifications in a Puppetfile will cause the R10K run to abort earlier than before. Prior to this release, the R10K run would complete, sync all other modules, and return an exit code of 1. R10K will now stop syncing modules and abort immediately. [#1161](https://github.com/puppetlabs/r10k/pull/1161)
+
 3.9.0
 -----
 

data/doc/dynamic-environments/configuration.mkd

@@ -165,9 +165,16 @@ interactions. See the global proxy setting documentation for more information an
 The 'baseurl' setting indicates where Forge modules should be installed from.
 This defaults to 'https://forgeapi.puppetlabs.com'
 
+#### authorization_token
+
+The 'authorization_token' setting allows you to provide a token for authenticating to a
+custom Forge server. When set, 'baseurl' must also be set.
+You will need to prepend your token with 'Bearer ' if using Artifactory as your Forge server.
+
 ```yaml
 forge:
   baseurl: 'https://private-forge.mysite'
+  authorization_token: 'Bearer mysupersecretauthtoken'
 ```
 
 Deployment options
@@ -586,13 +593,13 @@ When using the YAML source type, every environment is enumerated in a single yam
 ---
 production:
   type: git
-  remote: git@github.com:puppetlabs/control-repo.git
-  ref: 8820892
+  source: git@github.com:puppetlabs/control-repo.git
+  version: 8820892
 
 development:
   type: git
-  remote: git@github.com:puppetlabs/control-repo.git
-  ref: 8820892
+  source: git@github.com:puppetlabs/control-repo.git
+  version: 8820892
 ```
 
 ### YAMLdir Environment Source
@@ -623,8 +630,8 @@ The contents of the file should be a hash specifying the enviornment type, and a
 # production.yaml
 ---
 type: git
-remote: git@github.com:puppetlabs/control-repo.git
-ref: 8820892
+source: git@github.com:puppetlabs/control-repo.git
+version: 8820892
 ```
 
 ### Exec environment Source

data/integration/Rakefile

@@ -65,7 +65,7 @@ rototiller_task :beaker_hostgenerator do |t|
     end
 
     # This is a hack :(
-    t.add_flag(:name => '', :default => 'centos6-64mdca-64.fa', :override_env => 'TEST_TARGET')
+    t.add_flag(:name => '', :default => 'centos7-64mdca-64.fa', :override_env => 'TEST_TARGET')
 
     t.add_flag(:name => '--global-config', :default => '{forge_host=forge-aio01-petest.puppetlabs.com}', :override_env => 'BHG_GLOBAL_CONFIG')
   end

data/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb

@@ -10,7 +10,7 @@ last_commit = git_last_commit(master, git_environments_path)
 r10k_fqp = get_r10k_fqp(master)
 
 #Verification
-error_notification_regex = /Does 'puppetlabs-regret' have at least one published release?/
+error_notification_regex = /The module puppetlabs-regret does not appear to have any published releases/
 
 #File
 puppet_file = <<-PUPPETFILE
@@ -40,12 +40,6 @@ git_add_commit_push(master, 'production', 'Add module.', git_environments_path)
 
 #Tests
 step "Deploy production environment via r10k with specified module deleted"
-on(master, "#{r10k_fqp} deploy environment -p -v", :acceptable_exit_codes => 1) do |result|
-  if get_puppet_version(master) < 4.0
-    assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
-  else
-    expect_failure('expected to fail due to RK-135') do
-      assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
-    end
-  end
+on(master, "#{r10k_fqp} deploy environment -p -v --trace", :acceptable_exit_codes => 1) do |result|
+  assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
 end

data/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb

@@ -1,10 +1,11 @@
 require 'git_utils'
 require 'r10k_utils'
 require 'master_manipulator'
-test_name 'CODEMGMT-73 - C63184 - Single Environment Purge Unmanaged Modules'
+test_name 'CODEMGMT-78 - Puppetfile Purge --puppetfile & --moduledir flag usage'
 
 #Init
 master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip
+moduledir = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip + '/production/modules'
 git_environments_path = '/root/environments'
 last_commit = git_last_commit(master, git_environments_path)
 r10k_fqp = get_r10k_fqp(master)
@@ -14,8 +15,6 @@ motd_path = '/etc/motd'
 motd_contents = 'Hello!'
 motd_contents_regex = /\A#{motd_contents}\z/
 
-error_message_regex = /Blah/
-
 #File
 puppet_file = <<-PUPPETFILE
 mod "puppetlabs/xinetd"
@@ -48,9 +47,6 @@ stub_forge_on(master)
 step 'Checkout "production" Branch'
 git_on(master, 'checkout production', git_environments_path)
 
-step 'Manually Install the "motd" Module from the Forge'
-on(master, puppet('module install puppetlabs-motd'))
-
 step 'Create "Puppetfile" for the "production" Environment'
 create_remote_file(master, puppet_file_path, puppet_file)
 
@@ -60,13 +56,13 @@ inject_site_pp(master, site_pp_path, site_pp)
 step 'Push Changes'
 git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path)
 
-#Tests
 step 'Deploy Environments via r10k'
 on(master, "#{r10k_fqp} deploy environment -v -p")
 
-step 'Plug-in Sync Agents'
-on(agents, puppet("plugin download --server #{master}"))
+step 'Manually Install the "motd" Module from the Forge'
+on(master, puppet("module install puppetlabs-motd --modulepath #{moduledir}"))
 
+#Tests
 agents.each do |agent|
   step 'Run Puppet Agent Against "production" Environment'
   on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result|
@@ -80,14 +76,12 @@ agents.each do |agent|
 end
 
 step 'Use r10k to Purge Unmanaged Modules'
-on(master, "#{r10k_fqp} puppetfile purge -v", :acceptable_exit_codes => 1)
+on(master, "#{r10k_fqp} puppetfile purge -v --puppetfile #{puppet_file_path} --moduledir #{moduledir}")
 
 #Agent will fail because r10k will purge the "motd" module
 agents.each do |agent|
   step 'Attempt to Run Puppet Agent'
-  on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 0) do |result|
-    expect_failure('Expected to fail due to CODEMGMT-78') do
-      assert_match(error_message_regex, result.stderr, 'Expected error was not detected!')
-    end
+  on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 1) do |result|
+    assert_match(/Could not find declared class motd/, result.stderr, 'Module was not purged')
   end
 end

data/lib/r10k/action/base.rb

@@ -10,6 +10,16 @@ module R10K
 
       attr_accessor :settings
 
+      # @param opts [Hash] A hash of options defined in #allowed_initialized_opts
+      #   and managed by the SetOps mixin within the Action::Base class.
+      #   Corresponds to the CLI flags and options.
+      # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like
+      #   collection of the remaining arguments to the CLI invocation (after
+      #   removing flags and options).
+      # @param settings [Hash] A hash of configuration loaded from the relevant
+      #   config (r10k.yaml).
+      #
+      # @note All arguments will be required in the next major version
       def initialize(opts, argv, settings = {})
         @opts = opts
         @argv = argv

data/lib/r10k/action/deploy/display.rb

@@ -9,17 +9,48 @@ module R10K
 
         include R10K::Action::Deploy::DeployHelpers
 
+        # @param opts [Hash] A hash of options defined in #allowed_initialized_opts
+        #   and managed by the SetOps mixin within the Action::Base class.
+        #   Corresponds to the CLI flags and options.
+        # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like
+        #   collection of the remaining arguments to the CLI invocation (after
+        #   removing flags and options).
+        # @param settings [Hash] A hash of configuration loaded from the relevant
+        #   config (r10k.yaml).
+        #
+        # @note All arguments will be required in the next major version
+        def initialize(opts, argv, settings = {})
+          super
+
+          @settings = @settings.merge({
+            overrides: {
+              environments: {
+                preload_environments: @fetch,
+                requested_environments: @argv.map { |arg| arg.gsub(/\W/, '_') }
+              },
+              modules: {},
+              output: {
+                format: @format,
+                trace: @trace,
+                detail: @detail
+              },
+              purging: {}
+            }
+          })
+        end
+
         def call
           expect_config!
           deployment = R10K::Deployment.new(@settings)
 
-          if @fetch
+          if @settings.dig(:overrides, :environments, :preload_environments)
             deployment.preload!
+            deployment.validate!
           end
 
-          output = { :sources => deployment.sources.map { |source| source_info(source, @argv) } }
+          output = { :sources => deployment.sources.map { |source| source_info(source, @settings.dig(:overrides, :environments, :requested_environments)) } }
 
-          case @format
+          case @settings.dig(:overrides, :output, :format)
           when 'json' then json_format(output)
           else yaml_format(output)
           end
@@ -27,7 +58,7 @@ module R10K
           # exit 0
           true
         rescue => e
-          logger.error R10K::Errors::Formatting.format_exception(e, @trace)
+          logger.error R10K::Errors::Formatting.format_exception(e, @settings.dig(:overrides, :output, :trace))
           false
         end
 
@@ -43,7 +74,7 @@ module R10K
           puts output.to_yaml
         end
 
-        def source_info(source, argv=[])
+        def source_info(source, requested_environments = [])
           source_info = {
             :name => source.name,
             :basedir => source.basedir,
@@ -52,28 +83,30 @@ module R10K
           source_info[:prefix] = source.prefix if source.prefix
           source_info[:remote] = source.remote if source.respond_to?(:remote)
 
-          env_list = source.environments.select { |env| argv.empty? || argv.include?(env.name) }
+          select_all_envs = requested_environments.empty?
+          env_list = source.environments.select { |env| select_all_envs || requested_environments.include?(env.name) }
           source_info[:environments] = env_list.map { |env| environment_info(env) }
 
           source_info
         end
 
         def environment_info(env)
-          if !@modules && !@detail
+          modules = @settings.dig(:overrides, :environments, :deploy_modules)
+          if !modules && !@settings.dig(:overrides, :output, :detail)
             env.dirname
           else
             env_info = env.info.merge({
               :status => (env.status rescue nil),
             })
 
-            env_info[:modules] = env.modules.map { |mod| module_info(mod) } if @modules
+            env_info[:modules] = env.modules.map { |mod| module_info(mod) } if modules
 
             env_info
           end
         end
 
         def module_info(mod)
-          if @detail
+          if @settings.dig(:overrides, :output, :detail)
             { :name => mod.title, :properties => mod.properties }
           else
             mod.title

data/lib/r10k/action/deploy/environment.rb

@@ -1,4 +1,5 @@
 require 'r10k/util/setopts'
+require 'r10k/util/cleaner'
 require 'r10k/deployment'
 require 'r10k/logging'
 require 'r10k/action/visitor'
@@ -13,30 +14,64 @@ module R10K
 
         include R10K::Action::Deploy::DeployHelpers
 
+        # Deprecated
         attr_reader :force
 
-        def initialize(opts, argv, settings = nil)
-          settings ||= {}
-          @purge_levels = settings.fetch(:deploy, {}).fetch(:purge_levels, [])
-          @user_purge_allowlist = read_purge_allowlist(settings.fetch(:deploy, {}).fetch(:purge_whitelist, []),
-                                                       settings.fetch(:deploy, {}).fetch(:purge_allowlist, []))
-          @generate_types = settings.fetch(:deploy, {}).fetch(:generate_types, false)
-
+        attr_reader :settings
+
+        # @param opts [Hash] A hash of options defined in #allowed_initialized_opts
+        #   and managed by the SetOps mixin within the Action::Base class.
+        #   Corresponds to the CLI flags and options.
+        # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like
+        #   collection of the remaining arguments to the CLI invocation (after
+        #   removing flags and options).
+        # @param settings [Hash] A hash of configuration loaded from the relevant
+        #   config (r10k.yaml).
+        #
+        # @note All arguments will be required in the next major version
+        def initialize(opts, argv, settings = {})
           super
 
-          # @force here is used to make it easier to reason about
-          @force = !@no_force
-          @argv = @argv.map { |arg| arg.gsub(/\W/,'_') }
+          # instance variables below are set by the super class based on the
+          # spec of #allowed_initialize_opts and any command line flags. This
+          # gives a preference order of cli flags > config files > defaults.
+          @settings = @settings.merge({
+            overrides: {
+              environments: {
+                requested_environments: @argv.map { |arg| arg.gsub(/\W/,'_') },
+                default_branch_override: @default_branch_override,
+                generate_types: @generate_types || settings.dig(:deploy, :generate_types) || false,
+                preload_environments: true
+              },
+              modules: {
+                requested_modules: [],
+                deploy_modules: @modules,
+                force: !@no_force, # force here is used to make it easier to reason about
+              },
+              purging: {
+                purge_levels: settings.dig(:deploy, :purge_levels) || [],
+                purge_allowlist: read_purge_allowlist(settings.dig(:deploy, :purge_whitelist) || [],
+                                                      settings.dig(:deploy, :purge_allowlist) || [])
+              },
+              output: {}
+            }
+          })
         end
 
         def call
           @visit_ok = true
 
-          expect_config!
-          deployment = R10K::Deployment.new(@settings)
-          check_write_lock!(@settings)
+          begin
+            expect_config!
+            deployment = R10K::Deployment.new(@settings)
+            check_write_lock!(@settings)
+
+            deployment.accept(self)
+          rescue => e
+            @visit_ok = false
+            logger.error R10K::Errors::Formatting.format_exception(e, @trace)
+          end
 
-          deployment.accept(self)
           @visit_ok
         end
 
@@ -66,10 +101,12 @@ module R10K
           # sources then we can't fully enumerate all environments which
           # could be dangerous. If this fails then an exception will be raised
           # and execution will be halted.
-          deployment.preload!
-          deployment.validate!
+          if @settings.dig(:overrides, :environments, :preload_environments)
+            deployment.preload!
+            deployment.validate!
+          end
 
-          undeployable = undeployable_environment_names(deployment.environments, @argv)
+          undeployable = undeployable_environment_names(deployment.environments, @settings.dig(:overrides, :environments, :requested_environments))
           if !undeployable.empty?
             @visit_ok = false
             logger.error _("Environment(s) \'%{environments}\' cannot be found in any source and will not be deployed.") % {environments: undeployable.join(", ")}
@@ -77,15 +114,19 @@ module R10K
 
           yield
 
-          if @purge_levels.include?(:deployment)
+          if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment)
             logger.debug("Purging unmanaged environments for deployment...")
+            deployment.sources.each do |source|
+              source.reload!
+            end
             deployment.purge!
           end
         ensure
           if (postcmd = @settings[:postrun])
             if postcmd.grep('$modifiedenvs').any?
               envs = deployment.environments.map { |e| e.dirname }
-              envs.reject! { |e| !@argv.include?(e) } if @argv.any?
+              requested_envs = @settings.dig(:overrides, :environments, :requested_environments)
+              envs.reject! { |e| !requested_envs.include?(e) } if requested_envs.any?
               postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs.join(' ')) }
             end
             subproc = R10K::Util::Subprocess.new(postcmd)
@@ -99,7 +140,8 @@ module R10K
         end
 
         def visit_environment(environment)
-          if !(@argv.empty? || @argv.any? { |name| environment.dirname == name })
+          requested_envs = @settings.dig(:overrides, :environments, :requested_environments)
+          if !(requested_envs.empty? || requested_envs.any? { |name| environment.dirname == name })
             logger.debug1(_("Environment %{env_dir} does not match environment name filter, skipping") % {env_dir: environment.dirname})
             return
           end
@@ -113,28 +155,31 @@ module R10K
           environment.sync
           logger.info _("Environment %{env_dir} is now at %{env_signature}") % {env_dir: environment.dirname, env_signature: environment.signature}
 
-          if status == :absent || @modules
+          if status == :absent || @settings.dig(:overrides, :modules, :deploy_modules)
             if status == :absent
               logger.debug(_("Environment %{env_dir} is new, updating all modules") % {env_dir: environment.dirname})
             end
 
             previous_ok = @visit_ok
             @visit_ok = true
-            yield
+
+            environment.deploy
+
             @environment_ok = @visit_ok
             @visit_ok &&= previous_ok
           end
 
-          if @purge_levels.include?(:environment)
+
+          if @settings.dig(:overrides, :purging, :purge_levels).include?(:environment)
             if @visit_ok
               logger.debug("Purging unmanaged content for environment '#{environment.dirname}'...")
-              environment.purge!(:recurse => true, :whitelist => environment.whitelist(@user_purge_allowlist))
+              environment.purge!(:recurse => true, :whitelist => environment.whitelist(@settings.dig(:overrides, :purging, :purge_allowlist)))
             else
               logger.debug("Not purging unmanaged content for environment '#{environment.dirname}' due to prior deploy failures.")
             end
           end
 
-          if @generate_types
+          if @settings.dig(:overrides, :environments, :generate_types)
             if @environment_ok
               logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
               environment.generate_types!
@@ -146,22 +191,6 @@ module R10K
           write_environment_info!(environment, started_at, @visit_ok)
         end
 
-        def visit_puppetfile(puppetfile)
-          puppetfile.load(@opts[:'default-branch-override'])
-
-          yield
-
-          if @purge_levels.include?(:puppetfile)
-            logger.debug("Purging unmanaged Puppetfile content for environment '#{puppetfile.environment.dirname}'...")
-            puppetfile.purge!
-          end
-        end
-
-        def visit_module(mod)
-          logger.info _("Deploying %{origin} content %{path}") % {origin: mod.origin, path: mod.path}
-          mod.sync(force: @force)
-        end
-
         def write_environment_info!(environment, started_at, success)
           module_deploys =
             begin