r10k 3.9.3 → 3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46561d19b47798865a0cb7a986deb0dc5f4fadc9094414ca25958a41395f07fa
-  data.tar.gz: 3d604c77ed0d1735127c18737a1bbb0aacc65ad17a789e4b154e05283e240ac3
+  metadata.gz: db8e1373846da121e13c6bdd3f74c7eadac79d356d65628157d1f0b6c0939f23
+  data.tar.gz: 7fd4694a5e33f692213339db0422fa7db58d38f84d63f2586593665f8acae8cc
 SHA512:
-  metadata.gz: bfff74f4746cf25caf1586e6cfd4dea01e810203d59dab4de2072d35ba42bd90e5f1e2a904e117c1fba63de20666ab4b98f26ad6552ecfd1b4ce6c814eb65c0b
-  data.tar.gz: 5a3972217637238fd8f9bfd1c810e26a901764b496d7dc195c26a0c567111ab03c621e4cef20564f4c8ef57c6292255a8871625ffe3e15597ba6765d39428002
+  metadata.gz: 272021e80cc4ce7ba4d11f3316bd972d7498e102a1f4d4aa9c0bcf83eeef60bd283c9fb4db74ad37e854b2ebc60f6ba28d3c251ae51cb54fb2ae03a4842d4bb8
+  data.tar.gz: 3689ec04b7dd2a4597199fd2fddba3e06a903b682760aacd565f96e42a604c42a6ef340f598c37ee4e1c1cfc4f4912c1132880de98bb734d86aac39475a45342

data/.github/workflows/rspec_tests.yml

@@ -15,7 +15,7 @@ jobs:
           - {os: ubuntu-18.04, ruby: 2.5}
           - {os: ubuntu-18.04, ruby: 2.6}
           - {os: ubuntu-18.04, ruby: 2.7}
-          - {os: ubuntu-18.04, ruby: jruby-9.2.9.0}
+          - {os: ubuntu-18.04, ruby: jruby-9.2.10.0}
           - {os: windows-2016, ruby: 2.5}
           - {os: windows-2016, ruby: 2.6}
           - {os: windows-2016, ruby: 2.7}

data/CHANGELOG.mkd

@@ -4,6 +4,15 @@ CHANGELOG
 Unreleased
 ----------
 
+3.10.0
+------
+
+- Add `authorization_token` setting to allow authentication to a custom Forge server. [#1181](https://github.com/puppetlabs/r10k/pull/1181)
+- (RK-135) Attempting to download the latest version for a module that has no Forge releases will now issue a meaningful error. [#1177](https://github.com/puppetlabs/r10k/pull/1177)
+- Added an interface to R10K::Source::Base named `reload!` for updating the environments list for a given deployment; `reload!` is called before deployment purges to make r10k deploy pools more threadsafe. [#1172](https://github.com/puppetlabs/r10k/pull/1172)
+- Remove username and password from remote url in cache directory name [#1186](https://github.com/puppetlabs/r10k/pull/1186)
+- Purging efficiency is greatly improved. R10K will no longer recurse into directories that match recursive purge exclusions. This should significantly improve the deploy times for those users who enable the "environment" purge level. [#1178](https://github.com/puppetlabs/r10k/pull/1178)
+
 3.9.3
 -----
 

data/doc/dynamic-environments/configuration.mkd

@@ -165,9 +165,16 @@ interactions. See the global proxy setting documentation for more information an
 The 'baseurl' setting indicates where Forge modules should be installed from.
 This defaults to 'https://forgeapi.puppetlabs.com'
 
+#### authorization_token
+
+The 'authorization_token' setting allows you to provide a token for authenticating to a
+custom Forge server. When set, 'baseurl' must also be set.
+You will need to prepend your token with 'Bearer ' if using Artifactory as your Forge server.
+
 ```yaml
 forge:
   baseurl: 'https://private-forge.mysite'
+  authorization_token: 'Bearer mysupersecretauthtoken'
 ```
 
 Deployment options

data/integration/Rakefile

@@ -65,7 +65,7 @@ rototiller_task :beaker_hostgenerator do |t|
     end
 
     # This is a hack :(
-    t.add_flag(:name => '', :default => 'centos6-64mdca-64.fa', :override_env => 'TEST_TARGET')
+    t.add_flag(:name => '', :default => 'centos7-64mdca-64.fa', :override_env => 'TEST_TARGET')
 
     t.add_flag(:name => '--global-config', :default => '{forge_host=forge-aio01-petest.puppetlabs.com}', :override_env => 'BHG_GLOBAL_CONFIG')
   end

data/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb

@@ -10,7 +10,7 @@ last_commit = git_last_commit(master, git_environments_path)
 r10k_fqp = get_r10k_fqp(master)
 
 #Verification
-error_notification_regex = /Does 'puppetlabs-regret' have at least one published release?/
+error_notification_regex = /The module puppetlabs-regret does not appear to have any published releases/
 
 #File
 puppet_file = <<-PUPPETFILE
@@ -40,12 +40,6 @@ git_add_commit_push(master, 'production', 'Add module.', git_environments_path)
 
 #Tests
 step "Deploy production environment via r10k with specified module deleted"
-on(master, "#{r10k_fqp} deploy environment -p -v", :acceptable_exit_codes => 1) do |result|
-  if get_puppet_version(master) < 4.0
-    assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
-  else
-    expect_failure('expected to fail due to RK-135') do
-      assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
-    end
-  end
+on(master, "#{r10k_fqp} deploy environment -p -v --trace", :acceptable_exit_codes => 1) do |result|
+  assert_match(error_notification_regex, result.stderr, 'Unexpected error was detected!')
 end

data/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb

@@ -1,10 +1,11 @@
 require 'git_utils'
 require 'r10k_utils'
 require 'master_manipulator'
-test_name 'CODEMGMT-73 - C63184 - Single Environment Purge Unmanaged Modules'
+test_name 'CODEMGMT-78 - Puppetfile Purge --puppetfile & --moduledir flag usage'
 
 #Init
 master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip
+moduledir = on(master, puppet('config', 'print', 'environmentpath')).stdout.strip + '/production/modules'
 git_environments_path = '/root/environments'
 last_commit = git_last_commit(master, git_environments_path)
 r10k_fqp = get_r10k_fqp(master)
@@ -14,8 +15,6 @@ motd_path = '/etc/motd'
 motd_contents = 'Hello!'
 motd_contents_regex = /\A#{motd_contents}\z/
 
-error_message_regex = /Blah/
-
 #File
 puppet_file = <<-PUPPETFILE
 mod "puppetlabs/xinetd"
@@ -48,9 +47,6 @@ stub_forge_on(master)
 step 'Checkout "production" Branch'
 git_on(master, 'checkout production', git_environments_path)
 
-step 'Manually Install the "motd" Module from the Forge'
-on(master, puppet('module install puppetlabs-motd'))
-
 step 'Create "Puppetfile" for the "production" Environment'
 create_remote_file(master, puppet_file_path, puppet_file)
 
@@ -60,13 +56,13 @@ inject_site_pp(master, site_pp_path, site_pp)
 step 'Push Changes'
 git_add_commit_push(master, 'production', 'Update site.pp and add module.', git_environments_path)
 
-#Tests
 step 'Deploy Environments via r10k'
 on(master, "#{r10k_fqp} deploy environment -v -p")
 
-step 'Plug-in Sync Agents'
-on(agents, puppet("plugin download --server #{master}"))
+step 'Manually Install the "motd" Module from the Forge'
+on(master, puppet("module install puppetlabs-motd --modulepath #{moduledir}"))
 
+#Tests
 agents.each do |agent|
   step 'Run Puppet Agent Against "production" Environment'
   on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 2) do |result|
@@ -80,14 +76,12 @@ agents.each do |agent|
 end
 
 step 'Use r10k to Purge Unmanaged Modules'
-on(master, "#{r10k_fqp} puppetfile purge -v", :acceptable_exit_codes => 1)
+on(master, "#{r10k_fqp} puppetfile purge -v --puppetfile #{puppet_file_path} --moduledir #{moduledir}")
 
 #Agent will fail because r10k will purge the "motd" module
 agents.each do |agent|
   step 'Attempt to Run Puppet Agent'
-  on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 0) do |result|
-    expect_failure('Expected to fail due to CODEMGMT-78') do
-      assert_match(error_message_regex, result.stderr, 'Expected error was not detected!')
-    end
+  on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => 1) do |result|
+    assert_match(/Could not find declared class motd/, result.stderr, 'Module was not purged')
   end
 end

data/lib/r10k/action/deploy/environment.rb

@@ -116,6 +116,9 @@ module R10K
 
           if @settings.dig(:overrides, :purging, :purge_levels).include?(:deployment)
             logger.debug("Purging unmanaged environments for deployment...")
+            deployment.sources.each do |source|
+              source.reload!
+            end
             deployment.purge!
           end
         ensure

data/lib/r10k/action/runner.rb

@@ -67,15 +67,20 @@ module R10K
         exit(8)
       end
 
+      # Set up authorization from license file if it wasn't
+      # already set via the config
       def setup_authorization
-        begin
-          license = R10K::Util::License.load
+        if PuppetForge::Connection.authorization.nil?
+          begin
+            license = R10K::Util::License.load
 
-          if license.respond_to?(:authorization_token)
-            PuppetForge::Connection.authorization = license.authorization_token
+            if license.respond_to?(:authorization_token)
+              logger.debug "Using token from license to connect to the Forge."
+              PuppetForge::Connection.authorization = license.authorization_token
+            end
+          rescue R10K::Error => e
+            logger.warn e.message
           end
-        rescue R10K::Error => e
-          logger.warn e.message
         end
       end
 

data/lib/r10k/git/cache.rb

@@ -111,6 +111,6 @@ class R10K::Git::Cache
 
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/initializers.rb

@@ -63,6 +63,13 @@ module R10K
       def call
         with_setting(:baseurl) { |value| PuppetForge.host = value }
         with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value }
+        with_setting(:authorization_token) { |value|
+          if @settings[:baseurl]
+            PuppetForge::Connection.authorization = value
+          else
+            raise R10K::Error, "Cannot specify a Forge authorization token without configuring a custom baseurl."
+          end
+        }
       end
     end
   end

data/lib/r10k/module/forge.rb

@@ -83,7 +83,11 @@ class R10K::Module::Forge < R10K::Module::Base
   def expected_version
     if @expected_version == :latest
       begin
-        @expected_version = @v3_module.current_release.version
+        if @v3_module.current_release
+          @expected_version = @v3_module.current_release.version
+        else
+          raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title }
+        end
       rescue Faraday::ResourceNotFound => e
         raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace
       end

data/lib/r10k/module_loader/puppetfile.rb

@@ -0,0 +1,195 @@
+module R10K
+  module ModuleLoader
+    class Puppetfile
+
+      DEFAULT_MODULEDIR = 'modules'
+      DEFAULT_PUPPETFILE_NAME = 'Puppetfile'
+      DEFAULT_FORGE_API = 'forgeapi.puppetlabs.com'
+
+      attr_accessor :default_branch_override, :environment
+      attr_reader :modules, :moduledir,
+        :managed_directories, :desired_contents, :purge_exclusions
+
+      # @param basedir [String] The path that contains the moduledir &
+      #     Puppetfile by default. May be an environment, project, or
+      #     simple directory.
+      # @param puppetfile [String] The path to the Puppetfile, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param moduledir [String] The path to the moduledir, either an
+      #     absolute full path or a relative path with regards to the basedir.
+      # @param forge [String] The url (without protocol) to the Forge
+      # @param overrides [Hash] Configuration for loaded modules' behavior
+      # @param environment [R10K::Environment] When provided, the environment
+      #     in which loading takes place
+      def initialize(basedir:,
+                     moduledir: DEFAULT_MODULEDIR,
+                     puppetfile: DEFAULT_PUPPETFILE_NAME,
+                     forge: DEFAULT_FORGE_API,
+                     overrides: {},
+                     environment: nil)
+
+        @basedir     = cleanpath(basedir)
+        @moduledir   = resolve_path(@basedir, moduledir)
+        @puppetfile  = resolve_path(@basedir, puppetfile)
+        @forge       = forge
+        @overrides   = overrides
+        @environment = environment
+        @default_branch_override = @overrides.dig(:environments, :default_branch_override)
+
+        @modules = []
+
+        @managed_directories = []
+        @desired_contents = []
+        @purge_exclusions = []
+      end
+
+      def load
+        dsl = R10K::ModuleLoader::Puppetfile::DSL.new(self)
+        dsl.instance_eval(puppetfile_content(@puppetfile), @puppetfile)
+
+        validate_no_duplicate_names(@modules)
+        @modules
+
+        managed_content = @modules.group_by(&:dirname)
+
+        @managed_directories = determine_managed_directories(managed_content)
+        @desired_contents = determine_desired_contents(managed_content)
+        @purge_exclusions = determine_purge_exclusions(@managed_directories)
+
+        {
+          modules: @modules,
+          managed_directories: @managed_directories,
+          desired_contents: @desired_contents,
+          purge_exclusions: @purge_exclusions
+        }
+
+      rescue SyntaxError, LoadError, ArgumentError, NameError => e
+        raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile})
+      end
+
+
+      ##
+      ## set_forge, set_moduledir, and add_module are used directly by the DSL class
+      ##
+
+      # @param [String] forge
+      def set_forge(forge)
+        @forge = forge
+      end
+
+      # @param [String] moduledir
+      def set_moduledir(moduledir)
+        @moduledir = resolve_path(@basedir, moduledir)
+      end
+
+      # @param [String] name
+      # @param [Hash, String, Symbol, nil] module_info Calling with
+      #   anything but a Hash is deprecated. The DSL will now convert
+      #   String and Symbol versions to Hashes of the shape
+      #     { version: <String or Symbol> }
+      #
+      #   String inputs should be valid module versions, the Symbol
+      #   `:latest` is allowed, as well as `nil`.
+      #
+      #   Non-Hash inputs are only ever used by Forge modules. In
+      #   future versions this method will require the caller (the
+      #   DSL class, not the Puppetfile author) to do this conversion
+      #   itself.
+      #
+      def add_module(name, module_info)
+        if !module_info.is_a?(Hash)
+          module_info = { version: module_info }
+        end
+
+        module_info[:overrides] = @overrides
+
+        if install_path = module_info.delete(:install_path)
+          install_path = resolve_path(@basedir, install_path)
+          validate_install_path(install_path, name)
+        else
+          install_path = @moduledir
+        end
+
+        if @default_branch_override
+          module_info[:default_branch_override] = @default_branch_override
+        end
+
+        mod = R10K::Module.new(name, install_path, module_info, @environment)
+        mod.origin = :puppetfile
+
+        # Do not save modules if they would conflict with the attached
+        # environment
+        if @environment && @environment.module_conflicts?(mod)
+          return @modules
+        end
+
+        @modules << mod
+      end
+
+     private
+
+      # @param [Array<R10K::Module>] modules
+      def validate_no_duplicate_names(modules)
+        dupes = modules
+                .group_by { |mod| mod.name }
+                .select { |_, mods| mods.size > 1 }
+                .map(&:first)
+        unless dupes.empty?
+          msg = _('Puppetfiles cannot contain duplicate module names.')
+          msg += ' '
+          msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
+          raise R10K::Error.new(msg)
+        end
+      end
+
+      def resolve_path(base, path)
+        if Pathname.new(path).absolute?
+          cleanpath(path)
+        else
+          cleanpath(File.join(base, path))
+        end
+      end
+
+      def validate_install_path(path, modname)
+        unless /^#{Regexp.escape(@basedir)}.*/ =~ path
+          raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{@basedir}")
+        end
+
+        true
+      end
+
+      def determine_managed_directories(managed_content)
+        managed_content.keys.reject { |dir| dir == @basedir }
+      end
+
+      # Returns an array of the full paths to all the content being managed.
+      # @return [Array<String>]
+      def determine_desired_contents(managed_content)
+        managed_content.flat_map do |install_path, mods|
+          mods.collect { |mod| File.join(install_path, mod.name) }
+        end
+      end
+
+      def determine_purge_exclusions(managed_dirs)
+        if environment && environment.respond_to?(:desired_contents)
+          managed_dirs + environment.desired_contents
+        else
+          managed_dirs
+        end
+      end
+
+      # .cleanpath is as close to a canonical path as we can do without touching
+      # the filesystem. The .realpath methods will choke if some of the
+      # intermediate paths are missing, even though in some cases we will create
+      # them later as needed.
+      def cleanpath(path)
+        Pathname.new(path).cleanpath.to_s
+      end
+
+      # For testing purposes only
+      def puppetfile_content(path)
+        File.read(path)
+      end
+    end
+  end
+end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -0,0 +1,37 @@
+module R10K
+  module ModuleLoader
+    class Puppetfile
+      class DSL
+        # A barebones implementation of the Puppetfile DSL
+        #
+        # @api private
+
+        def initialize(librarian)
+          @librarian = librarian
+        end
+
+        def mod(name, args = nil)
+          if args.is_a?(Hash)
+            opts = args
+          else
+            opts = { version: args }
+          end
+
+          @librarian.add_module(name, opts)
+        end
+
+        def forge(location)
+          @librarian.set_forge(location)
+        end
+
+        def moduledir(location)
+          @librarian.set_moduledir(location)
+        end
+
+        def method_missing(method, *args)
+          raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
+        end
+      end
+    end
+  end
+end