r10k 3.7.0 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8b9e50ee8c5d0887c46aaf294074c57626229cb19218fc08dd87f5ae1812888
-  data.tar.gz: 96c59ea0332a86792c6922f354fdf5bed5c9cb31a37b608e09692fbce00cb1bb
+  metadata.gz: dddce45b116f972f8efe10ad99c330b60186cfbcdb8747fbab2edc37d8634efb
+  data.tar.gz: 26ac288406c3e4fb3862aa5ca64aba4b7ed102dcc407ba59d364f094b1491480
 SHA512:
-  metadata.gz: 45e41d47b78d5951e533f6b2dece2f2f127610b6ac6a9d3d95927699eb11369159add33e467198e0b7869442c1803b5e183e02a44c00ccc8eadf43aabf263b8e
-  data.tar.gz: 4790f4189c4e15502cfc633d52605e000a9607acc1b59ec2b011d14736eed3dbcb05333e3bb2817d6bad3e08603c8472267f3a19dca0d64c3f1df10c99eb4200
+  metadata.gz: dbe3bd1fbc2ca1f1fee87fb5ce9e613baf9783cd1a728f50e0de65718a50f7321609458a1e60817fee94ec32464b95ebf3ecf32dde5c065ccabe369fdc2c6b0d
+  data.tar.gz: 1d33d1f5772a71c0e6dae7b36730a810b8d2bc59e8ec71a4639b3e4f38a5394396dd62270a63032619c2a3cadeb1e1066a0242e8d9b4b1436933660aec806eab

data/CHANGELOG.mkd

@@ -4,6 +4,13 @@ CHANGELOG
 Unreleased
 ----------
 
+3.8.0
+-----
+
+- When a forge module fails name validation the offending name will now be printed in the error message. [#1126](https://github.com/puppetlabs/r10k/pull/1126)
+- Module ref resolution will now fall back to the normal default branch if the default branch override cannot be resolved. [#1122](https://github.com/puppetlabs/r10k/pull/1122)
+- Experimental feature change: conflicts between environment-defined modules and Puppetfile-defined modules now default to logging a warning and deploying the environment module version, overriding the Puppetfile. Previously, conflicts would result in an error. The behavior is now configurable via the `module_conflicts` environment setting [#1107](https://github.com/puppetlabs/r10k/pull/1107)
+
 3.7.0
 -----
 

data/CODEOWNERS

@@ -1,2 +1,2 @@
-*        @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger
+*        @puppetlabs/puppetserver-maintainers
 /docker/ @puppetlabs/pupperware

data/doc/dynamic-environments/configuration.mkd

@@ -673,6 +673,31 @@ modules:
     ref: 62d07f2
 ```
 
+#### Puppetfile module conflicts
+
+When a module is defined in an environment and also in a Puppetfile, the default behavior is for the environment definition of the module to take precedence, a warning to be logged, and the Puppetfile definition to be ignored. The behavior is configurable to optionally skip the warning, or allow a hard failure instead. Use the `module_conflicts` option in an environment definition to control this.
+
+Available `module_conflicts` options:
+
+* `override_and_warn` (default): the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. A warning will be printed.
+* `override`: the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored.
+* `error`: an error will be raised alerting the user to the conflict. The environment will not be deployed.
+
+```yaml
+# production.yaml
+---
+type: git
+remote: git@github.com:puppetlabs/control-repo.git
+ref: 8820892
+module_conflicts: override_puppetfile_and_warn
+modules:
+  puppetlabs-stdlib: 6.0.0
+  puppetlabs-concat: 6.1.0
+  reidmv-xampl:
+    git: https://github.com/reidmv/reidmv-xampl.git
+    ref: 62d07f2
+```
+
 ### Bare Environment Type
 
 A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it.

data/doc/dynamic-environments/usage.mkd

@@ -66,11 +66,12 @@ Update a single environment and specify a default branch override:
 
     r10k deploy environment my_working_environment --puppetfile --default-branch-override default_branch_override
 
-This will update the given environment and update all contained modules, overrideing 
-the :default_branch entry in the Puppetfile of each module. This is used primarily to allow
+This will update the given environment and update all contained modules, overriding
+the :default_branch entry in the Puppetfile of each module. If the specified override branch is not
+found, it will fall back to the normal default branch and attempt to use that. This is used primarily to allow
 automated r10k solutions using the control_branch pattern with a temporary branch deployment to 
-ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch
-is only ever utilized if the desired ref cannot be located.
+ensure the deployment is pushed to the correct module repository branch. Note that the :default_branch and its
+override are only ever used if the specific desired ref cannot be located.
 
 ### Deploying modules
 

data/lib/r10k/action/deploy/environment.rb

@@ -189,6 +189,8 @@ module R10K
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
+                      'private-key': :self,
+                      'oauth-token': :self,
                       'default-branch-override': :self)
         end
       end

data/lib/r10k/action/deploy/module.rb

@@ -77,7 +77,9 @@ module R10K
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
-                      'puppet-conf': :self)
+                      'puppet-conf': :self,
+                      'private-key': :self,
+                      'oauth-token': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -55,6 +55,10 @@ module R10K
           newval
         end
 
+        # Credentials from the CLI override both the global and per-repo
+        # credentials from the config, and so need to be handled specially
+        with_overrides = add_credential_overrides(with_overrides)
+
         @settings = R10K::Settings.global_settings.evaluate(with_overrides)
 
         R10K::Initializers::GlobalInitializer.new(@settings).call
@@ -92,6 +96,35 @@ module R10K
 
         results
       end
+
+      def add_credential_overrides(overrides)
+        sshkey_path = @opts[:'private-key']
+        token_path = @opts[:'oauth-token']
+
+        if sshkey_path && token_path
+          raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
+        end
+
+        if sshkey_path
+          overrides[:git] ||= {}
+          overrides[:git][:private_key] = sshkey_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:private_key] = sshkey_path
+            end
+          end
+        elsif token_path
+          overrides[:git] ||= {}
+          overrides[:git][:oauth_token] = token_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:oauth_token] = token_path
+            end
+          end
+        end
+
+        overrides
+      end
     end
   end
 end

data/lib/r10k/cli/deploy.rb

@@ -21,7 +21,7 @@ module R10K::CLI
 (https://puppet.com/docs/puppet/latest/environments_about.html).
         DESCRIPTION
 
-        required nil, :cachedir, 'Specify a cachedir, overriding the value in config'
+        option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
@@ -32,6 +32,8 @@ module R10K::CLI
           end
         end
         option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
+        option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])
@@ -62,7 +64,8 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
           DESCRIPTION
 
           flag :p, :puppetfile, 'Deploy modules from a puppetfile'
-          required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
+          option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
         end
@@ -82,7 +85,7 @@ It will load the Puppetfile configurations out of all environments, and will
 try to deploy the given module names in all environments.
           DESCRIPTION
 
-          required :e, :environment, 'Update the modules in the given environment'
+          option :e, :environment, 'Update the modules in the given environment', argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module)
         end
@@ -100,7 +103,8 @@ try to deploy the given module names in all environments.
           flag :p, :puppetfile, 'Display Puppetfile modules'
           flag nil, :detail, 'Display detailed information'
           flag nil, :fetch, 'Update available environment lists from all remote sources'
-          required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml'
+          option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display)
         end

data/lib/r10k/cli/puppetfile.rb

@@ -30,8 +30,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           name    'install'
           usage   'install'
           summary 'Install all modules from a Puppetfile'
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to puppetfile', argument: :required
           flag     nil, :force, 'Force locally changed files to be overwritten'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install)
         end
@@ -45,7 +45,7 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'check'
           summary 'Try and load the Puppetfile to verify the syntax is correct.'
 
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check)
         end
       end
@@ -58,8 +58,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'purge'
           summary 'Purge unmanaged modules from a Puppetfile managed directory'
 
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge)
         end
       end

data/lib/r10k/environment/base.rb

@@ -103,6 +103,13 @@ class R10K::Environment::Base
     @puppetfile.modules
   end
 
+  # @return [Array<R10K::Module::Base>] Whether or not the given module
+  #   conflicts with any modules already defined in the r10k environment
+  #   object.
+  def module_conflicts?(mod)
+    false
+  end
+
   def accept(visitor)
     visitor.visit(:environment, self) do
       puppetfile.accept(visitor)

data/lib/r10k/environment/with_modules.rb

@@ -46,10 +46,33 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   #     - The r10k environment object
   #     - A Puppetfile in the environment's content
   def modules
-    return @modules if @puppetfile.nil?
+    return @modules if puppetfile.nil?
 
-    @puppetfile.load unless @puppetfile.loaded?
-    @modules + @puppetfile.modules
+    puppetfile.load unless puppetfile.loaded?
+    @modules + puppetfile.modules
+  end
+
+  def module_conflicts?(mod_b)
+    conflict = @modules.any? { |mod_a| mod_a.name == mod_b.name }
+    return false unless conflict
+
+    msg_vars = {src: mod_b.origin, name: mod_b.name}
+    msg_error = _('Environment and %{src} both define the "%{name}" module' % msg_vars)
+    msg_continue = _("#{msg_error}. The %{src} definition will be ignored" % msg_vars)
+
+    case conflict_opt = @options[:module_conflicts]
+    when 'override_and_warn', nil
+      logger.warn msg_continue
+    when 'override'
+      logger.debug msg_continue
+    when 'error'
+      raise R10K::Error, msg_error
+    else
+      raise R10K::Error, _('Unexpected value for `module_conflicts` setting in %{env} ' \
+                           'environment: %{val}' % {env: self.name, val: conflict_opt})
+    end
+
+    true
   end
 
   def accept(visitor)
@@ -59,7 +82,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
       end
 
       puppetfile.accept(visitor)
-      validate_no_module_conflicts
     end
   end
 
@@ -88,26 +110,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
     @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
 
     mod = R10K::Module.new(name, install_path, args, self.name)
-    mod.origin = 'Environment'
+    mod.origin = :environment
 
     @managed_content[install_path] << mod.name
     @modules << mod
   end
 
-  def validate_no_module_conflicts
-    @puppetfile.load unless @puppetfile.loaded?
-    conflicts = (@modules + @puppetfile.modules)
-                .group_by { |mod| mod.name }
-                .select { |_, v| v.size > 1 }
-                .map(&:first)
-    unless conflicts.empty?
-      msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name}
-      msg += ' '
-      msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') })
-      raise R10K::Error.new(msg)
-    end
-  end
-
   include R10K::Util::Purgeable
 
   # Returns an array of the full paths that can be purged.

data/lib/r10k/git.rb

@@ -134,6 +134,7 @@ module R10K
     extend R10K::Settings::Mixin::ClassMethods
 
     def_setting_attr :private_key
+    def_setting_attr :oauth_token
     def_setting_attr :proxy
     def_setting_attr :username
     def_setting_attr :repositories, {}

data/lib/r10k/git/rugged/credentials.rb

@@ -61,11 +61,41 @@ class R10K::Git::Rugged::Credentials
   end
 
   def get_plaintext_credentials(url, username_from_url)
-    user = get_git_username(url, username_from_url)
-    password = URI.parse(url).password || ''
+    per_repo_oauth_token = nil
+    if per_repo_settings = R10K::Git.get_repo_settings(url)
+      per_repo_oauth_token = per_repo_settings[:oauth_token]
+    end
+
+    if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
+      if token_path == '-'
+        token = $stdin.read.strip
+        logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url }
+      elsif File.readable?(token_path)
+        token = File.read(token_path).strip
+        logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url }
+      else
+        raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path }
+      end
+
+      unless valid_token?(token)
+        raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.")
+      end
+
+      user = 'x-oauth-token'
+      password = token
+    else
+      user = get_git_username(url, username_from_url)
+      password = URI.parse(url).password || ''
+    end
     Rugged::Credentials::UserPassword.new(username: user, password: password)
   end
 
+  # This regex is the only real requirement for OAuth token format,
+  # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
+  def valid_token?(token)
+    return token =~ /^[\w\-\.~\+\/]+$/
+  end
+
   def get_default_credentials(url, username_from_url)
     Rugged::Credentials::Default.new
   end

data/lib/r10k/initializers.rb

@@ -55,6 +55,7 @@ module R10K
         with_setting(:private_key) { |value| R10K::Git.settings[:private_key] = value }
         with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value }
         with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value }
+        with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value }
       end
     end
 

data/lib/r10k/module/forge.rb

@@ -171,7 +171,7 @@ class R10K::Module::Forge < R10K::Module::Base
     if (match = title.match(/\A(\w+)[-\/](\w+)\Z/))
       [match[1], match[2]]
     else
-      raise ArgumentError, _("Forge module names must match 'owner/modulename'")
+      raise ArgumentError, _("Forge module names must match 'owner/modulename', instead got #{title}")
     end
   end
 end

data/lib/r10k/module/git.rb

@@ -28,6 +28,11 @@ class R10K::Module::Git < R10K::Module::Base
   #   @return [String]
   attr_reader :default_ref
 
+  # @!attribute [r] default_override_ref
+  #   @api private
+  #   @return [String]
+  attr_reader :default_override_ref
+
   def initialize(title, dirname, args, environment=nil)
     super
 
@@ -37,7 +42,7 @@ class R10K::Module::Git < R10K::Module::Base
   end
 
   def version
-    validate_ref(@desired_ref, @default_ref)
+    validate_ref(@desired_ref, @default_ref, @default_override_ref)
   end
 
   def properties
@@ -63,9 +68,11 @@ class R10K::Module::Git < R10K::Module::Base
 
   private
 
-  def validate_ref(desired, default)
+  def validate_ref(desired, default, default_override)
     if desired && desired != :control_branch && @repo.resolve(desired)
       return desired
+    elsif default_override && @repo.resolve(default_override)
+      return default_override
     elsif default && @repo.resolve(default)
       return default
     else
@@ -81,6 +88,11 @@ class R10K::Module::Git < R10K::Module::Base
         msg << "Could not determine desired ref"
       end
 
+      if default_override
+        msg << "or resolve the default branch override '%{default_override}',"
+        vars[:default_override] = default_override
+      end
+
       if default
         msg << "or resolve default ref '%{default}'"
         vars[:default] = default
@@ -94,7 +106,7 @@ class R10K::Module::Git < R10K::Module::Base
 
   def parse_options(options)
     ref_opts = [:branch, :tag, :commit, :ref]
-    known_opts = [:git, :default_branch] + ref_opts
+    known_opts = [:git, :default_branch, :default_branch_override] + ref_opts
 
     unhandled = options.keys - known_opts
     unless unhandled.empty?
@@ -105,6 +117,7 @@ class R10K::Module::Git < R10K::Module::Base
 
     @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master'
     @default_ref = options[:default_branch]
+    @default_override_ref = options[:default_branch_override]
 
     if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref)
       @desired_ref = @environment.ref