r10k 3.5.0 → 3.8.0

data/lib/r10k/environment/base.rb

@@ -103,6 +103,13 @@ class R10K::Environment::Base
     @puppetfile.modules
   end
 
+  # @return [Array<R10K::Module::Base>] Whether or not the given module
+  #   conflicts with any modules already defined in the r10k environment
+  #   object.
+  def module_conflicts?(mod)
+    false
+  end
+
   def accept(visitor)
     visitor.visit(:environment, self) do
       puppetfile.accept(visitor)
@@ -137,7 +144,7 @@ class R10K::Environment::Base
   end
 
   def generate_types!
-    argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir]
+    argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
     subproc = R10K::Util::Subprocess.new(argv)
     subproc.raise_on_fail = true
     subproc.logger = logger

data/lib/r10k/environment/with_modules.rb

@@ -46,10 +46,33 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   #     - The r10k environment object
   #     - A Puppetfile in the environment's content
   def modules
-    return @modules if @puppetfile.nil?
+    return @modules if puppetfile.nil?
 
-    @puppetfile.load unless @puppetfile.loaded?
-    @modules + @puppetfile.modules
+    puppetfile.load unless puppetfile.loaded?
+    @modules + puppetfile.modules
+  end
+
+  def module_conflicts?(mod_b)
+    conflict = @modules.any? { |mod_a| mod_a.name == mod_b.name }
+    return false unless conflict
+
+    msg_vars = {src: mod_b.origin, name: mod_b.name}
+    msg_error = _('Environment and %{src} both define the "%{name}" module' % msg_vars)
+    msg_continue = _("#{msg_error}. The %{src} definition will be ignored" % msg_vars)
+
+    case conflict_opt = @options[:module_conflicts]
+    when 'override_and_warn', nil
+      logger.warn msg_continue
+    when 'override'
+      logger.debug msg_continue
+    when 'error'
+      raise R10K::Error, msg_error
+    else
+      raise R10K::Error, _('Unexpected value for `module_conflicts` setting in %{env} ' \
+                           'environment: %{val}' % {env: self.name, val: conflict_opt})
+    end
+
+    true
   end
 
   def accept(visitor)
@@ -59,7 +82,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
       end
 
       puppetfile.accept(visitor)
-      validate_no_module_conflicts
     end
   end
 
@@ -88,26 +110,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
     @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
 
     mod = R10K::Module.new(name, install_path, args, self.name)
-    mod.origin = 'Environment'
+    mod.origin = :environment
 
     @managed_content[install_path] << mod.name
     @modules << mod
   end
 
-  def validate_no_module_conflicts
-    @puppetfile.load unless @puppetfile.loaded?
-    conflicts = (@modules + @puppetfile.modules)
-                .group_by { |mod| mod.name }
-                .select { |_, v| v.size > 1 }
-                .map(&:first)
-    unless conflicts.empty?
-      msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name}
-      msg += ' '
-      msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') })
-      raise R10K::Error.new(msg)
-    end
-  end
-
   include R10K::Util::Purgeable
 
   # Returns an array of the full paths that can be purged.

data/lib/r10k/forge/module_release.rb

@@ -212,14 +212,14 @@ module R10K
 
       # Remove the temporary directory used for unpacking the module.
       def cleanup_unpack_path
-        if unpack_path.exist?
+        if unpack_path.parent.exist?
           unpack_path.parent.rmtree
         end
       end
 
       # Remove the downloaded module release.
       def cleanup_download_path
-        if download_path.exist?
+        if download_path.parent.exist?
           download_path.parent.rmtree
         end
       end

data/lib/r10k/git.rb

@@ -134,6 +134,7 @@ module R10K
     extend R10K::Settings::Mixin::ClassMethods
 
     def_setting_attr :private_key
+    def_setting_attr :oauth_token
     def_setting_attr :proxy
     def_setting_attr :username
     def_setting_attr :repositories, {}

data/lib/r10k/git/cache.rb

@@ -16,7 +16,17 @@ class R10K::Git::Cache
 
   include R10K::Settings::Mixin
 
-  def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
+  #@api private
+  def self.determine_cache_root
+    if R10K::Util::Platform.windows?
+      File.join(ENV['LOCALAPPDATA'], 'r10k', 'git')
+    else
+      File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
+    end
+  end
+  private_class_method :determine_cache_root
+
+  def_setting_attr :cache_root, determine_cache_root
 
   @instance_cache = R10K::InstanceCache.new(self)
 
@@ -99,10 +109,8 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  private
-
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/git/rugged/credentials.rb

@@ -61,11 +61,41 @@ class R10K::Git::Rugged::Credentials
   end
 
   def get_plaintext_credentials(url, username_from_url)
-    user = get_git_username(url, username_from_url)
-    password = URI.parse(url).password || ''
+    per_repo_oauth_token = nil
+    if per_repo_settings = R10K::Git.get_repo_settings(url)
+      per_repo_oauth_token = per_repo_settings[:oauth_token]
+    end
+
+    if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
+      if token_path == '-'
+        token = $stdin.read.strip
+        logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url }
+      elsif File.readable?(token_path)
+        token = File.read(token_path).strip
+        logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url }
+      else
+        raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path }
+      end
+
+      unless valid_token?(token)
+        raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.")
+      end
+
+      user = 'x-oauth-token'
+      password = token
+    else
+      user = get_git_username(url, username_from_url)
+      password = URI.parse(url).password || ''
+    end
     Rugged::Credentials::UserPassword.new(username: user, password: password)
   end
 
+  # This regex is the only real requirement for OAuth token format,
+  # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
+  def valid_token?(token)
+    return token =~ /^[\w\-\.~\+\/]+$/
+  end
+
   def get_default_credentials(url, username_from_url)
     Rugged::Credentials::Default.new
   end

data/lib/r10k/git/stateful_repository.rb

@@ -12,6 +12,10 @@ class R10K::Git::StatefulRepository
   #   @api private
   attr_reader :repo
 
+  # @!attribute [r] cache
+  #   @api private
+  attr_reader :cache
+
   extend Forwardable
   def_delegators :@repo, :head, :tracked_paths
 

data/lib/r10k/initializers.rb

@@ -44,6 +44,7 @@ module R10K
     class DeployInitializer < BaseInitializer
       def call
         with_setting(:puppet_path) { |value| R10K::Settings.puppet_path = value }
+        with_setting(:puppet_conf) { |value| R10K::Settings.puppet_conf = value }
       end
     end
 
@@ -54,6 +55,7 @@ module R10K
         with_setting(:private_key) { |value| R10K::Git.settings[:private_key] = value }
         with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value }
         with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value }
+        with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value }
       end
     end
 

data/lib/r10k/module/base.rb

@@ -99,6 +99,14 @@ class R10K::Module::Base
     raise NotImplementedError
   end
 
+  # Return the module's cachedir. Subclasses that implement a cache
+  # will override this to return a real directory location.
+  #
+  # @return [String, :none]
+  def cachedir
+    :none
+  end
+
   private
 
   def parse_title(title)

data/lib/r10k/module/forge.rb

@@ -171,7 +171,7 @@ class R10K::Module::Forge < R10K::Module::Base
     if (match = title.match(/\A(\w+)[-\/](\w+)\Z/))
       [match[1], match[2]]
     else
-      raise ArgumentError, _("Forge module names must match 'owner/modulename'")
+      raise ArgumentError, _("Forge module names must match 'owner/modulename', instead got #{title}")
     end
   end
 end

data/lib/r10k/module/git.rb

@@ -28,6 +28,11 @@ class R10K::Module::Git < R10K::Module::Base
   #   @return [String]
   attr_reader :default_ref
 
+  # @!attribute [r] default_override_ref
+  #   @api private
+  #   @return [String]
+  attr_reader :default_override_ref
+
   def initialize(title, dirname, args, environment=nil)
     super
 
@@ -37,7 +42,7 @@ class R10K::Module::Git < R10K::Module::Base
   end
 
   def version
-    validate_ref(@desired_ref, @default_ref)
+    validate_ref(@desired_ref, @default_ref, @default_override_ref)
   end
 
   def properties
@@ -57,11 +62,17 @@ class R10K::Module::Git < R10K::Module::Base
     @repo.status(version)
   end
 
+  def cachedir
+    @repo.cache.sanitized_dirname
+  end
+
   private
 
-  def validate_ref(desired, default)
+  def validate_ref(desired, default, default_override)
     if desired && desired != :control_branch && @repo.resolve(desired)
       return desired
+    elsif default_override && @repo.resolve(default_override)
+      return default_override
     elsif default && @repo.resolve(default)
       return default
     else
@@ -77,6 +88,11 @@ class R10K::Module::Git < R10K::Module::Base
         msg << "Could not determine desired ref"
       end
 
+      if default_override
+        msg << "or resolve the default branch override '%{default_override}',"
+        vars[:default_override] = default_override
+      end
+
       if default
         msg << "or resolve default ref '%{default}'"
         vars[:default] = default
@@ -90,7 +106,7 @@ class R10K::Module::Git < R10K::Module::Base
 
   def parse_options(options)
     ref_opts = [:branch, :tag, :commit, :ref]
-    known_opts = [:git, :default_branch] + ref_opts
+    known_opts = [:git, :default_branch, :default_branch_override] + ref_opts
 
     unhandled = options.keys - known_opts
     unless unhandled.empty?
@@ -101,6 +117,7 @@ class R10K::Module::Git < R10K::Module::Base
 
     @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master'
     @default_ref = options[:default_branch]
+    @default_override_ref = options[:default_branch_override]
 
     if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref)
       @desired_ref = @environment.ref

data/lib/r10k/puppetfile.rb

@@ -10,7 +10,7 @@ class Puppetfile
 
   include R10K::Settings::Mixin
 
-  def_setting_attr :pool_size, 1
+  def_setting_attr :pool_size, 4
 
   include R10K::Logging
 
@@ -77,7 +77,7 @@ class Puppetfile
 
     dsl = R10K::Puppetfile::DSL.new(self)
     dsl.instance_eval(puppetfile_contents, @puppetfile_path)
-    
+
     validate_no_duplicate_names(@modules)
     @loaded = true
   rescue SyntaxError, LoadError, ArgumentError, NameError => e
@@ -127,16 +127,23 @@ class Puppetfile
     end
 
     if args.is_a?(Hash) && @default_branch_override != nil
-      args[:default_branch] = @default_branch_override
+      args[:default_branch_override] = @default_branch_override
     end
 
-    # Keep track of all the content this Puppetfile is managing to enable purging.
-    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
-
     mod = R10K::Module.new(name, install_path, args, @environment)
-    mod.origin = 'Puppetfile'
+    mod.origin = :puppetfile
 
+    # Do not load modules if they would conflict with the attached
+    # environment
+    if environment && environment.module_conflicts?(mod)
+      mod = nil
+      return @modules
+    end
+
+    # Keep track of all the content this Puppetfile is managing to enable purging.
+    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
     @managed_content[install_path] << mod.name
+
     @modules << mod
   end
 
@@ -145,7 +152,9 @@ class Puppetfile
   def managed_directories
     self.load unless @loaded
 
-    @managed_content.keys
+    dirs = @managed_content.keys
+    dirs.delete(real_basedir)
+    dirs
   end
 
   # Returns an array of the full paths to all the content being managed.
@@ -212,15 +221,22 @@ class Puppetfile
   def modules_queue(visitor)
     Queue.new.tap do |queue|
       visitor.visit(:puppetfile, self) do
-        modules.each { |mod| queue << mod }
+        modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
+        modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+        modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+        modules_by_cachedir.values.each {|mods| queue << mods }
       end
     end
   end
+  public :modules_queue
 
   def visitor_thread(visitor, mods_queue)
     Thread.new do
       begin
-        while mod = mods_queue.pop(true) do mod.accept(visitor) end
+        while mods = mods_queue.pop(true) do
+          mods.each {|mod| mod.accept(visitor) }
+        end
       rescue ThreadError => e
         logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
         Thread.exit
@@ -248,8 +264,6 @@ class Puppetfile
   end
 
   def validate_install_path(path, modname)
-    real_basedir = Pathname.new(basedir).cleanpath.to_s
-
     unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
       raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
     end
@@ -257,6 +271,10 @@ class Puppetfile
     true
   end
 
+  def real_basedir
+    Pathname.new(basedir).cleanpath.to_s
+  end
+
   class DSL
     # A barebones implementation of the Puppetfile DSL
     #

data/lib/r10k/settings.rb

@@ -12,6 +12,8 @@ module R10K
     class << self
       # Path to puppet executable
       attr_accessor :puppet_path
+      # Path to puppet.conf
+      attr_accessor :puppet_conf
     end
 
     def self.git_settings
@@ -35,6 +37,11 @@ module R10K
                     Only used by the 'rugged' Git provider.",
         }),
 
+        Definition.new(:oauth_token, {
+          :desc => "The path to a token file for Git OAuth remotes.
+                    Only used by the 'rugged' Git provider."
+        }),
+
         URIDefinition.new(:proxy, {
           :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
           :default => :inherit,
@@ -52,11 +59,17 @@ module R10K
               :default => :inherit,
             }),
 
+            Definition.new(:oauth_token, {
+              :desc => "The path to a token file for Git OAuth remotes.
+                        Only used by the 'rugged' Git provider.",
+              :default => :inherit
+            }),
+
             URIDefinition.new(:proxy, {
               :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
               :default => :inherit,
             }),
-            
+
             Definition.new(:ignore_branch_prefixes, {
               :desc => "Array of strings used to prefix branch names that will not be deployed as environments.",
             }),
@@ -131,6 +144,15 @@ module R10K
             end
           end
         }),
+        Definition.new(:puppet_conf, {
+          :desc => "Path to puppet.conf. Defaults to /etc/puppetlabs/puppet/puppet.conf.",
+          :default => '/etc/puppetlabs/puppet/puppet.conf',
+          :validate => lambda do |value|
+            unless File.readable? value
+              raise ArgumentError, "The specified puppet.conf #{value} is not readable"
+            end
+          end
+        }),
       ])
     end
 
@@ -160,7 +182,7 @@ module R10K
 
         Definition.new(:pool_size, {
           :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
-          :default => 1,
+          :default => 4,
           :validate => lambda do |value|
             if !value.is_a?(Integer)
               raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"