r10k 3.5.2 → 3.9.1

data/lib/r10k/module/git.rb

@@ -8,7 +8,7 @@ class R10K::Module::Git < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a? Hash and args.has_key?(:git)
+    args.has_key?(:git) || args[:type].to_s == 'git'
   rescue
     false
   end
@@ -28,16 +28,50 @@ class R10K::Module::Git < R10K::Module::Base
   #   @return [String]
   attr_reader :default_ref
 
-  def initialize(title, dirname, args, environment=nil)
-    super
+  # @!attribute [r] default_override_ref
+  #   @api private
+  #   @return [String]
+  attr_reader :default_override_ref
 
-    parse_options(@args)
+  include R10K::Util::Setopts
+
+  def initialize(title, dirname, opts, environment=nil)
+
+    super
+    setopts(opts, {
+      # Standard option interface
+      :version                 => :desired_ref,
+      :source                  => :remote,
+      :type                    => ::R10K::Util::Setopts::Ignore,
+
+      # Type-specific options
+      :branch                  => :desired_ref,
+      :tag                     => :desired_ref,
+      :commit                  => :desired_ref,
+      :ref                     => :desired_ref,
+      :git                     => :remote,
+      :default_branch          => :default_ref,
+      :default_branch_override => :default_override_ref,
+    })
+
+    force = @overrides.dig(:modules, :force)
+    @force = force == false ? false : true
+
+    @desired_ref ||= 'master'
+
+    if @desired_ref == :control_branch
+      if @environment && @environment.respond_to?(:ref)
+        @desired_ref = @environment.ref
+      else
+        logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a git-backed environment, will use default if available." % {name: name})
+      end
+    end
 
     @repo = R10K::Git::StatefulRepository.new(@remote, @dirname, @name)
   end
 
   def version
-    validate_ref(@desired_ref, @default_ref)
+    validate_ref(@desired_ref, @default_ref, @default_override_ref)
   end
 
   def properties
@@ -48,20 +82,27 @@ class R10K::Module::Git < R10K::Module::Base
     }
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
-    force = opts && opts.fetch(:force, true)
-    @repo.sync(version, force)
+    force = opts[:force] || @force
+    @repo.sync(version, force) if should_sync?
   end
 
   def status
     @repo.status(version)
   end
 
+  def cachedir
+    @repo.cache.sanitized_dirname
+  end
+
   private
 
-  def validate_ref(desired, default)
+  def validate_ref(desired, default, default_override)
     if desired && desired != :control_branch && @repo.resolve(desired)
       return desired
+    elsif default_override && @repo.resolve(default_override)
+      return default_override
     elsif default && @repo.resolve(default)
       return default
     else
@@ -77,6 +118,11 @@ class R10K::Module::Git < R10K::Module::Base
         msg << "Could not determine desired ref"
       end
 
+      if default_override
+        msg << "or resolve the default branch override '%{default_override}',"
+        vars[:default_override] = default_override
+      end
+
       if default
         msg << "or resolve default ref '%{default}'"
         vars[:default] = default
@@ -87,23 +133,4 @@ class R10K::Module::Git < R10K::Module::Base
       raise ArgumentError, _(msg.join(' ')) % vars
     end
   end
-
-  def parse_options(options)
-    ref_opts = [:branch, :tag, :commit, :ref]
-    known_opts = [:git, :default_branch] + ref_opts
-
-    unhandled = options.keys - known_opts
-    unless unhandled.empty?
-      raise ArgumentError, _("Unhandled options %{unhandled} specified for %{class}") % {unhandled: unhandled, class: self.class}
-    end
-
-    @remote = options[:git]
-
-    @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master'
-    @default_ref = options[:default_branch]
-
-    if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref)
-      @desired_ref = @environment.ref
-    end
-  end
 end

data/lib/r10k/module/local.rb

@@ -9,7 +9,7 @@ class R10K::Module::Local < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a?(Hash) && args[:local]
+    args.is_a?(Hash) && (args[:local] || args[:type].to_s == 'local')
   end
 
   include R10K::Logging
@@ -30,6 +30,7 @@ class R10K::Module::Local < R10K::Module::Base
     :insync
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
     logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title}
   end

data/lib/r10k/module/svn.rb

@@ -7,7 +7,7 @@ class R10K::Module::SVN < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    args.is_a? Hash and args.has_key? :svn
+    args.has_key?(:svn) || args[:type].to_s == 'svn'
   end
 
   # @!attribute [r] expected_revision
@@ -36,18 +36,21 @@ class R10K::Module::SVN < R10K::Module::Base
 
   include R10K::Util::Setopts
 
-  INITIALIZE_OPTS = {
-    :svn => :url,
-    :rev => :expected_revision,
-    :revision => :expected_revision,
-    :username => :self,
-    :password => :self
-  }
-
   def initialize(name, dirname, opts, environment=nil)
     super
-
-    setopts(opts, INITIALIZE_OPTS)
+    setopts(opts, {
+      # Standard option interface
+      :source   => :url,
+      :version  => :expected_revision,
+      :type     => ::R10K::Util::Setopts::Ignore,
+
+      # Type-specific options
+      :svn      => :url,
+      :rev      => :expected_revision,
+      :revision => :expected_revision,
+      :username => :self,
+      :password => :self
+    })
 
     @working_dir = R10K::SVN::WorkingDir.new(@path, :username => @username, :password => @password)
   end
@@ -66,14 +69,17 @@ class R10K::Module::SVN < R10K::Module::Base
     end
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
-    case status
-    when :absent
-      install
-    when :mismatched
-      reinstall
-    when :outdated
-      update
+    if should_sync?
+      case status
+      when :absent
+        install
+      when :mismatched
+        reinstall
+      when :outdated
+        update
+      end
     end
   end
 

data/lib/r10k/puppetfile.rb

@@ -3,6 +3,7 @@ require 'pathname'
 require 'r10k/module'
 require 'r10k/util/purgeable'
 require 'r10k/errors'
+require 'r10k/content_synchronizer'
 
 module R10K
 class Puppetfile
@@ -10,7 +11,7 @@ class Puppetfile
 
   include R10K::Settings::Mixin
 
-  def_setting_attr :pool_size, 1
+  def_setting_attr :pool_size, 4
 
   include R10K::Logging
 
@@ -42,17 +43,32 @@ class Puppetfile
   #   @return [Boolean] Overwrite any locally made changes
   attr_accessor :force
 
+  # @!attribute [r] overrides
+  #   @return [Hash] Various settings overridden from normal configs
+  attr_reader :overrides
+
   # @param [String] basedir
-  # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules
-  # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile
-  # @param [String] puppetfile_name The name of the Puppetfile, default to 'Puppetfile'
-  # @param [Boolean] force Shall we overwrite locally made changes?
-  def initialize(basedir, moduledir = nil, puppetfile_path = nil, puppetfile_name = nil, force = nil )
+  # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options.
+  #         Usage as moduledir is deprecated. Only use as options, defaults to nil
+  # @param [String, nil] puppetfile_path Deprecated - The path to the Puppetfile, defaults to nil
+  # @param [String, nil] puppetfile_name Deprecated - The name of the Puppetfile, defaults to nil
+  # @param [Boolean, nil] force Deprecated - Shall we overwrite locally made changes?
+  def initialize(basedir, options_or_moduledir = nil, deprecated_path_arg = nil, deprecated_name_arg = nil, deprecated_force_arg = nil)
     @basedir         = basedir
-    @force           = force || false
-    @moduledir       = moduledir  || File.join(basedir, 'modules')
-    @puppetfile_name = puppetfile_name || 'Puppetfile'
-    @puppetfile_path = puppetfile_path || File.join(basedir, @puppetfile_name)
+    if options_or_moduledir.is_a? Hash
+      options = options_or_moduledir
+      deprecated_moduledir_arg = nil
+    else
+      options = {}
+      deprecated_moduledir_arg = options_or_moduledir
+    end
+
+    @force           = deprecated_force_arg     || options.delete(:force)           || false
+    @moduledir       = deprecated_moduledir_arg || options.delete(:moduledir)       || File.join(basedir, 'modules')
+    @puppetfile_name = deprecated_name_arg      || options.delete(:puppetfile_name) || 'Puppetfile'
+    @puppetfile_path = deprecated_path_arg      || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
+
+    @overrides       = options.delete(:overrides) || {}
 
     logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
 
@@ -77,7 +93,7 @@ class Puppetfile
 
     dsl = R10K::Puppetfile::DSL.new(self)
     dsl.instance_eval(puppetfile_contents, @puppetfile_path)
-    
+
     validate_no_duplicate_names(@modules)
     @loaded = true
   rescue SyntaxError, LoadError, ArgumentError, NameError => e
@@ -117,26 +133,44 @@ class Puppetfile
   end
 
   # @param [String] name
-  # @param [*Object] args
+  # @param [Hash, String, Symbol] args Calling with anything but a Hash is
+  #   deprecated. The DSL will now convert String and Symbol versions to
+  #   Hashes of the shape
+  #     { version: <String or Symbol> }
+  #
   def add_module(name, args)
-    if args.is_a?(Hash) && install_path = args.delete(:install_path)
+    if !args.is_a?(Hash)
+      args = { version: args }
+    end
+
+    args[:overrides] = @overrides
+
+    if install_path = args.delete(:install_path)
       install_path = resolve_install_path(install_path)
       validate_install_path(install_path, name)
     else
       install_path = @moduledir
     end
 
-    if args.is_a?(Hash) && @default_branch_override != nil
-      args[:default_branch] = @default_branch_override
+    if @default_branch_override != nil
+      args[:default_branch_override] = @default_branch_override
     end
 
-    # Keep track of all the content this Puppetfile is managing to enable purging.
-    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
 
     mod = R10K::Module.new(name, install_path, args, @environment)
-    mod.origin = 'Puppetfile'
+    mod.origin = :puppetfile
+
+    # Do not load modules if they would conflict with the attached
+    # environment
+    if environment && environment.module_conflicts?(mod)
+      mod = nil
+      return @modules
+    end
 
+    # Keep track of all the content this Puppetfile is managing to enable purging.
+    @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
     @managed_content[install_path] << mod.name
+
     @modules << mod
   end
 
@@ -145,7 +179,9 @@ class Puppetfile
   def managed_directories
     self.load unless @loaded
 
-    @managed_content.keys
+    dirs = @managed_content.keys
+    dirs.delete(real_basedir)
+    dirs
   end
 
   # Returns an array of the full paths to all the content being managed.
@@ -172,63 +208,22 @@ class Puppetfile
   def accept(visitor)
     pool_size = self.settings[:pool_size]
     if pool_size > 1
-      concurrent_accept(visitor, pool_size)
+      R10K::ContentSynchronizer.concurrent_accept(modules, visitor, self, pool_size, logger)
     else
-      serial_accept(visitor)
-    end
-  end
-
-  private
-
-  def serial_accept(visitor)
-    visitor.visit(:puppetfile, self) do
-      modules.each do |mod|
-        mod.accept(visitor)
-      end
-    end
-  end
-
-  def concurrent_accept(visitor, pool_size)
-    logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
-    mods_queue = modules_queue(visitor)
-    thread_pool = pool_size.times.map { visitor_thread(visitor, mods_queue) }
-    thread_exception = nil
-
-    # If any threads raise an exception the deployment is considered a failure.
-    # In that event clear the queue, wait for other threads to finish their
-    # current work, then re-raise the first exception caught.
-    begin
-      thread_pool.each(&:join)
-    rescue => e
-      logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
-      mods_queue.clear
-      thread_exception ||= e
-      retry
-    ensure
-      raise thread_exception unless thread_exception.nil?
+      R10K::ContentSynchronizer.serial_accept(modules, visitor, self)
     end
   end
 
-  def modules_queue(visitor)
-    Queue.new.tap do |queue|
-      visitor.visit(:puppetfile, self) do
-        modules.each { |mod| queue << mod }
-      end
+  def sync
+    pool_size = self.settings[:pool_size]
+    if pool_size > 1
+      R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger)
+    else
+      R10K::ContentSynchronizer.serial_sync(modules)
     end
   end
 
-  def visitor_thread(visitor, mods_queue)
-    Thread.new do
-      begin
-        while mod = mods_queue.pop(true) do mod.accept(visitor) end
-      rescue ThreadError => e
-        logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
-        Thread.exit
-      rescue => e
-        Thread.main.raise(e)
-      end
-    end
-  end
+  private
 
   def puppetfile_contents
     File.read(@puppetfile_path)
@@ -248,8 +243,6 @@ class Puppetfile
   end
 
   def validate_install_path(path, modname)
-    real_basedir = Pathname.new(basedir).cleanpath.to_s
-
     unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
       raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
     end
@@ -257,6 +250,10 @@ class Puppetfile
     true
   end
 
+  def real_basedir
+    Pathname.new(basedir).cleanpath.to_s
+  end
+
   class DSL
     # A barebones implementation of the Puppetfile DSL
     #
@@ -267,7 +264,13 @@ class Puppetfile
     end
 
     def mod(name, args = nil)
-      @librarian.add_module(name, args)
+      if args.is_a?(Hash)
+        opts = args
+      else
+        opts = { version: args }
+      end
+
+      @librarian.add_module(name, opts)
     end
 
     def forge(location)

data/lib/r10k/settings.rb

@@ -12,6 +12,8 @@ module R10K
     class << self
       # Path to puppet executable
       attr_accessor :puppet_path
+      # Path to puppet.conf
+      attr_accessor :puppet_conf
     end
 
     def self.git_settings
@@ -35,6 +37,11 @@ module R10K
                     Only used by the 'rugged' Git provider.",
         }),
 
+        Definition.new(:oauth_token, {
+          :desc => "The path to a token file for Git OAuth remotes.
+                    Only used by the 'rugged' Git provider."
+        }),
+
         URIDefinition.new(:proxy, {
           :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
           :default => :inherit,
@@ -52,11 +59,17 @@ module R10K
               :default => :inherit,
             }),
 
+            Definition.new(:oauth_token, {
+              :desc => "The path to a token file for Git OAuth remotes.
+                        Only used by the 'rugged' Git provider.",
+              :default => :inherit
+            }),
+
             URIDefinition.new(:proxy, {
               :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
               :default => :inherit,
             }),
-            
+
             Definition.new(:ignore_branch_prefixes, {
               :desc => "Array of strings used to prefix branch names that will not be deployed as environments.",
             }),
@@ -109,11 +122,16 @@ module R10K
           end,
         }),
 
-        Definition.new(:purge_whitelist, {
+        Definition.new(:purge_allowlist, {
           :desc => "A list of filename patterns to be excluded from any purge operations. Patterns are matched relative to the root of each deployed environment, if you want a pattern to match recursively you need to use the '**' glob in your pattern. Basic shell style globs are supported.",
           :default => [],
         }),
 
+        Definition.new(:purge_whitelist, {
+          :desc => "Deprecated; please use purge_allowlist instead. This setting will be removed in a future version.",
+          :default => [],
+        }),
+
         Definition.new(:generate_types, {
           :desc => "Controls whether to generate puppet types after deploying an environment. Defaults to false.",
           :default => false,
@@ -131,6 +149,15 @@ module R10K
             end
           end
         }),
+        Definition.new(:puppet_conf, {
+          :desc => "Path to puppet.conf. Defaults to /etc/puppetlabs/puppet/puppet.conf.",
+          :default => '/etc/puppetlabs/puppet/puppet.conf',
+          :validate => lambda do |value|
+            unless File.readable? value
+              raise ArgumentError, "The specified puppet.conf #{value} is not readable"
+            end
+          end
+        }),
       ])
     end
 
@@ -160,7 +187,7 @@ module R10K
 
         Definition.new(:pool_size, {
           :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
-          :default => 1,
+          :default => 4,
           :validate => lambda do |value|
             if !value.is_a?(Integer)
               raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"