r10k 3.5.2 → 3.9.1

data/lib/r10k/action/puppetfile/check.rb

@@ -8,7 +8,9 @@ module R10K
       class Check < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
+          pf = R10K::Puppetfile.new(@root,
+                                    {moduledir: @moduledir,
+                                     puppetfile_path: @puppetfile})
           begin
             pf.load!
             $stderr.puts _("Syntax OK")

data/lib/r10k/action/puppetfile/install.rb

@@ -2,6 +2,7 @@ require 'r10k/puppetfile'
 require 'r10k/errors/formatting'
 require 'r10k/action/visitor'
 require 'r10k/action/base'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,33 +10,29 @@ module R10K
       class Install < R10K::Action::Base
 
         def call
-          @visit_ok = true
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force)
-          pf.accept(self)
-          @visit_ok
-        end
-
-        private
-
-        include R10K::Action::Visitor
-
-        def visit_puppetfile(pf)
-          pf.load!
-          yield
-          pf.purge!
-        end
-
-        def visit_module(mod)
-          @force ||= false
-          logger.info _("Updating module %{mod_path}") % {mod_path: mod.path}
-
-          if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch
-            logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name})
+          @ok = true
+          begin
+            pf = R10K::Puppetfile.new(@root,
+                                      {moduledir: @moduledir,
+                                       puppetfile_path: @puppetfile,
+                                       force: @force || false})
+            pf.load!
+            pf.sync
+
+            R10K::Util::Cleaner.new(pf.managed_directories,
+                                    pf.desired_contents,
+                                    pf.purge_exclusions).purge!
+
+          rescue => e
+            @ok = false
+            logger.error R10K::Errors::Formatting.format_exception(e, @trace)
           end
 
-          mod.sync(force: @force)
+          @ok
         end
 
+        private
+
         def allowed_initialize_opts
           super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self )
         end

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,4 +1,5 @@
 require 'r10k/puppetfile'
+require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
 
@@ -8,9 +9,14 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
+          pf = R10K::Puppetfile.new(@root,
+                                    {moduledir: @moduledir,
+                                     puppetfile_path: @puppetfile})
           pf.load!
-          pf.purge!
+          R10K::Util::Cleaner.new(pf.managed_directories,
+                                  pf.desired_contents,
+                                  pf.purge_exclusions).purge!
+
           true
         rescue => e
           logger.error R10K::Errors::Formatting.format_exception(e, @trace)

data/lib/r10k/action/runner.rb

@@ -46,6 +46,7 @@ module R10K
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
         overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
         overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
         overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
@@ -54,6 +55,10 @@ module R10K
           newval
         end
 
+        # Credentials from the CLI override both the global and per-repo
+        # credentials from the config, and so need to be handled specially
+        with_overrides = add_credential_overrides(with_overrides)
+
         @settings = R10K::Settings.global_settings.evaluate(with_overrides)
 
         R10K::Initializers::GlobalInitializer.new(@settings).call
@@ -91,6 +96,35 @@ module R10K
 
         results
       end
+
+      def add_credential_overrides(overrides)
+        sshkey_path = @opts[:'private-key']
+        token_path = @opts[:'oauth-token']
+
+        if sshkey_path && token_path
+          raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
+        end
+
+        if sshkey_path
+          overrides[:git] ||= {}
+          overrides[:git][:private_key] = sshkey_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:private_key] = sshkey_path
+            end
+          end
+        elsif token_path
+          overrides[:git] ||= {}
+          overrides[:git][:oauth_token] = token_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:oauth_token] = token_path
+            end
+          end
+        end
+
+        overrides
+      end
     end
   end
 end

data/lib/r10k/cli/deploy.rb

@@ -21,7 +21,7 @@ module R10K::CLI
 (https://puppet.com/docs/puppet/latest/environments_about.html).
         DESCRIPTION
 
-        required nil, :cachedir, 'Specify a cachedir, overriding the value in config'
+        option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
@@ -31,6 +31,9 @@ module R10K::CLI
             exit 1
           end
         end
+        option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
+        option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])
@@ -52,7 +55,7 @@ branches.
 
 Environments can provide a Puppetfile at the root of the directory to deploy
 independent Puppet modules. To recursively deploy an environment, pass the
-`--puppetfile` flag to the command.
+`--modules` flag to the command.
 
 **NOTE**: If an environment has a Puppetfile when it is instantiated a
 recursive update will be forced. It is assumed that environments are dependent
@@ -60,8 +63,10 @@ on modules specified in the Puppetfile and an update will be automatically
 scheduled. On subsequent deployments, Puppetfile deployment will default to off.
           DESCRIPTION
 
-          flag :p, :puppetfile, 'Deploy modules from a puppetfile'
-          required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
+          flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
+          flag :m, :modules, 'Deploy modules'
+          option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
         end
@@ -81,7 +86,7 @@ It will load the Puppetfile configurations out of all environments, and will
 try to deploy the given module names in all environments.
           DESCRIPTION
 
-          required :e, :environment, 'Update the modules in the given environment'
+          option :e, :environment, 'Update the modules in the given environment', argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module)
         end
@@ -96,10 +101,12 @@ try to deploy the given module names in all environments.
           usage 'display'
           summary 'Display environments and modules in the deployment'
 
-          flag :p, :puppetfile, 'Display Puppetfile modules'
+          flag :p, :puppetfile, 'Display modules (deprecated, use -m)'
+          flag :m, :modules, 'Display modules'
           flag nil, :detail, 'Display detailed information'
           flag nil, :fetch, 'Update available environment lists from all remote sources'
-          required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml'
+          option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display)
         end

data/lib/r10k/cli/puppetfile.rb

@@ -30,8 +30,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           name    'install'
           usage   'install'
           summary 'Install all modules from a Puppetfile'
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to puppetfile', argument: :required
           flag     nil, :force, 'Force locally changed files to be overwritten'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install)
         end
@@ -45,7 +45,7 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'check'
           summary 'Try and load the Puppetfile to verify the syntax is correct.'
 
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check)
         end
       end
@@ -58,8 +58,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'purge'
           summary 'Purge unmanaged modules from a Puppetfile managed directory'
 
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge)
         end
       end

data/lib/r10k/content_synchronizer.rb

@@ -0,0 +1,83 @@
+module R10K
+  module ContentSynchronizer
+
+    def self.serial_accept(modules, visitor, loader)
+      visitor.visit(:puppetfile, loader) do
+        serial_sync(modules)
+      end
+    end
+
+    def self.serial_sync(modules)
+      modules.each do |mod|
+        mod.sync
+      end
+    end
+
+    def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
+      mods_queue = modules_visit_queue(modules, visitor, loader)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    def self.concurrent_sync(modules, pool_size, logger)
+      mods_queue = modules_sync_queue(modules)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    def self.sync_queue(mods_queue, pool_size, logger)
+      logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
+      thread_exception = nil
+
+      # If any threads raise an exception the deployment is considered a failure.
+      # In that event clear the queue, wait for other threads to finish their
+      # current work, then re-raise the first exception caught.
+      begin
+        thread_pool.each(&:join)
+      rescue => e
+        logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
+        mods_queue.clear
+        thread_exception ||= e
+        retry
+      ensure
+        raise thread_exception unless thread_exception.nil?
+      end
+    end
+
+    def self.modules_visit_queue(modules, visitor, loader)
+      Queue.new.tap do |queue|
+        visitor.visit(:puppetfile, loader) do
+          enqueue_modules(queue, modules)
+        end
+      end
+    end
+
+    def self.modules_sync_queue(modules)
+      Queue.new.tap do |queue|
+        enqueue_modules(queue, modules)
+      end
+    end
+
+    def self.enqueue_modules(queue, modules)
+      modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
+      modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+      modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+      modules_by_cachedir.values.each {|mods| queue << mods }
+    end
+
+    def self.sync_thread(mods_queue, logger)
+      Thread.new do
+        begin
+          while mods = mods_queue.pop(true) do
+            mods.each { |mod| mod.sync }
+          end
+        rescue ThreadError => e
+          logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
+          Thread.exit
+        rescue => e
+          Thread.main.raise(e)
+        end
+      end
+    end
+  end
+end

data/lib/r10k/deployment.rb

@@ -118,7 +118,7 @@ module R10K
         raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key")
       end
       @_sources = sources.map do |(name, hash)|
-        R10K::Source.from_hash(name, hash)
+        R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]}))
       end
     end
 

data/lib/r10k/environment/base.rb

@@ -1,10 +1,13 @@
 require 'r10k/util/subprocess'
+require 'r10k/logging'
 
 # This class defines a common interface for environment implementations.
 #
 # @since 1.3.0
 class R10K::Environment::Base
 
+  include R10K::Logging
+
   # @!attribute [r] name
   #   @return [String] A name for this environment that is unique to the given source
   attr_reader :name
@@ -43,12 +46,16 @@ class R10K::Environment::Base
     @basedir = basedir
     @dirname = dirname
     @options = options
-    @puppetfile_name = options[:puppetfile_name]
+    @puppetfile_name = options.delete(:puppetfile_name)
+    @overrides = options.delete(:overrides) || {}
 
     @full_path = File.join(@basedir, @dirname)
     @path = Pathname.new(File.join(@basedir, @dirname))
 
-    @puppetfile  = R10K::Puppetfile.new(@full_path, nil, nil, @puppetfile_name)
+    @puppetfile  = R10K::Puppetfile.new(@full_path,
+                                        {overrides: @overrides,
+                                         force: @overrides.dig(:modules, :force),
+                                         puppetfile_name: @puppetfile_name})
     @puppetfile.environment = self
   end
 
@@ -103,12 +110,32 @@ class R10K::Environment::Base
     @puppetfile.modules
   end
 
+  # @return [Array<R10K::Module::Base>] Whether or not the given module
+  #   conflicts with any modules already defined in the r10k environment
+  #   object.
+  def module_conflicts?(mod)
+    false
+  end
+
   def accept(visitor)
     visitor.visit(:environment, self) do
       puppetfile.accept(visitor)
     end
   end
 
+  def deploy
+    puppetfile.load(@overrides.dig(:environments, :default_branch_override))
+
+    puppetfile.sync
+
+    if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
+      logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
+      R10K::Util::Cleaner.new(puppetfile.managed_directories,
+                              puppetfile.desired_contents,
+                              puppetfile.purge_exclusions).purge!
+    end
+  end
+
   def whitelist(user_whitelist=[])
     user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
   end
@@ -137,7 +164,7 @@ class R10K::Environment::Base
   end
 
   def generate_types!
-    argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir]
+    argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
     subproc = R10K::Util::Subprocess.new(argv)
     subproc.raise_on_fail = true
     subproc.logger = logger

data/lib/r10k/environment/git.rb

@@ -1,4 +1,3 @@
-require 'r10k/logging'
 require 'r10k/puppetfile'
 require 'r10k/git/stateful_repository'
 require 'forwardable'
@@ -8,8 +7,6 @@ require 'forwardable'
 # @since 1.3.0
 class R10K::Environment::Git < R10K::Environment::WithModules
 
-  include R10K::Logging
-
   R10K::Environment.register(:git, self)
   # Register git as the default environment type
   R10K::Environment.register(nil, self)
@@ -27,6 +24,8 @@ class R10K::Environment::Git < R10K::Environment::WithModules
   #   @return [R10K::Git::StatefulRepository] The git repo backing this environment
   attr_reader :repo
 
+  include R10K::Util::Setopts
+
   # Initialize the given Git environment.
   #
   # @param name [String] The unique name describing this environment.
@@ -38,8 +37,21 @@ class R10K::Environment::Git < R10K::Environment::WithModules
   # @param options [String] :ref The git reference to use for this environment
   def initialize(name, basedir, dirname, options = {})
     super
-    @remote = options[:remote]
-    @ref    = options[:ref]
+    setopts(options, {
+      # Standard option interface
+      :version => :ref,
+      :source  => :remote,
+      :type    => ::R10K::Util::Setopts::Ignore,
+
+      # Type-specific options
+      :ref     => :self,
+      :remote  => :self,
+
+    }, raise_on_unhandled: false)
+    # TODO: in r10k 4.0.0, a major version bump, stop allowing garbage options.
+    # We only allow them now, here, on this object, because prior to adopting
+    # setopts in the constructor, this object type didn't do any validation
+    # checking of options passed, and would permit garbage parameters.
 
     @repo = R10K::Git::StatefulRepository.new(@remote, @basedir, @dirname)
   end