r10k 3.9.0 → 3.9.1

data/lib/r10k/action/puppetfile/check.rb

@@ -8,7 +8,9 @@ module R10K
       class Check < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
+          pf = R10K::Puppetfile.new(@root,
+                                    {moduledir: @moduledir,
+                                     puppetfile_path: @puppetfile})
           begin
             pf.load!
             $stderr.puts _("Syntax OK")

data/lib/r10k/action/puppetfile/install.rb

@@ -2,6 +2,7 @@ require 'r10k/puppetfile'
 require 'r10k/errors/formatting'
 require 'r10k/action/visitor'
 require 'r10k/action/base'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,33 +10,29 @@ module R10K
       class Install < R10K::Action::Base
 
         def call
-          @visit_ok = true
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force)
-          pf.accept(self)
-          @visit_ok
-        end
-
-        private
-
-        include R10K::Action::Visitor
-
-        def visit_puppetfile(pf)
-          pf.load!
-          yield
-          pf.purge!
-        end
-
-        def visit_module(mod)
-          @force ||= false
-          logger.info _("Updating module %{mod_path}") % {mod_path: mod.path}
-
-          if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch
-            logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name})
+          @ok = true
+          begin
+            pf = R10K::Puppetfile.new(@root,
+                                      {moduledir: @moduledir,
+                                       puppetfile_path: @puppetfile,
+                                       force: @force || false})
+            pf.load!
+            pf.sync
+
+            R10K::Util::Cleaner.new(pf.managed_directories,
+                                    pf.desired_contents,
+                                    pf.purge_exclusions).purge!
+
+          rescue => e
+            @ok = false
+            logger.error R10K::Errors::Formatting.format_exception(e, @trace)
           end
 
-          mod.sync(force: @force)
+          @ok
         end
 
+        private
+
         def allowed_initialize_opts
           super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self )
         end

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,4 +1,5 @@
 require 'r10k/puppetfile'
+require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
 
@@ -8,9 +9,14 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
+          pf = R10K::Puppetfile.new(@root,
+                                    {moduledir: @moduledir,
+                                     puppetfile_path: @puppetfile})
           pf.load!
-          pf.purge!
+          R10K::Util::Cleaner.new(pf.managed_directories,
+                                  pf.desired_contents,
+                                  pf.purge_exclusions).purge!
+
           true
         rescue => e
           logger.error R10K::Errors::Formatting.format_exception(e, @trace)

data/lib/r10k/content_synchronizer.rb

@@ -0,0 +1,83 @@
+module R10K
+  module ContentSynchronizer
+
+    def self.serial_accept(modules, visitor, loader)
+      visitor.visit(:puppetfile, loader) do
+        serial_sync(modules)
+      end
+    end
+
+    def self.serial_sync(modules)
+      modules.each do |mod|
+        mod.sync
+      end
+    end
+
+    def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
+      mods_queue = modules_visit_queue(modules, visitor, loader)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    def self.concurrent_sync(modules, pool_size, logger)
+      mods_queue = modules_sync_queue(modules)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    def self.sync_queue(mods_queue, pool_size, logger)
+      logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
+      thread_exception = nil
+
+      # If any threads raise an exception the deployment is considered a failure.
+      # In that event clear the queue, wait for other threads to finish their
+      # current work, then re-raise the first exception caught.
+      begin
+        thread_pool.each(&:join)
+      rescue => e
+        logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
+        mods_queue.clear
+        thread_exception ||= e
+        retry
+      ensure
+        raise thread_exception unless thread_exception.nil?
+      end
+    end
+
+    def self.modules_visit_queue(modules, visitor, loader)
+      Queue.new.tap do |queue|
+        visitor.visit(:puppetfile, loader) do
+          enqueue_modules(queue, modules)
+        end
+      end
+    end
+
+    def self.modules_sync_queue(modules)
+      Queue.new.tap do |queue|
+        enqueue_modules(queue, modules)
+      end
+    end
+
+    def self.enqueue_modules(queue, modules)
+      modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
+      modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+      modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+      modules_by_cachedir.values.each {|mods| queue << mods }
+    end
+
+    def self.sync_thread(mods_queue, logger)
+      Thread.new do
+        begin
+          while mods = mods_queue.pop(true) do
+            mods.each { |mod| mod.sync }
+          end
+        rescue ThreadError => e
+          logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
+          Thread.exit
+        rescue => e
+          Thread.main.raise(e)
+        end
+      end
+    end
+  end
+end

data/lib/r10k/deployment.rb

@@ -118,7 +118,7 @@ module R10K
         raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key")
       end
       @_sources = sources.map do |(name, hash)|
-        R10K::Source.from_hash(name, hash)
+        R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]}))
       end
     end
 

data/lib/r10k/environment/base.rb

@@ -1,10 +1,13 @@
 require 'r10k/util/subprocess'
+require 'r10k/logging'
 
 # This class defines a common interface for environment implementations.
 #
 # @since 1.3.0
 class R10K::Environment::Base
 
+  include R10K::Logging
+
   # @!attribute [r] name
   #   @return [String] A name for this environment that is unique to the given source
   attr_reader :name
@@ -44,11 +47,15 @@ class R10K::Environment::Base
     @dirname = dirname
     @options = options
     @puppetfile_name = options.delete(:puppetfile_name)
+    @overrides = options.delete(:overrides) || {}
 
     @full_path = File.join(@basedir, @dirname)
     @path = Pathname.new(File.join(@basedir, @dirname))
 
-    @puppetfile  = R10K::Puppetfile.new(@full_path, nil, nil, @puppetfile_name)
+    @puppetfile  = R10K::Puppetfile.new(@full_path,
+                                        {overrides: @overrides,
+                                         force: @overrides.dig(:modules, :force),
+                                         puppetfile_name: @puppetfile_name})
     @puppetfile.environment = self
   end
 
@@ -116,6 +123,19 @@ class R10K::Environment::Base
     end
   end
 
+  def deploy
+    puppetfile.load(@overrides.dig(:environments, :default_branch_override))
+
+    puppetfile.sync
+
+    if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
+      logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
+      R10K::Util::Cleaner.new(puppetfile.managed_directories,
+                              puppetfile.desired_contents,
+                              puppetfile.purge_exclusions).purge!
+    end
+  end
+
   def whitelist(user_whitelist=[])
     user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
   end

data/lib/r10k/environment/git.rb

@@ -1,4 +1,3 @@
-require 'r10k/logging'
 require 'r10k/puppetfile'
 require 'r10k/git/stateful_repository'
 require 'forwardable'
@@ -8,8 +7,6 @@ require 'forwardable'
 # @since 1.3.0
 class R10K::Environment::Git < R10K::Environment::WithModules
 
-  include R10K::Logging
-
   R10K::Environment.register(:git, self)
   # Register git as the default environment type
   R10K::Environment.register(nil, self)

data/lib/r10k/environment/svn.rb

@@ -7,8 +7,6 @@ require 'r10k/util/setopts'
 # @since 1.3.0
 class R10K::Environment::SVN < R10K::Environment::Base
 
-  include R10K::Logging
-
   R10K::Environment.register(:svn, self)
 
   # @!attribute [r] remote
@@ -46,12 +44,12 @@ class R10K::Environment::SVN < R10K::Environment::Base
     super
     setopts(options, {
       # Standard option interface
-      :source  => :remote,
-      :version => :expected_revision,
-      :type    => ::R10K::Util::Setopts::Ignore,
+      :source   => :remote,
+      :version  => :expected_revision,
+      :type     => ::R10K::Util::Setopts::Ignore,
 
       # Type-specific options
-      :remote => :self,
+      :remote   => :self,
       :username => :self,
       :password => :self,
     })

data/lib/r10k/environment/with_modules.rb

@@ -1,4 +1,3 @@
-require 'r10k/logging'
 require 'r10k/util/purgeable'
 
 # This abstract base class implements an environment that can include module
@@ -7,8 +6,6 @@ require 'r10k/util/purgeable'
 # @since 3.4.0
 class R10K::Environment::WithModules < R10K::Environment::Base
 
-  include R10K::Logging
-
   # @!attribute [r] moduledir
   #   @return [String] The directory to install environment-defined modules
   #     into (default: #{basedir}/modules)
@@ -78,28 +75,39 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   def accept(visitor)
     visitor.visit(:environment, self) do
       @modules.each do |mod|
-        mod.accept(visitor)
+        mod.sync
       end
 
       puppetfile.accept(visitor)
     end
   end
 
+  def deploy
+    @modules.each do |mod|
+      mod.sync
+    end
+
+    super
+  end
+
   def load_modules(module_hash)
     module_hash.each do |name, args|
+      if !args.is_a?(Hash)
+        args = { version: args }
+      end
+
       add_module(name, args)
     end
   end
 
   # @param [String] name
-  # @param [*Object] args
+  # @param [Hash] args
   def add_module(name, args)
-    if args.is_a?(Hash)
-      # symbolize keys in the args hash
-      args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo }
-    end
+    # symbolize keys in the args hash
+    args = args.inject({}) { |memo,(k,v)| memo[k.to_sym] = v; memo }
+    args[:overrides] = @overrides
 
-    if args.is_a?(Hash) && install_path = args.delete(:install_path)
+    if install_path = args.delete(:install_path)
       install_path = resolve_install_path(install_path)
       validate_install_path(install_path, name)
     else

data/lib/r10k/module.rb

@@ -17,7 +17,7 @@ module R10K::Module
   #
   # @param [String] name The unique name of the module
   # @param [String] basedir The root to install the module in
-  # @param [Object] args An arbitary value or set of values that specifies the implementation
+  # @param [Hash] args An arbitary Hash that specifies the implementation
   # @param [R10K::Environment] environment Optional environment that this module is a part of
   #
   # @return [Object < R10K::Module] A member of the implementing subclass

data/lib/r10k/module/base.rb

@@ -51,7 +51,11 @@ class R10K::Module::Base
     @owner, @name = parse_title(@title)
     @path = Pathname.new(File.join(@dirname, @name))
     @environment = environment
+    @overrides = args.delete(:overrides) || {}
     @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
+
+    @requested_modules = @overrides.dig(:modules, :requested_modules) || []
+    @should_sync = (@requested_modules.empty? || @requested_modules.include?(@name))
   end
 
   # @deprecated
@@ -61,11 +65,22 @@ class R10K::Module::Base
   end
 
   # Synchronize this module with the indicated state.
-  # @abstract
+  # @param [Hash] opts Deprecated
   def sync(opts={})
     raise NotImplementedError
   end
 
+  def should_sync?
+    if @should_sync
+      logger.info _("Deploying module to %{path}") % {path: path}
+      true
+    else
+      logger.debug1(_("Only updating modules %{modules}, skipping module %{name}") % {modules: @requested_modules.inspect, name: name})
+      false
+    end
+  end
+
+
   # Return the desired version of this module
   # @abstract
   def version
@@ -87,6 +102,7 @@ class R10K::Module::Base
     raise NotImplementedError
   end
 
+  # Deprecated
   def accept(visitor)
     visitor.visit(:module, self)
   end

data/lib/r10k/module/forge.rb

@@ -13,7 +13,12 @@ class R10K::Module::Forge < R10K::Module::Base
   R10K::Module.register(self)
 
   def self.implement?(name, args)
-    (args.is_a?(Hash) && args[:type].to_s == 'forge') || (!!(name.match %r[\w+[/-]\w+]) && valid_version?(args))
+    args[:type].to_s == 'forge' ||
+      (!!
+       ((args.keys & %i{git svn type}).empty? &&
+        args.has_key?(:version) &&
+        name.match(%r[\w+[/-]\w+]) &&
+        valid_version?(args[:version])))
   end
 
   def self.valid_version?(expected_version)
@@ -40,28 +45,29 @@ class R10K::Module::Forge < R10K::Module::Base
     @metadata_file = R10K::Module::MetadataFile.new(path + 'metadata.json')
     @metadata = @metadata_file.read
 
-    if opts.is_a?(Hash)
-      setopts(opts, {
-        # Standard option interface
-        :version => :expected_version,
-        :source  => ::R10K::Util::Setopts::Ignore,
-        :type    => ::R10K::Util::Setopts::Ignore,
-      })
-    else
-      @expected_version = opts || current_version || :latest
-    end
+    setopts(opts, {
+      # Standard option interface
+      :version => :expected_version,
+      :source  => ::R10K::Util::Setopts::Ignore,
+      :type    => ::R10K::Util::Setopts::Ignore,
+    })
+
+    @expected_version ||= current_version || :latest
 
     @v3_module = PuppetForge::V3::Module.new(:slug => @title)
   end
 
+  # @param [Hash] opts Deprecated
   def sync(opts={})
-    case status
-    when :absent
-      install
-    when :outdated
-      upgrade
-    when :mismatched
-      reinstall
+    if should_sync?
+      case status
+      when :absent
+        install
+      when :outdated
+        upgrade
+      when :mismatched
+        reinstall
+      end
     end
   end