r10k 3.11.0 → 3.12.0

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,13 +9,16 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
-          pf.load!
-          R10K::Util::Cleaner.new(pf.managed_directories,
-                                  pf.desired_contents,
-                                  pf.purge_exclusions).purge!
+          options = { basedir: @root }
+
+          options[:moduledir]  = @moduledir  if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
+          loaded_content = loader.load!
+          R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                  loaded_content[:desired_contents],
+                                  loaded_content[:purge_exclusions]).purge!
 
           true
         rescue => e

data/lib/r10k/cli/deploy.rb

@@ -69,6 +69,7 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
 
           flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
           flag :m, :modules, 'Deploy modules'
+          flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float'
           option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
                  argument: :required
 

data/lib/r10k/cli/puppetfile.rb

@@ -1,5 +1,4 @@
 require 'r10k/cli'
-require 'r10k/puppetfile'
 require 'r10k/action/puppetfile'
 
 require 'cri'

data/lib/r10k/content_synchronizer.rb

@@ -8,24 +8,31 @@ module R10K
     end
 
     def self.serial_sync(modules)
+      updated_modules = []
       modules.each do |mod|
-        mod.sync
+        updated = mod.sync
+        updated_modules << mod.name if updated
       end
+      updated_modules
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
       mods_queue = modules_visit_queue(modules, visitor, loader)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_sync(modules, pool_size, logger)
       mods_queue = modules_sync_queue(modules)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.sync_queue(mods_queue, pool_size, logger)
       logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
-      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
+      updated_modules = Queue.new
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) }
       thread_exception = nil
 
       # If any threads raise an exception the deployment is considered a failure.
@@ -33,6 +40,8 @@ module R10K
       # current work, then re-raise the first exception caught.
       begin
         thread_pool.each(&:join)
+        # Return the list of all modules that were actually updated
+        updated_modules
       rescue => e
         logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
         mods_queue.clear
@@ -65,11 +74,14 @@ module R10K
       modules_by_cachedir.values.each {|mods| queue << mods }
     end
 
-    def self.sync_thread(mods_queue, logger)
+    def self.sync_thread(mods_queue, logger, updated_modules)
       Thread.new do
         begin
           while mods = mods_queue.pop(true) do
-            mods.each { |mod| mod.sync }
+            mods.each do |mod|
+              updated = mod.sync
+              updated_modules << mod.name if updated
+            end
           end
         rescue ThreadError => e
           logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}

data/lib/r10k/environment/base.rb

@@ -1,5 +1,8 @@
-require 'r10k/util/subprocess'
+require 'r10k/content_synchronizer'
 require 'r10k/logging'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
+require 'r10k/util/subprocess'
 
 # This class defines a common interface for environment implementations.
 #
@@ -34,6 +37,10 @@ class R10K::Environment::Base
   #   @return [String] The puppetfile name (relative)
   attr_reader :puppetfile_name
 
+  attr_reader :managed_directories, :desired_contents
+
+  attr_reader :loader
+
   # Initialize the given environment.
   #
   # @param name [String] The unique name describing this environment.
@@ -57,6 +64,20 @@ class R10K::Environment::Base
                                          force: @overrides.dig(:modules, :force),
                                          puppetfile_name: @puppetfile_name})
     @puppetfile.environment = self
+
+    loader_options = { basedir: @full_path, overrides: @overrides, environment: self }
+    loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name
+
+    @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options)
+
+    if @overrides.dig(:environments, :incremental)
+      @loader.load_metadata
+    end
+
+    @base_modules = nil
+    @purge_exclusions = nil
+    @managed_directories = [ @full_path ]
+    @desired_contents = []
   end
 
   # Synchronize the given environment.
@@ -106,8 +127,11 @@ class R10K::Environment::Base
   # @return [Array<R10K::Module::Base>] All modules defined in the Puppetfile
   #   associated with this environment.
   def modules
-    @puppetfile.load
-    @puppetfile.modules
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
+
+    @base_modules
   end
 
   # @return [Array<R10K::Module::Base>] Whether or not the given module
@@ -123,29 +147,50 @@ class R10K::Environment::Base
     end
   end
 
+
+  # Returns a Queue of the names of modules actually updated
   def deploy
-    puppetfile.load(@overrides.dig(:environments, :default_branch_override))
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
 
-    puppetfile.sync
+    if ! @base_modules.empty?
+      pool_size = @overrides.dig(:modules, :pool_size)
+      updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger)
+    end
 
     if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
       logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
-      R10K::Util::Cleaner.new(puppetfile.managed_directories,
-                              puppetfile.desired_contents,
-                              puppetfile.purge_exclusions).purge!
+      @puppetfile_cleaner.purge!
     end
+
+    updated_modules
+  end
+
+  def load_puppetfile_modules
+    loaded_content = @loader.load
+    @base_modules = loaded_content[:modules]
+
+    @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories],
+                                                   loaded_content[:desired_contents])
+
+    @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                                  loaded_content[:desired_contents],
+                                                  loaded_content[:purge_exclusions])
   end
 
   def whitelist(user_whitelist=[])
     user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
   end
 
-  def purge_exclusions
+  def determine_purge_exclusions(pf_managed_dirs     = @puppetfile.managed_directories,
+                                 pf_desired_contents = @puppetfile.desired_contents)
+
     list = [File.join(@full_path, '.r10k-deploy.json')].to_set
 
-    list += @puppetfile.managed_directories
+    list += pf_managed_dirs
 
-    list += @puppetfile.desired_contents.flat_map do |item|
+    list += pf_desired_contents.flat_map do |item|
       desired_tree = []
 
       if File.directory?(item)
@@ -163,6 +208,14 @@ class R10K::Environment::Base
     list.to_a
   end
 
+  def purge_exclusions
+    if @purge_exclusions.nil?
+      load_puppetfile_modules
+    end
+
+    @purge_exclusions
+  end
+
   def generate_types!
     argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
     subproc = R10K::Util::Subprocess.new(argv)

data/lib/r10k/environment/with_modules.rb

@@ -23,6 +23,7 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   def initialize(name, basedir, dirname, options = {})
     super
 
+    @all_modules = nil
     @managed_content = {}
     @modules = []
     @moduledir = case options[:moduledir]
@@ -43,10 +44,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   #     - The r10k environment object
   #     - A Puppetfile in the environment's content
   def modules
-    return @modules if puppetfile.nil?
+    if @all_modules.nil?
+      puppetfile_modules = super()
+      @all_modules = @modules + puppetfile_modules
+    end
 
-    puppetfile.load unless puppetfile.loaded?
-    @modules + puppetfile.modules
+    @all_modules
   end
 
   def module_conflicts?(mod_b)
@@ -126,13 +129,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
 
   include R10K::Util::Purgeable
 
-  # Returns an array of the full paths that can be purged.
-  # @note This implements a required method for the Purgeable mixin
-  # @return [Array<String>]
-  def managed_directories
-    [@full_path]
-  end
-
   # Returns an array of the full paths of filenames that should exist. Files
   # inside managed_directories that are not listed in desired_contents will
   # be purged.

data/lib/r10k/git/stateful_repository.rb

@@ -35,6 +35,7 @@ class R10K::Git::StatefulRepository
     @cache.resolve(ref)
   end
 
+  # Returns true if the sync actually updated the repo, false otherwise
   def sync(ref, force=true)
     @cache.sync if sync_cache?(ref)
 
@@ -46,6 +47,7 @@ class R10K::Git::StatefulRepository
 
     workdir_status = status(ref)
 
+    updated = true
     case workdir_status
     when :absent
       logger.debug(_("Cloning %{repo_path} and checking out %{ref}") % {repo_path: @repo.path, ref: ref })
@@ -64,15 +66,20 @@ class R10K::Git::StatefulRepository
         @repo.checkout(sha, {:force => force})
       else
         logger.warn(_("Skipping %{repo_path} due to local modifications") % {repo_path: @repo.path})
+        updated = false
       end
     else
       logger.debug(_("%{repo_path} is already at Git ref %{ref}") % {repo_path: @repo.path, ref: ref })
+      updated = false
     end
+    updated
   end
 
   def status(ref)
     if !@repo.exist?
       :absent
+    elsif !@cache.exist?
+      :mismatched
     elsif !@repo.git_dir.exist?
       :mismatched
     elsif !@repo.git_dir.directory?

data/lib/r10k/initializers.rb

@@ -66,13 +66,7 @@ module R10K
       def call
         with_setting(:baseurl) { |value| PuppetForge.host = value }
         with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value }
-        with_setting(:authorization_token) { |value|
-          if @settings[:baseurl]
-            PuppetForge::Connection.authorization = value
-          else
-            raise R10K::Error, "Cannot specify a Forge authorization token without configuring a custom baseurl."
-          end
-        }
+        with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value }
       end
     end
   end

data/lib/r10k/module/base.rb

@@ -1,9 +1,12 @@
 require 'r10k/module'
+require 'r10k/logging'
 require 'puppet_forge'
 
 # This class defines a common interface for module implementations.
 class R10K::Module::Base
 
+  include R10K::Logging
+
   # @!attribute [r] title
   #   @return [String] The forward slash separated owner and name of the module
   attr_reader :title
@@ -47,7 +50,7 @@ class R10K::Module::Base
 
   # @param title [String]
   # @param dirname [String]
-  # @param args [Array]
+  # @param args [Hash]
   def initialize(title, dirname, args, environment=nil)
     @title   = PuppetForge::V3.normalize_name(title)
     @dirname = dirname
@@ -103,6 +106,7 @@ class R10K::Module::Base
 
   # Synchronize this module with the indicated state.
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     raise NotImplementedError
   end

data/lib/r10k/module/definition.rb

@@ -0,0 +1,64 @@
+require 'r10k/module'
+
+class R10K::Module::Definition < R10K::Module::Base
+
+  attr_reader :version
+
+  def initialize(name, dirname:, args:, implementation:, environment: nil)
+    @original_name  = name
+    @original_args  = args.dup
+    @implementation = implementation
+    @version        = implementation.statically_defined_version(name, args)
+
+    super(name, dirname, args, environment)
+  end
+
+  def to_implementation
+    mod = @implementation.new(@title, @dirname, @original_args, @environment)
+
+    mod.origin = origin
+    mod.spec_deletable = spec_deletable
+
+    mod
+  end
+
+  # syncing is a noop for module definitions
+  # Returns false to inidicate the module was not updated
+  def sync(args = {})
+    logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name})
+    false
+  end
+
+  def status
+    :insync
+  end
+
+  def properties
+    type = nil
+
+    if @args[:type]
+      type = @args[:type]
+    elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag]
+      type = 'git'
+    elsif @args[:svn]
+      # This logic is clear and included for completeness sake, though at
+      # this time module definitions do not support SVN versions.
+      type = 'svn'
+    else
+      type = 'forge'
+    end
+
+    {
+      expected: version,
+      # We can't get the value for `actual` here because that requires the
+      # implementation (and potentially expensive operations by the
+      # implementation). Some consumers will check this value, if it exists
+      # and if not, fall back to the expected version. That is the correct
+      # behavior when assuming modules are unchanged, and why `actual` is set
+      # to `nil` here.
+      actual: nil,
+      type: type
+    }
+  end
+end
+

data/lib/r10k/module/forge.rb

@@ -25,6 +25,10 @@ class R10K::Module::Forge < R10K::Module::Base
     expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
   end
 
+  def self.statically_defined_version(name, args)
+    args[:version] if args[:version].is_a?(String)
+  end
+
   # @!attribute [r] metadata
   #   @api private
   #   @return [PuppetForge::Metadata]
@@ -35,8 +39,6 @@ class R10K::Module::Forge < R10K::Module::Base
   #   @return [PuppetForge::V3::Module] The Puppet Forge module metadata
   attr_reader :v3_module
 
-  include R10K::Logging
-
   include R10K::Util::Setopts
 
   def initialize(title, dirname, opts, environment=nil)
@@ -58,18 +60,24 @@ class R10K::Module::Forge < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :outdated
         upgrade
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       end
       maybe_delete_spec_dir
     end
+    updated
   end
 
   def properties