r10k 3.11.0 → 3.14.0

data/lib/r10k/action/puppetfile/install.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/errors/formatting'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
+require 'r10k/content_synchronizer'
+require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
 require 'r10k/util/cleaner'
 
 module R10K
@@ -10,25 +10,31 @@ module R10K
       class Install < R10K::Action::Base
 
         def call
-          @ok = true
           begin
-            pf = R10K::Puppetfile.new(@root,
-                                      {moduledir: @moduledir,
-                                       puppetfile_path: @puppetfile,
-                                       force: @force || false})
-            pf.load!
-            pf.sync
+            options = { basedir: @root, overrides: { force: @force || false } }
+            options[:moduledir]  = @moduledir  if @moduledir
+            options[:puppetfile] = @puppetfile if @puppetfile
+
+            loader = R10K::ModuleLoader::Puppetfile.new(**options)
+            loaded_content = loader.load!
 
-            R10K::Util::Cleaner.new(pf.managed_directories,
-                                    pf.desired_contents,
-                                    pf.purge_exclusions).purge!
+            pool_size = @settings[:pool_size] || 4
+            modules   = loaded_content[:modules]
+            if pool_size > 1
+              R10K::ContentSynchronizer.concurrent_sync(modules, pool_size, logger)
+            else
+              R10K::ContentSynchronizer.serial_sync(modules, logger)
+            end
 
+            R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                    loaded_content[:desired_contents],
+                                    loaded_content[:purge_exclusions]).purge!
+
+            true
           rescue => e
-            @ok = false
             logger.error R10K::Errors::Formatting.format_exception(e, @trace)
+            false
           end
-
-          @ok
         end
 
         private

data/lib/r10k/action/puppetfile/purge.rb

@@ -1,7 +1,7 @@
-require 'r10k/puppetfile'
-require 'r10k/util/cleaner'
 require 'r10k/action/base'
 require 'r10k/errors/formatting'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
 
 module R10K
   module Action
@@ -9,13 +9,16 @@ module R10K
       class Purge < R10K::Action::Base
 
         def call
-          pf = R10K::Puppetfile.new(@root,
-                                    {moduledir: @moduledir,
-                                     puppetfile_path: @puppetfile})
-          pf.load!
-          R10K::Util::Cleaner.new(pf.managed_directories,
-                                  pf.desired_contents,
-                                  pf.purge_exclusions).purge!
+          options = { basedir: @root }
+
+          options[:moduledir]  = @moduledir  if @moduledir
+          options[:puppetfile] = @puppetfile if @puppetfile
+
+          loader = R10K::ModuleLoader::Puppetfile.new(**options)
+          loaded_content = loader.load!
+          R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                  loaded_content[:desired_contents],
+                                  loaded_content[:purge_exclusions]).purge!
 
           true
         rescue => e

data/lib/r10k/action/runner.rb

@@ -51,6 +51,12 @@ module R10K
           overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
           overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
         end
+        # If the log level has been given as an argument, ensure that output happens on stderr
+        if @opts.key?(:loglevel)
+          overrides[:logging] = {}
+          overrides[:logging][:level] = @opts[:loglevel]
+          overrides[:logging][:disable_default_stderr] = false
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash

data/lib/r10k/action/visitor.rb

@@ -1,4 +1,5 @@
 require 'r10k/errors/formatting'
+require 'r10k/logging'
 
 module R10K
   module Action
@@ -13,6 +14,8 @@ module R10K
     # @api private
     module Visitor
 
+      include R10K::Logging
+
       # Dispatch to the type specific visitor method
       #
       # @param type [Symbol] The object type to dispatch for

data/lib/r10k/cli/deploy.rb

@@ -69,6 +69,7 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
 
           flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
           flag :m, :modules, 'Deploy modules'
+          flag nil, :incremental, 'Used with the --modules flag, only update those modules whose definition has changed or whose definition allows the version to float'
           option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
                  argument: :required
 

data/lib/r10k/cli/puppetfile.rb

@@ -1,5 +1,4 @@
 require 'r10k/cli'
-require 'r10k/puppetfile'
 require 'r10k/action/puppetfile'
 
 require 'cri'

data/lib/r10k/content_synchronizer.rb

@@ -8,24 +8,31 @@ module R10K
     end
 
     def self.serial_sync(modules)
+      updated_modules = []
       modules.each do |mod|
-        mod.sync
+        updated = mod.sync
+        updated_modules << mod.name if updated
       end
+      updated_modules
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
       mods_queue = modules_visit_queue(modules, visitor, loader)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.concurrent_sync(modules, pool_size, logger)
       mods_queue = modules_sync_queue(modules)
       sync_queue(mods_queue, pool_size, logger)
     end
 
+    # Returns a Queue of the names of modules actually updated
     def self.sync_queue(mods_queue, pool_size, logger)
       logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
-      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
+      updated_modules = Queue.new
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) }
       thread_exception = nil
 
       # If any threads raise an exception the deployment is considered a failure.
@@ -33,6 +40,8 @@ module R10K
       # current work, then re-raise the first exception caught.
       begin
         thread_pool.each(&:join)
+        # Return the list of all modules that were actually updated
+        updated_modules
       rescue => e
         logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
         mods_queue.clear
@@ -65,11 +74,14 @@ module R10K
       modules_by_cachedir.values.each {|mods| queue << mods }
     end
 
-    def self.sync_thread(mods_queue, logger)
+    def self.sync_thread(mods_queue, logger, updated_modules)
       Thread.new do
         begin
           while mods = mods_queue.pop(true) do
-            mods.each { |mod| mod.sync }
+            mods.each do |mod|
+              updated = mod.sync
+              updated_modules << mod.name if updated
+            end
           end
         rescue ThreadError => e
           logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}

data/lib/r10k/environment/bare.rb

@@ -1,13 +1,10 @@
-class R10K::Environment::Bare < R10K::Environment::WithModules
+class R10K::Environment::Bare < R10K::Environment::Plain
 
   R10K::Environment.register(:bare, self)
 
-  def sync
-    path.mkpath
-  end
-
-  def status
-    :not_applicable
+  def initialize(name, basedir, dirname, options = {})
+    logger.warn _('"bare" environment type is deprecated; please use "plain" instead (environment: %{name})') % {name: name}
+    super
   end
 
   def signature

data/lib/r10k/environment/base.rb

@@ -1,5 +1,8 @@
-require 'r10k/util/subprocess'
+require 'r10k/content_synchronizer'
 require 'r10k/logging'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
+require 'r10k/util/subprocess'
 
 # This class defines a common interface for environment implementations.
 #
@@ -34,6 +37,10 @@ class R10K::Environment::Base
   #   @return [String] The puppetfile name (relative)
   attr_reader :puppetfile_name
 
+  attr_reader :managed_directories, :desired_contents
+
+  attr_reader :loader
+
   # Initialize the given environment.
   #
   # @param name [String] The unique name describing this environment.
@@ -57,6 +64,20 @@ class R10K::Environment::Base
                                          force: @overrides.dig(:modules, :force),
                                          puppetfile_name: @puppetfile_name})
     @puppetfile.environment = self
+
+    loader_options = { basedir: @full_path, overrides: @overrides, environment: self }
+    loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name
+
+    @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options)
+
+    if @overrides.dig(:environments, :incremental)
+      @loader.load_metadata
+    end
+
+    @base_modules = nil
+    @purge_exclusions = nil
+    @managed_directories = [ @full_path ]
+    @desired_contents = []
   end
 
   # Synchronize the given environment.
@@ -106,8 +127,11 @@ class R10K::Environment::Base
   # @return [Array<R10K::Module::Base>] All modules defined in the Puppetfile
   #   associated with this environment.
   def modules
-    @puppetfile.load
-    @puppetfile.modules
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
+
+    @base_modules
   end
 
   # @return [Array<R10K::Module::Base>] Whether or not the given module
@@ -123,29 +147,50 @@ class R10K::Environment::Base
     end
   end
 
+
+  # Returns a Queue of the names of modules actually updated
   def deploy
-    puppetfile.load(@overrides.dig(:environments, :default_branch_override))
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
 
-    puppetfile.sync
+    if ! @base_modules.empty?
+      pool_size = @overrides.dig(:modules, :pool_size)
+      updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger)
+    end
 
     if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
       logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
-      R10K::Util::Cleaner.new(puppetfile.managed_directories,
-                              puppetfile.desired_contents,
-                              puppetfile.purge_exclusions).purge!
+      @puppetfile_cleaner.purge!
     end
+
+    updated_modules
+  end
+
+  def load_puppetfile_modules
+    loaded_content = @loader.load
+    @base_modules = loaded_content[:modules]
+
+    @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories],
+                                                   loaded_content[:desired_contents])
+
+    @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                                  loaded_content[:desired_contents],
+                                                  loaded_content[:purge_exclusions])
   end
 
   def whitelist(user_whitelist=[])
     user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
   end
 
-  def purge_exclusions
+  def determine_purge_exclusions(pf_managed_dirs     = @puppetfile.managed_directories,
+                                 pf_desired_contents = @puppetfile.desired_contents)
+
     list = [File.join(@full_path, '.r10k-deploy.json')].to_set
 
-    list += @puppetfile.managed_directories
+    list += pf_managed_dirs
 
-    list += @puppetfile.desired_contents.flat_map do |item|
+    list += pf_desired_contents.flat_map do |item|
       desired_tree = []
 
       if File.directory?(item)
@@ -163,6 +208,14 @@ class R10K::Environment::Base
     list.to_a
   end
 
+  def purge_exclusions
+    if @purge_exclusions.nil?
+      load_puppetfile_modules
+    end
+
+    @purge_exclusions
+  end
+
   def generate_types!
     argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
     subproc = R10K::Util::Subprocess.new(argv)

data/lib/r10k/environment/name.rb

@@ -6,17 +6,22 @@ module R10K
     class Name
 
       # @!attribute [r] name
-      #   @return [String] The unmodified name of the environment
+      #   @return [String] The functional name of the environment derived from inputs and options.
       attr_reader :name
 
+      # @!attribute [r] original_name
+      #   @return [String] The unmodified name originally given to create the object.
+      attr_reader :original_name
+
       INVALID_CHARACTERS = %r[\W]
 
-      def initialize(name, opts)
+      def initialize(original_name, opts)
         @source  = opts[:source]
         @prefix  = opts[:prefix]
         @invalid = opts[:invalid]
 
-        @name = derive_name(name, opts[:strip_component])
+        @name = derive_name(original_name, opts[:strip_component])
+        @original_name = original_name
         @opts = opts
 
         case @invalid
@@ -71,8 +76,8 @@ module R10K
 
       private
 
-      def derive_name(name, strip_component)
-        return name unless strip_component
+      def derive_name(original_name, strip_component)
+        return original_name unless strip_component
 
         unless strip_component.is_a?(String)
           raise _('Improper configuration value given for strip_component setting in %{src} source. ' \
@@ -82,11 +87,11 @@ module R10K
 
         if %r{^/.*/$}.match(strip_component)
           regex = Regexp.new(strip_component[1..-2])
-          name.gsub(regex, '')
-        elsif name.start_with?(strip_component)
-          name[strip_component.size..-1]
+          original_name.gsub(regex, '')
+        elsif original_name.start_with?(strip_component)
+          original_name[strip_component.size..-1]
         else
-          name
+          original_name
         end
       end
 

data/lib/r10k/environment/plain.rb

@@ -0,0 +1,16 @@
+class R10K::Environment::Plain < R10K::Environment::WithModules
+
+  R10K::Environment.register(:plain, self)
+
+  def sync
+    path.mkpath
+  end
+
+  def status
+    :not_applicable
+  end
+
+  def signature
+    'plain-default'
+  end
+end

data/lib/r10k/environment/tarball.rb

@@ -0,0 +1,78 @@
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+require 'r10k/environment'
+
+class R10K::Environment::Tarball < R10K::Environment::WithModules
+
+  R10K::Environment.register(:tarball, self)
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  # Initialize the given tarball environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :source Where to get the tarball from
+  # @param options [String] :version The sha256 digest of the tarball
+  def initialize(name, basedir, dirname, options = {})
+    super
+    setopts(options, {
+      # Standard option interface
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :source    => :self,
+      :version   => :checksum,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  def path
+    @path ||= Pathname.new(File.join(@basedir, @dirname))
+  end
+
+  def sync
+    tarball.get unless tarball.cache_valid?
+    case status
+    when :absent, :mismatched
+      tarball.unpack(path.to_s)
+      # Untracked files left behind from previous extractions are expected to
+      # be deleted by r10k's purge facility.
+    end
+  end
+
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s, ignore_untracked_files: true))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  def signature
+    @checksum || @tarball.cache_checksum
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths to all the content being managed.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    desired = []
+    desired += @tarball.paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
+  end
+end

data/lib/r10k/environment/with_modules.rb

@@ -23,6 +23,7 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   def initialize(name, basedir, dirname, options = {})
     super
 
+    @all_modules = nil
     @managed_content = {}
     @modules = []
     @moduledir = case options[:moduledir]
@@ -43,10 +44,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   #     - The r10k environment object
   #     - A Puppetfile in the environment's content
   def modules
-    return @modules if puppetfile.nil?
+    if @all_modules.nil?
+      puppetfile_modules = super()
+      @all_modules = @modules + puppetfile_modules
+    end
 
-    puppetfile.load unless puppetfile.loaded?
-    @modules + puppetfile.modules
+    @all_modules
   end
 
   def module_conflicts?(mod_b)
@@ -126,13 +129,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
 
   include R10K::Util::Purgeable
 
-  # Returns an array of the full paths that can be purged.
-  # @note This implements a required method for the Purgeable mixin
-  # @return [Array<String>]
-  def managed_directories
-    [@full_path]
-  end
-
   # Returns an array of the full paths of filenames that should exist. Files
   # inside managed_directories that are not listed in desired_contents will
   # be purged.

data/lib/r10k/environment.rb

@@ -30,8 +30,10 @@ module R10K
 
     require 'r10k/environment/base'
     require 'r10k/environment/with_modules'
+    require 'r10k/environment/plain'
     require 'r10k/environment/bare'
     require 'r10k/environment/git'
     require 'r10k/environment/svn'
+    require 'r10k/environment/tarball'
   end
 end

data/lib/r10k/errors.rb

@@ -58,4 +58,9 @@ module R10K
       str.gsub(/^/, prefix)
     end
   end
+
+  # An error class for configuration errors
+  #
+  class ConfigError < Error
+  end
 end

data/lib/r10k/forge/module_release.rb

@@ -1,5 +1,6 @@
 require 'r10k/logging'
 require 'r10k/settings/mixin'
+require 'r10k/util/cacheable'
 require 'fileutils'
 require 'tmpdir'
 require 'puppet_forge'
@@ -13,7 +14,7 @@ module R10K
 
       def_setting_attr :proxy
       def_setting_attr :baseurl
-      def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache')
+      def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
 
       include R10K::Logging
 

data/lib/r10k/git/cache.rb

@@ -3,6 +3,7 @@ require 'r10k/git'
 require 'r10k/settings'
 require 'r10k/instance_cache'
 require 'forwardable'
+require 'r10k/util/cacheable'
 
 # Cache Git repository mirrors for object database reuse.
 #
@@ -15,18 +16,9 @@ require 'forwardable'
 class R10K::Git::Cache
 
   include R10K::Settings::Mixin
+  include R10K::Util::Cacheable
 
-  #@api private
-  def self.determine_cache_root
-    if R10K::Util::Platform.windows?
-      File.join(ENV['LOCALAPPDATA'], 'r10k', 'git')
-    else
-      File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
-    end
-  end
-  private_class_method :determine_cache_root
-
-  def_setting_attr :cache_root, determine_cache_root
+  def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir('git')
 
   @instance_cache = R10K::InstanceCache.new(self)
 
@@ -109,8 +101,7 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= super(@remote)
   end
 end

data/lib/r10k/git/rugged/base_repository.rb

@@ -20,7 +20,8 @@ class R10K::Git::Rugged::BaseRepository
     else
       object.oid
     end
-  rescue ::Rugged::ReferenceError
+  rescue ::Rugged::ReferenceError, ::Rugged::OdbError => e
+    logger.debug2(_("Unable to resolve %{pattern}: %{e} ") % {pattern: pattern, e: e })
     nil
   end
 
@@ -60,6 +61,16 @@ class R10K::Git::Rugged::BaseRepository
     remotes_hash
   end
 
+  # Update a remote URL
+  # @param [String] The remote URL of the git repository
+  # @param [String] An optional remote name for the git repository
+  def update_remote(remote, remote_name='origin')
+    if @_rugged_repo
+      logger.debug2(_("Remote URL is different from cache, updating %{orig} to %{update}") % {orig: remotes[remote_name], update: remote})
+      @_rugged_repo.remotes.set_url(remote_name, remote)
+    end
+  end
+
   private
 
   def with_repo(opts={})

data/lib/r10k/git/rugged/cache.rb

@@ -8,4 +8,12 @@ class R10K::Git::Rugged::Cache < R10K::Git::Cache
   def self.bare_repository
     R10K::Git::Rugged::BareRepository
   end
+
+  # Update the remote URL if the cache differs from the current configuration
+  def sync!
+    if cached? && @repo.remotes['origin'] != @remote
+      @repo.update_remote(@remote)
+    end
+    super
+  end
 end