akerl-r10k 3.14.2.1

data/lib/r10k/content_synchronizer.rb

@@ -0,0 +1,95 @@
+module R10K
+  module ContentSynchronizer
+
+    def self.serial_accept(modules, visitor, loader)
+      visitor.visit(:puppetfile, loader) do
+        serial_sync(modules)
+      end
+    end
+
+    def self.serial_sync(modules)
+      updated_modules = []
+      modules.each do |mod|
+        updated = mod.sync
+        updated_modules << mod.name if updated
+      end
+      updated_modules
+    end
+
+    # Returns a Queue of the names of modules actually updated
+    def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
+      mods_queue = modules_visit_queue(modules, visitor, loader)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    # Returns a Queue of the names of modules actually updated
+    def self.concurrent_sync(modules, pool_size, logger)
+      mods_queue = modules_sync_queue(modules)
+      sync_queue(mods_queue, pool_size, logger)
+    end
+
+    # Returns a Queue of the names of modules actually updated
+    def self.sync_queue(mods_queue, pool_size, logger)
+      logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
+      updated_modules = Queue.new
+      thread_pool = pool_size.times.map { sync_thread(mods_queue, logger, updated_modules) }
+      thread_exception = nil
+
+      # If any threads raise an exception the deployment is considered a failure.
+      # In that event clear the queue, wait for other threads to finish their
+      # current work, then re-raise the first exception caught.
+      begin
+        thread_pool.each(&:join)
+        # Return the list of all modules that were actually updated
+        updated_modules
+      rescue => e
+        logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
+        mods_queue.clear
+        thread_exception ||= e
+        retry
+      ensure
+        raise thread_exception unless thread_exception.nil?
+      end
+    end
+
+    def self.modules_visit_queue(modules, visitor, loader)
+      Queue.new.tap do |queue|
+        visitor.visit(:puppetfile, loader) do
+          enqueue_modules(queue, modules)
+        end
+      end
+    end
+
+    def self.modules_sync_queue(modules)
+      Queue.new.tap do |queue|
+        enqueue_modules(queue, modules)
+      end
+    end
+
+    def self.enqueue_modules(queue, modules)
+      modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
+      modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+      modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+      modules_by_cachedir.values.each {|mods| queue << mods }
+    end
+
+    def self.sync_thread(mods_queue, logger, updated_modules)
+      Thread.new do
+        begin
+          while mods = mods_queue.pop(true) do
+            mods.each do |mod|
+              updated = mod.sync
+              updated_modules << mod.name if updated
+            end
+          end
+        rescue ThreadError => e
+          logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
+          Thread.exit
+        rescue => e
+          Thread.main.raise(e)
+        end
+      end
+    end
+  end
+end

data/lib/r10k/deployment/config.rb

@@ -0,0 +1,56 @@
+require 'r10k/deployment'
+require 'r10k/settings/loader'
+require 'r10k/util/symbolize_keys'
+require 'r10k/errors'
+require 'r10k/initializers'
+require 'yaml'
+
+module R10K
+class Deployment
+class Config
+
+  include R10K::Logging
+
+  attr_accessor :configfile
+
+  def initialize(configfile, overrides={})
+    @configfile = configfile
+    @overrides = overrides
+
+    load_config
+  end
+
+  # Perform a scan for key and check for both string and symbol keys
+  def setting(key)
+    @config[key]
+  end
+
+  alias [] setting
+
+  def settings
+    @config
+  end
+
+  # Load and store a config file, and set relevant options
+  #
+  # @param [String] configfile The path to the YAML config file
+  def load_config
+    loader = R10K::Settings::Loader.new
+    hash = loader.read(@configfile)
+
+    with_overrides = hash.merge(@overrides) do |key, oldval, newval|
+      logger.debug2 _("Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'") % {key: key, old_val: oldval, new_val: newval}
+      newval
+    end
+
+    @config = R10K::Settings.global_settings.evaluate(with_overrides)
+
+    initializer = R10K::Initializers::GlobalInitializer.new(@config)
+    initializer.call
+  end
+
+  class ConfigError < R10K::Error
+  end
+end
+end
+end

data/lib/r10k/deployment.rb

@@ -0,0 +1,132 @@
+require 'r10k/source'
+require 'r10k/util/basedir'
+require 'r10k/errors'
+require 'set'
+
+module R10K
+  # A deployment models the entire state of the configuration that a Puppet
+  # master can use. It contains a set of sources that can produce environments
+  # and manages the contents of directories where environments are deployed.
+  #
+  # @api private
+  class Deployment
+
+    require 'r10k/deployment/config'
+
+    # Generate a deployment object based on a config
+    #
+    # @deprecated
+    #
+    # @param path [String] The path to the deployment config
+    # @return [R10K::Deployment] The deployment loaded with the given config
+    def self.load_config(path, overrides={})
+      config = R10K::Deployment::Config.new(path, overrides)
+      new(config)
+    end
+
+    # @!attribute [r] config
+    #   @return [R10K::Deployment::Config]
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    def preload!
+      sources.each(&:preload!)
+    end
+
+    # Lazily load all sources
+    #
+    # This instantiates the @_sources instance variable, but should not be
+    # used directly as it could be legitimately unset if we're doing lazy
+    # loading.
+    #
+    # @return [Array<R10K::Source::Base>] All repository sources
+    #   specified in the config
+    def sources
+      load_sources if @_sources.nil?
+      @_sources
+    end
+
+    # Lazily load all environments
+    #
+    # This instantiates the @_environments instance variable, but should not be
+    # used directly as it could be legitimately unset if we're doing lazy
+    # loading.
+    #
+    # @return [Array<R10K::Environment::Base>] All enviroments across
+    #   all sources
+    def environments
+      load_environments if @_environments.nil?
+      @_environments
+    end
+
+    # @return [Array<String>] The paths used by all contained sources
+    def paths
+      paths_and_sources.keys
+    end
+
+    # @return [Hash<String, Array<R10K::Source::Base>]
+    def paths_and_sources
+      pathmap = Hash.new { |h, k| h[k] = [] }
+      sources.each { |source| pathmap[source.basedir] << source }
+      pathmap
+    end
+
+    # Remove unmanaged content from all source paths
+    def purge!
+      paths_and_sources.each_pair do |path, sources_at_path|
+        R10K::Util::Basedir.new(path, sources_at_path).purge!
+      end
+    end
+
+    def validate!
+      hash = {}
+      sources.each do |source|
+        source.environments.each do |environment|
+          if hash.key?(environment.path)
+            osource, oenvironment = hash[environment.path]
+            msg = _("Environment collision at %{env_path} between %{source}:%{env_name} and %{osource}:%{oenv_name}") % 
+              {env_path: environment.path,
+               source: source.name,
+               env_name: environment.name,
+               osource: osource.name,
+               oenv_name: oenvironment.name}
+
+            raise R10K::Error, msg
+          else
+            hash[environment.path] = [source, environment]
+          end
+        end
+      end
+    end
+
+    def accept(visitor)
+      visitor.visit(:deployment, self) do
+        sources.each do |source|
+          source.accept(visitor)
+        end
+      end
+    end
+
+    private
+
+    def load_sources
+      sources = @config[:sources]
+      if sources.nil? || sources.empty?
+        raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key")
+      end
+      @_sources = sources.map do |(name, hash)|
+        R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]}))
+      end
+    end
+
+    def load_environments
+      @_environments = []
+      sources.each do |source|
+        @_environments += source.environments
+      end
+    end
+  end
+end

data/lib/r10k/environment/bare.rb

@@ -0,0 +1,13 @@
+class R10K::Environment::Bare < R10K::Environment::Plain
+
+  R10K::Environment.register(:bare, self)
+
+  def initialize(name, basedir, dirname, options = {})
+    logger.warn _('"bare" environment type is deprecated; please use "plain" instead (environment: %{name})') % {name: name}
+    super
+  end
+
+  def signature
+    'bare-default'
+  end
+end

data/lib/r10k/environment/base.rb

@@ -0,0 +1,230 @@
+require 'r10k/content_synchronizer'
+require 'r10k/logging'
+require 'r10k/module_loader/puppetfile'
+require 'r10k/util/cleaner'
+require 'r10k/util/subprocess'
+
+# This class defines a common interface for environment implementations.
+#
+# @since 1.3.0
+class R10K::Environment::Base
+
+  include R10K::Logging
+
+  # @!attribute [r] name
+  #   @return [String] A name for this environment that is unique to the given source
+  attr_reader :name
+
+  # @!attribute [r] basedir
+  #   @return [String] The path that this environment will be created in
+  attr_reader :basedir
+
+  # @!attribute [r] dirname
+  #   @return [String] The directory name for the given environment
+  attr_reader :dirname
+
+  # @!attribute [r] path
+  #   @return [Pathname] The full path to the given environment
+  attr_reader :path
+
+  # @!attribute [r] puppetfile
+  #   @api public
+  #   @return [R10K::Puppetfile] The puppetfile instance associated with this environment
+  attr_reader :puppetfile
+
+  # @!attribute [r] puppetfile_name
+  #   @api public
+  #   @return [String] The puppetfile name (relative)
+  attr_reader :puppetfile_name
+
+  attr_reader :managed_directories, :desired_contents
+
+  attr_reader :loader
+
+  # Initialize the given environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #   The semantics of this environment may depend on the environment implementation.
+  def initialize(name, basedir, dirname, options = {})
+    @name    = name
+    @basedir = basedir
+    @dirname = dirname
+    @options = options
+    @puppetfile_name = options.delete(:puppetfile_name)
+    @overrides = options.delete(:overrides) || {}
+
+    @full_path = File.join(@basedir, @dirname)
+    @path = Pathname.new(File.join(@basedir, @dirname))
+
+    @puppetfile  = R10K::Puppetfile.new(@full_path,
+                                        {overrides: @overrides,
+                                         force: @overrides.dig(:modules, :force),
+                                         puppetfile_name: @puppetfile_name})
+    @puppetfile.environment = self
+
+    loader_options = { basedir: @full_path, overrides: @overrides, environment: self }
+    loader_options[:puppetfile] = @puppetfile_name if @puppetfile_name
+
+    @loader = R10K::ModuleLoader::Puppetfile.new(**loader_options)
+
+    if @overrides.dig(:environments, :incremental)
+      @loader.load_metadata
+    end
+
+    @base_modules = nil
+    @purge_exclusions = nil
+    @managed_directories = [ @full_path ]
+    @desired_contents = []
+  end
+
+  # Synchronize the given environment.
+  #
+  # @api public
+  # @abstract
+  # @return [void]
+  def sync
+    raise NotImplementedError, _("%{class} has not implemented method %{method}") % {class: self.class, method: __method__}
+  end
+
+  # Determine the current status of the environment.
+  #
+  # This can return the following values:
+  #
+  #   * :absent - there is no module installed
+  #   * :mismatched - there is a module installed but it must be removed and reinstalled
+  #   * :outdated - the correct module is installed but it needs to be updated
+  #   * :insync - the correct module is installed and up to date, or the module is actually a boy band.
+  #
+  # @api public
+  # @abstract
+  # @return [Symbol]
+  def status
+    raise NotImplementedError, _("%{class} has not implemented method %{method}") % {class: self.class, method: __method__}
+  end
+
+  # Returns a unique identifier for the environment's current state.
+  #
+  # @api public
+  # @abstract
+  # @return [String]
+  def signature
+    raise NotImplementedError, _("%{class} has not implemented method %{method}") %{class: self.class, method: __method__}
+  end
+
+  # Returns a hash describing the current state of the environment.
+  #
+  # @return [Hash]
+  def info
+    {
+      :name => self.name,
+      :signature => self.signature,
+    }
+  end
+
+  # @return [Array<R10K::Module::Base>] All modules defined in the Puppetfile
+  #   associated with this environment.
+  def modules
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
+
+    @base_modules
+  end
+
+  # @return [Array<R10K::Module::Base>] Whether or not the given module
+  #   conflicts with any modules already defined in the r10k environment
+  #   object.
+  def module_conflicts?(mod)
+    false
+  end
+
+  def accept(visitor)
+    visitor.visit(:environment, self) do
+      puppetfile.accept(visitor)
+    end
+  end
+
+
+  # Returns a Queue of the names of modules actually updated
+  def deploy
+    if @base_modules.nil?
+      load_puppetfile_modules
+    end
+
+    if ! @base_modules.empty?
+      pool_size = @overrides.dig(:modules, :pool_size)
+      updated_modules = R10K::ContentSynchronizer.concurrent_sync(@base_modules, pool_size, logger)
+    end
+
+    if (@overrides.dig(:purging, :purge_levels) || []).include?(:puppetfile)
+      logger.debug("Purging unmanaged Puppetfile content for environment '#{dirname}'...")
+      @puppetfile_cleaner.purge!
+    end
+
+    updated_modules
+  end
+
+  def load_puppetfile_modules
+    loaded_content = @loader.load
+    @base_modules = loaded_content[:modules]
+
+    @purge_exclusions = determine_purge_exclusions(loaded_content[:managed_directories],
+                                                   loaded_content[:desired_contents])
+
+    @puppetfile_cleaner = R10K::Util::Cleaner.new(loaded_content[:managed_directories],
+                                                  loaded_content[:desired_contents],
+                                                  loaded_content[:purge_exclusions])
+  end
+
+  def whitelist(user_whitelist=[])
+    user_whitelist.collect { |pattern| File.join(@full_path, pattern) }
+  end
+
+  def determine_purge_exclusions(pf_managed_dirs     = @puppetfile.managed_directories,
+                                 pf_desired_contents = @puppetfile.desired_contents)
+
+    list = [File.join(@full_path, '.r10k-deploy.json')].to_set
+
+    list += pf_managed_dirs
+
+    list += pf_desired_contents.flat_map do |item|
+      desired_tree = []
+
+      if File.directory?(item)
+        desired_tree << File.join(item, '**', '*')
+      end
+
+      Pathname.new(item).ascend do |path|
+        break if path.to_s == @full_path
+        desired_tree << path.to_s
+      end
+
+      desired_tree
+    end
+
+    list.to_a
+  end
+
+  def purge_exclusions
+    if @purge_exclusions.nil?
+      load_puppetfile_modules
+    end
+
+    @purge_exclusions
+  end
+
+  def generate_types!
+    argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
+    subproc = R10K::Util::Subprocess.new(argv)
+    subproc.raise_on_fail = true
+    subproc.logger = logger
+    result = subproc.execute
+    unless result.stderr.empty?
+      logger.warn "There were problems generating types for environment #{dirname}:"
+      result.stderr.split(%r{\n}).map { |msg| logger.warn msg }
+    end
+  end
+end

data/lib/r10k/environment/git.rb

@@ -0,0 +1,93 @@
+require 'r10k/puppetfile'
+require 'r10k/git/stateful_repository'
+require 'forwardable'
+
+# This class implements an environment based on a Git branch.
+#
+# @since 1.3.0
+class R10K::Environment::Git < R10K::Environment::WithModules
+
+  R10K::Environment.register(:git, self)
+  # Register git as the default environment type
+  R10K::Environment.register(nil, self)
+
+  # @!attribute [r] remote
+  #   @return [String] The URL to the remote git repository
+  attr_reader :remote
+
+  # @!attribute [r] ref
+  #   @return [String] The git reference to use for this environment
+  attr_reader :ref
+
+  # @!attribute [r] repo
+  #   @api private
+  #   @return [R10K::Git::StatefulRepository] The git repo backing this environment
+  attr_reader :repo
+
+  include R10K::Util::Setopts
+
+  # Initialize the given Git environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :remote The URL to the remote git repository
+  # @param options [String] :ref The git reference to use for this environment
+  def initialize(name, basedir, dirname, options = {})
+    super
+    setopts(options, {
+      # Standard option interface
+      :version => :ref,
+      :source  => :remote,
+      :type    => ::R10K::Util::Setopts::Ignore,
+
+      # Type-specific options
+      :ref     => :self,
+      :remote  => :self,
+
+    }, raise_on_unhandled: false)
+    # TODO: in r10k 4.0.0, a major version bump, stop allowing garbage options.
+    # We only allow them now, here, on this object, because prior to adopting
+    # setopts in the constructor, this object type didn't do any validation
+    # checking of options passed, and would permit garbage parameters.
+
+    @repo = R10K::Git::StatefulRepository.new(@remote, @basedir, @dirname)
+  end
+
+  # Clone or update the given Git environment.
+  #
+  # If the environment is being created for the first time, it will
+  # automatically update all modules to ensure that the environment is complete.
+  #
+  # @api public
+  # @return [void]
+  def sync
+    @repo.sync(@ref)
+  end
+
+  def status
+    @repo.status(@ref)
+  end
+
+  # Return a sting which uniquely identifies (per source) the current state of the
+  # environment.
+  #
+  # @api public
+  # @return [String]
+  def signature
+    @repo.head
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths to all the content being managed.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    desired = [File.join(@full_path, '.git')]
+    desired += @repo.tracked_paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
+  end
+end